Verified Commit 0862adf7 authored by 35V LG84's avatar 35V LG84

Merge branch '35VLG84/tep/tep-1010-txn-geo-loc'

See merge request: !12Signed-off-by: 35V LG84's avatar35V LG84 <35vlg84-x4e6b92@e257.fi>
parents 07e09249 ff6ab39a
Pipeline #60180704 passed with stage
in 5 minutes and 39 seconds
......@@ -13,7 +13,10 @@ Current published release is:
New features and changes in this release:
* link:https://tackler.e257.fi/docs/journal/format/#value-pos[Closing position with total amount (`=` syntax)]
* Support for link:https://tackler.e257.fi/docs/gis/[Geographic Information System (GIS)]
** See link:https://tackler.e257.fi/docs/gis/txn-geo-location/[Transaction Geo Location]
** See link:https://tackler.e257.fi/docs/gis/txn-geo-filter/[Transaction Geo Filter]
* Add support for Value Position with link:https://tackler.e257.fi/docs/journal/format/#value-pos[total amount (`=` syntax)]
==== Fixes
......
/*
* Copyright 2019 E257.FI
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package fi.e257.tackler.api
import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder}
import io.circe.{Decoder, Encoder}
import scala.util.{Failure, Success, Try}
/**
* WGS84 (EPSG:4326) based Geo Location
* This is based on simplified geo uri scheme, see TEP-1010 for details.
*
* @param lat Latitude in decimal degrees
* @param lon Longitude in decimal degrees
* @param alt optional altitude, in meters
*/
class GeoPoint protected (val lat: BigDecimal, val lon: BigDecimal, val alt: Option[BigDecimal]) {
if (lat < -90 || 90 < lat) {
throw new IllegalArgumentException("Value out of specification for Latitude: " + GeoPoint.frmt(lat))
}
if (lon < -180 || 180 < lon) {
throw new IllegalArgumentException("Value out of specification for Longitude: " + GeoPoint.frmt(lon))
}
alt.foreach(z => {
if (z < -6378137) {
// Jules Verne: Voyage au centre de la Terre
throw new IllegalArgumentException("Value Out of specification for Altitude: " + GeoPoint.frmt(z))
}
})
override def toString: String = {
"geo:" + GeoPoint.frmt(lat) + "," + GeoPoint.frmt(lon) + alt.map("," + GeoPoint.frmt(_)).getOrElse("")
}
}
object GeoPoint {
def frmt(v: BigDecimal): String = {
s"%.${v.scale}f".format(v)
}
/**
* Create [[GeoPoint]] from given coordinates
*
* @param lat Latitude in decimal degrees
* @param lon Longitude in decimal degrees
* @param alt optional altitude, in meters
* @return Success with GeoPoint or Failure
*/
def toPoint(lat: BigDecimal, lon: BigDecimal, alt: Option[BigDecimal]): Try[GeoPoint] = {
try {
Success(new GeoPoint(lat, lon, alt))
} catch {
case ex: IllegalArgumentException => Failure[GeoPoint](ex)
}
}
implicit val decodeTxnHeader: Decoder[GeoPoint] = deriveDecoder
implicit val encodeTxnHeader: Encoder[GeoPoint] = deriveEncoder
}
/*
* Copyright 2019 E257.FI
* Copyright 2018-2019 E257.FI
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......@@ -158,12 +158,17 @@ sealed abstract class TxnFilterRegex(regex: String) extends TxnFilter {
Seq(indent + target + ": " + "\"" + s"${regex}" + "\"")
}
}
/**
* Selects transaction if txn timestamp is on or after specified time.
*
* @param begin txn timestamp must be on or after this
*/
/**
* Select transaction if regular expression matches txn code.
*
* Used regular expression engine is java.util.regex.Pattern.
*
* @param regex to match txn code.
*/
final case class TxnFilterTxnCode(regex: String) extends TxnFilterRegex(regex) {
val target = "Txn Code"
}
/**
* Select transaction if regular expression matches txn description.
......@@ -178,25 +183,127 @@ final case class TxnFilterTxnDescription(regex: String) extends TxnFilterRegex(r
}
/**
* Select transaction if regular expression matches txn code.
* Select transaction if txn UUID is same as specified uuid.
*
* Used regular expression engine is java.util.regex.Pattern.
* @param uuid
*/
final case class TxnFilterTxnUUID(uuid: UUID) extends TxnFilter {
override def text(indent: String): Seq[String] = {
Seq(indent + "Txn UUID: " + uuid.toString)
}
}
sealed abstract class BBoxLatLon extends TxnFilter {
val south: BigDecimal
val west: BigDecimal
val north: BigDecimal
val east: BigDecimal
if (north < south) {
throw new IllegalArgumentException("Invalid Bounding Box: North is below South. " +
"South: " + GeoPoint.frmt(south) + "; North: " + GeoPoint.frmt(north))
}
// east < west case is needed for filter over 180th meridian
// other pole is tested by north < south check
if (south < -90) {
throw new IllegalArgumentException("Invalid Bounding Box: South is beyond pole. " +
"South: " + GeoPoint.frmt(south))
}
// other pole is tested by north < south check
if (90 < north) {
throw new IllegalArgumentException("Invalid Bounding Box: North is beyond pole. " +
"North: " + GeoPoint.frmt(north))
}
if (west < -180 || 180 < west) {
throw new IllegalArgumentException("Invalid Bounding Box: West is beyond 180th Meridian. " +
"West: " + GeoPoint.frmt(west))
}
if (east < -180 || 180 < east) {
throw new IllegalArgumentException("Invalid Bounding Box: East is beyond 180th Meridian. " +
"East: " + GeoPoint.frmt(east))
}
}
/**
* Select Transaction if it has location attribute,
* and it's geo location is inside Bounding Box.
*
* This will ignore altitude, e.g. it will select 3D transaction if it fits 2D BBox.
* If transaction doesn't have location information, it will not be selected.
*
* @param regex to match txn code.
* @param south bbox bottom edge
* @param west bbox left edge
* @param north bbox top edge
* @param east bbox right edge
*/
final case class TxnFilterTxnCode(regex: String) extends TxnFilterRegex(regex) {
val target = "Txn Code"
final case class TxnFilterBBoxLatLon(
south: BigDecimal,
west: BigDecimal,
north: BigDecimal,
east: BigDecimal
) extends BBoxLatLon {
override def text(indent: String): Seq[String] = {
val myIndent = indent + " "
Seq(
indent + "Txn Bounding Box 2D",
myIndent + "North, East: " + "geo:" + GeoPoint.frmt(north) + "," + GeoPoint.frmt(east),
myIndent + "South, West: " + "geo:" + GeoPoint.frmt(south) + "," + GeoPoint.frmt(west),
)
}
}
/**
* Select transaction if txn UUID is same as specified uuid.
* Select Transaction if it has location attribute,
* and it's geo location is inside Bounding Box.
*
* @param uuid
* This will select only transactions with altitude,
* e.g. it will not select any 2D transaction, even if it fits 2D BBox.
*
* If transaction doesn't have location information, it will not be selected.
*
* @param south bbox bottom edge
* @param west bbox left dege
* @param depth bbox floor
* @param north bbox top edge
* @param east bbox right edge
* @param height bbox ceiling
*/
final case class TxnFilterTxnUUID(uuid: UUID) extends TxnFilter {
final case class TxnFilterBBoxLatLonAlt(
south: BigDecimal,
west: BigDecimal,
depth: BigDecimal,
north: BigDecimal,
east: BigDecimal,
height: BigDecimal
) extends BBoxLatLon {
if (height < depth) {
throw new IllegalArgumentException("Invalid Bounding Box: height is less than depth. " +
"Depth: " + GeoPoint.frmt(depth) + "; Height: " + GeoPoint.frmt(height))
}
// height is tested by height < depth test
if (depth < -6378137) {
throw new IllegalArgumentException("Invalid Bounding Box: Depth is beyond center of Earth. " +
"Depth: " + GeoPoint.frmt(depth))
}
override def text(indent: String): Seq[String] = {
Seq(indent + "Txn UUID: " + uuid.toString)
val myIndent = indent + " "
Seq(
indent + "Txn Bounding Box 3D",
myIndent + "North, East, Height: " + "geo:" + GeoPoint.frmt(north) + "," + GeoPoint.frmt(east) + "," + GeoPoint.frmt(height),
myIndent + "South, West, Depth: " + "geo:" + GeoPoint.frmt(south) + "," + GeoPoint.frmt(west) + "," + GeoPoint.frmt(depth)
)
}
}
......
......@@ -31,6 +31,7 @@ final case class TxnHeader(
code: Option[String],
description: Option[String],
uuid: Option[UUID],
location: Option[GeoPoint],
comments: Option[List[String]]
) {
......@@ -86,12 +87,14 @@ final case class TxnHeader(
val codeStr = code.map(c => " (" + c + ") ")
val uuidStr = uuid.map(u => indent + "# uuid: " + u.toString + "\n")
val locStr = location.map(loc => indent + "# location: " + loc.toString + "\n")
val commentsStr = comments.map(cs =>
cs.map(c => indent + "; " + c + "\n").mkString
)
tsFormatter(timestamp) + codeStr.getOrElse(" ") + description.fold("")("'" + _) + "\n" +
uuidStr.getOrElse("") +
locStr.getOrElse("") +
commentsStr.getOrElse("")
}
}
......
......@@ -21,13 +21,13 @@ import java.nio.file.{Files, NoSuchFileException, Path, Paths}
import java.util.Base64
import better.files._
import org.slf4j.{Logger, LoggerFactory}
import io.circe.parser.decode
import fi.e257.tackler.api.TxnFilterDefinition
import fi.e257.tackler.core.{FilesystemStorageType, GitStorageType, Settings, TacklerException, TxnException}
import fi.e257.tackler.core._
import fi.e257.tackler.model.TxnData
import fi.e257.tackler.parser.{TacklerParseException, TacklerTxns}
import fi.e257.tackler.report.Reports
import io.circe.parser.decode
import org.slf4j.{Logger, LoggerFactory}
......
......@@ -209,6 +209,30 @@ class DirsuiteCoreTest extends DirSuiteLike {
}
}
/**
* Location
*/
class DirsuiteLocationTest extends DirSuiteLike {
val basedir = Paths.get("tests")
runDirSuiteTestCases(basedir, Glob("location/ex/TacklerParseException-*.exec")) { args: Array[String] =>
assertThrows[TacklerParseException] {
TacklerCli.runExceptions(args)
}
}
runDirSuiteTestCases(basedir, Glob("location/ex/TacklerException-*.exec")) { args: Array[String] =>
assertThrows[TacklerException] {
TacklerCli.runExceptions(args)
}
}
runDirSuiteTestCases(basedir, Glob("location/ok/*.exec")) { args: Array[String] =>
assertResult(TacklerCli.SUCCESS) {
TacklerCli.runReturnValue(args)
}
}
}
/**
* Parser
......
......@@ -17,6 +17,8 @@
lexer grammar TxnLexer;
UUID_NAME: 'uuid';
LOCATION_NAME: 'location';
GEO_NAME: 'geo';
UUID_VALUE: HEX HEX HEX HEX HEX HEX HEX HEX '-' HEX HEX HEX HEX '-' HEX HEX HEX HEX '-' HEX HEX HEX HEX '-' HEX HEX HEX HEX HEX HEX HEX HEX HEX HEX HEX HEX;
......@@ -26,7 +28,7 @@ TS_TZ: TS TZ;
INT: DIGIT+;
NUMBER: ('+' | '-')? (INT | FLOAT);
NUMBER: '-'? (INT | FLOAT);
ID: NameStartChar (NameChar)*;
......@@ -88,6 +90,7 @@ AT: '@';
EQUAL: '=';
SPACE: ' ';
TAB: '\t';
COMMA: ',';
SEMICOLON: ';';
COLON: ':';
NL: '\r'? '\n';
......
......@@ -39,10 +39,24 @@ description: sp '\'' text;
text: ~(NL)*;
txn_meta: indent '#' sp txn_meta_uuid NL;
txn_meta:
txn_meta_uuid NL
| txn_meta_uuid NL txn_meta_location NL
| txn_meta_location NL
| txn_meta_location NL txn_meta_uuid NL
;
txn_meta_uuid: indent '#' sp UUID_NAME ':' sp UUID_VALUE opt_sp;
txn_meta_location: indent '#' sp LOCATION_NAME ':' sp geo_uri opt_sp;
geo_uri: GEO_NAME ':' lat ',' lon (',' alt)?;
lat: INT | NUMBER;
lon: INT | NUMBER;
txn_meta_uuid: UUID_NAME ':' sp UUID_VALUE opt_sp;
alt: INT | NUMBER;
txn_comment: indent comment NL;
......@@ -71,9 +85,9 @@ opt_opening_pos: sp '{' opt_sp amount sp unit opt_sp '}';
closing_pos: sp ('@' | '=') sp amount sp unit;
account: ID (':' (ID|SUBID|INT))*;
account: ID (':' (ID | SUBID | INT))*;
amount: INT|NUMBER;
amount: INT | NUMBER;
unit: ID;
......
......@@ -281,8 +281,8 @@ class Settings(optPath: Option[Path], providedConfig: Config) {
val accounts: List[String] = getReportAccounts(keys.accounts)
// todo: this is lazy evaluated?
// to trigger, remove output from
// tests/reporting/ex/GroupByException-unknown-group-by.exec
// test:uuid: 31e0bd80-d4a9-4d93-915d-fa2424aedb84
// test: 31e0bd80-d4a9-4d93-915d-fa2424aedb84
// exec: tests/reporting/ex/GroupByException-unknown-group-by.exec
val groupBy: GroupBy = GroupBy(cfg.getString(keys.groupBy))
}
......
/*
* Copyright 2018 E257.FI
* Copyright 2018-2019 E257.FI
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......@@ -16,8 +16,22 @@
*/
package fi.e257.tackler.filter
import fi.e257.tackler.api.{BBoxLatLon, GeoPoint}
import fi.e257.tackler.model.Transaction
trait CanTxnFilter[A] {
def filter(tf: A, txn: Transaction): Boolean
}
trait CanBBoxLatLonFilter[A <: BBoxLatLon] {
def bbox2d(tf: A, geo: GeoPoint): Boolean = {
if (tf.east < tf.west) {
// BBox is over 180th meridian
tf.south <= geo.lat && geo.lat <= tf.north &&
(tf.west <= geo.lon || geo.lon <= tf.east)
} else {
tf.south <= geo.lat && geo.lat <= tf.north &&
tf.west <= geo.lon && geo.lon <= tf.east
}
}
}
/*
* Copyright 2018 E257.FI
* Copyright 2019 E257.FI
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......@@ -16,9 +16,9 @@
*/
package fi.e257.tackler
import cats.implicits._
import fi.e257.tackler.api._
import fi.e257.tackler.model.Transaction
import cats.implicits._
package object filter {
......@@ -41,6 +41,8 @@ package object filter {
case tf: TxnFilterTxnCode => TxnFilterTxnCodeF.filter(tf, txn)
case tf: TxnFilterTxnDescription => TxnFilterTxnDescriptionF.filter(tf, txn)
case tf: TxnFilterTxnUUID => TxnFilterTxnUUIDF.filter(tf, txn)
case tf: TxnFilterBBoxLatLon => TxnFilterBBoxLatLonF.filter(tf, txn)
case tf: TxnFilterBBoxLatLonAlt => TxnFilterBBoxLatLonAltF.filter(tf, txn)
case tf: TxnFilterTxnComments => TxnFilterTxnCommentsF.filter(tf, txn)
// TXN Postings
......@@ -137,6 +139,42 @@ package object filter {
}
}
implicit object TxnFilterBBoxLatLonF
extends CanTxnFilter[TxnFilterBBoxLatLon]
with CanBBoxLatLonFilter[TxnFilterBBoxLatLon] {
override def filter(tf: TxnFilterBBoxLatLon, txn: Transaction): Boolean = {
txn.header.location match {
case Some(geo) => bbox2d(tf, geo)
case None => false
}
}
}
implicit object TxnFilterBBoxLatLonAltF
extends CanTxnFilter[TxnFilterBBoxLatLonAlt]
with CanBBoxLatLonFilter[TxnFilterBBoxLatLonAlt] {
override def filter(tf: TxnFilterBBoxLatLonAlt, txn: Transaction): Boolean = {
txn.header.location match {
case Some(geo) => {
geo.alt match {
case Some(z) => {
if (bbox2d(tf, geo) === false) {
false
}
else {
tf.depth <= z && z <= tf.height
}
}
case None => false // geo but no alt
}
}
case None => false // no geo
}
}
}
implicit object TxnFilterTxnCommentsF extends CanTxnFilter[TxnFilterTxnComments] {
override def filter(tf: TxnFilterTxnComments, txn: Transaction): Boolean = {
......
......@@ -17,16 +17,18 @@
package fi.e257.tackler.parser
import java.time.{LocalDate, LocalDateTime, ZonedDateTime}
import java.util.UUID
import cats.implicits._
import scala.collection.JavaConverters
import fi.e257.tackler.api.TxnHeader
import fi.e257.tackler.core.{AccountException, CfgKeys, CommodityException, Settings, TxnException}
import fi.e257.tackler.api.{GeoPoint, TxnHeader}
import fi.e257.tackler.core.{AccountException, CfgKeys, CommodityException, Settings, TacklerException, TxnException}
import fi.e257.tackler.model.{AccountTreeNode, Commodity, Posting, Posts, Transaction, Txns}
import fi.e257.tackler.parser.TxnParser._
import org.slf4j.{Logger, LoggerFactory}
import scala.util.{Failure, Success}
import scala.util.control.NonFatal
/**
......@@ -109,7 +111,7 @@ abstract class CtxHandler {
}
protected def handleAmount(amountCtx: AmountContext): BigDecimal = {
BigDecimal(Option(amountCtx.INT()).getOrElse(amountCtx.NUMBER()).getText())
BigDecimal(amountCtx.getText())
}
protected def handleValuePosition(postingCtx: PostingContext): (
......@@ -164,7 +166,15 @@ abstract class CtxHandler {
// Ok, we have closing position
Option(cp.AT()).fold({
// this is '=', e.g. total price
(handleAmount(cp.amount()), true)
val total_price = handleAmount(cp.amount())
if ((total_price < 0 && 0 <= postAmount) || (postAmount < 0 && 0 <= total_price)) {
val lineNro = postingCtx.start.getLine
val msg = "Error on line: " + lineNro.toString +
"; Value position (total price) has different sign than primary posting value"
log.error(msg)
throw new CommodityException(msg)
}
(total_price, true)
})(_ => {
// this is '@', e.g. unit price
(postAmount * handleAmount(cp.amount()), false)
......@@ -225,6 +235,26 @@ abstract class CtxHandler {
Posting(acctn, foo._1, foo._2, foo._3, foo._5, comment)
}
protected def handleMeta(metaCtx: Txn_metaContext): (Option[UUID], Option[GeoPoint]) = {
val uuid = Option(metaCtx.txn_meta_uuid()).map(muuid => {
java.util.UUID.fromString(muuid.UUID_VALUE().getText)
})
val geo = Option(metaCtx.txn_meta_location()).map(geoCtx => {
GeoPoint.toPoint(
BigDecimal(geoCtx.geo_uri().lat().getText),
BigDecimal(geoCtx.geo_uri().lon().getText()),
Option(geoCtx.geo_uri().alt()).map(a => BigDecimal(a.getText))
) match {
case Success(g) => g
case Failure(ex) => {
log.error("Invalid geo-uri:" + ex.getMessage)
throw new TacklerException("Invalid geo-uri: " + ex.getMessage)
}
}
})
(uuid, geo)
}
/**
* Handle one Transaction (txn -rule).
*
......@@ -250,9 +280,12 @@ abstract class CtxHandler {
})
val uuid = Option(txnCtx.txn_meta()).map(meta => {
java.util.UUID.fromString(meta.txn_meta_uuid().UUID_VALUE().getText)
val meta = Option(txnCtx.txn_meta())
.fold[(Option[UUID], Option[GeoPoint])]((None, None))(metaCtx => {
handleMeta(metaCtx)
})
val uuid: Option[UUID] = meta._1
val geo: Option[GeoPoint] = meta._2
if (settings.Auditing.txnSetChecksum && uuid.isEmpty) {
val msg = "" +
......@@ -296,7 +329,7 @@ abstract class CtxHandler {
List(Posting(ate, -amount, -amount, false, posts.head.txnCommodity, comment))
})
Transaction(TxnHeader(date, code, desc, uuid, comments), posts ++ last_posting.getOrElse(Nil))
Transaction(TxnHeader(date, code, desc, uuid, geo, comments), posts ++ last_posting.getOrElse(Nil))
}
/**
......
......@@ -33,7 +33,7 @@ class StorageTypeTest extends FlatSpec {
}
/**
* test:uuid: 195971d7-f16f-4c1c-a761-6764b28fd4db
* test: 195971d7-f16f-4c1c-a761-6764b28fd4db
*/
it should "handle unknown type" in {
assertThrows[TacklerException]{
......
/*
* Copyright 2019 E257.FI
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package fi.e257.tackler.filter
import fi.e257.tackler.api.{GeoPoint, TxnFilterBBoxLatLonAlt, TxnFilterDefinition, TxnHeader}
import fi.e257.tackler.model.Transaction
import io.circe.parser.decode
import org.scalatest.FunSpec
class TxnFilterBBoxLatLonAltTest extends FunSpec with TxnFilterBBoxSpec {
describe("BBox 3D (Latitude, Longitude, Altitude)") {
/**
* test: 00d5f743-4eca-4d06-a5e5-4de035909828
*/
it("Filter 2D Txns") {
val geo2DTxnStr =
s"""
|2019-02-02 'Helsinki
| # uuid: ${geo03}
| # location: geo:60.170833,24.9375
| e 1
| a
|
|""".stripMargin
val geo2DTxnData = tt.string2Txns(geo2DTxnStr)
val txnFilter = TxnFilterBBoxLatLonAlt(
40, 20, -2000, 65, 26, 14000
)
val txnData = geo2DTxnData.filter(TxnFilterDefinition(txnFilter))
assert(txnData.txns.size === 0)
}
/**
* test: 607d4e0e-e05b-43cf-87b6-d3cad309be73
*/
it ("Filter 3D Txns") {
val geo3DTxnData = tt.string2Txns(geo3DTxnStr)
val txnFilter = TxnFilterBBoxLatLonAlt(
40, 20,-2000, 65, 26, 14000,