From ba8c35addc3b5045370abf830c932ec3a43d5736 Mon Sep 17 00:00:00 2001 From: Manfred Zingl Date: Wed, 15 May 2024 11:09:26 +0200 Subject: [PATCH 1/7] add row annotation info to cell --- .../tableaux/database/domain/cell.scala | 17 +++++++++++++++-- .../tableaux/database/model/TableauxModel.scala | 5 ++++- .../database/domain/domainobjectTest.scala | 15 ++++++++++----- 3 files changed, 29 insertions(+), 8 deletions(-) diff --git a/src/main/scala/com/campudus/tableaux/database/domain/cell.scala b/src/main/scala/com/campudus/tableaux/database/domain/cell.scala index fbc572ab..d70f9f8b 100644 --- a/src/main/scala/com/campudus/tableaux/database/domain/cell.scala +++ b/src/main/scala/com/campudus/tableaux/database/domain/cell.scala @@ -4,7 +4,20 @@ import com.campudus.tableaux.database.model.TableauxModel._ import org.vertx.scala.core.json._ -case class Cell[A](column: ColumnType[A], rowId: RowId, value: A) extends DomainObject { +case class Cell[A](column: ColumnType[A], rowId: RowId, value: A, rowLevelAnnotations: RowLevelAnnotations) + extends DomainObject { - override def getJson: JsonObject = Json.obj("value" -> compatibilityGet(value)) + override def getJson: JsonObject = { + val finalFlagJson = rowLevelAnnotations.finalFlag match { + case true => Json.obj("final" -> rowLevelAnnotations.finalFlag) + case false => Json.emptyObj() + } + + val archivedFlagJson = rowLevelAnnotations.archivedFlag match { + case true => Json.obj("archived" -> rowLevelAnnotations.archivedFlag) + case false => Json.emptyObj() + } + + Json.obj("value" -> compatibilityGet(value)).mergeIn(finalFlagJson).mergeIn(archivedFlagJson) + } } diff --git a/src/main/scala/com/campudus/tableaux/database/model/TableauxModel.scala b/src/main/scala/com/campudus/tableaux/database/model/TableauxModel.scala index 7896fa47..ca7cd26c 100644 --- a/src/main/scala/com/campudus/tableaux/database/model/TableauxModel.scala +++ b/src/main/scala/com/campudus/tableaux/database/model/TableauxModel.scala @@ -1017,8 +1017,11 @@ class TableauxModel( } else { Future.successful(value) } + + // TODO use cache for rowLevelAnnotations + (rowLevelAnnotations, _, _) <- retrieveRowModel.retrieveAnnotations(column.table.id, rowId, Seq(column)) } yield { - Cell(column, rowId, resultValue) + Cell(column, rowId, resultValue, rowLevelAnnotations) } } diff --git a/src/test/scala/com/campudus/tableaux/database/domain/domainobjectTest.scala b/src/test/scala/com/campudus/tableaux/database/domain/domainobjectTest.scala index a5304d18..2950629a 100644 --- a/src/test/scala/com/campudus/tableaux/database/domain/domainobjectTest.scala +++ b/src/test/scala/com/campudus/tableaux/database/domain/domainobjectTest.scala @@ -42,16 +42,21 @@ class DomainObjectTest { @Test def nestedCells(): Unit = { - val innerCell1 = Cell(null, 1, 1) - val innerCell2 = Cell(null, 1, 2) - val innerCell3 = Cell(null, 1, 3) - val cell = Cell(null, 1, Seq(innerCell1, innerCell2, innerCell3)) + val innerRowLevelAnnotations1 = RowLevelAnnotations(finalFlag = true, archivedFlag = false) + val innerRowLevelAnnotations2 = RowLevelAnnotations(finalFlag = false, archivedFlag = true) + val innerRowLevelAnnotations3 = RowLevelAnnotations(finalFlag = false, archivedFlag = false) + val rowLevelAnnotations = RowLevelAnnotations(finalFlag = true, archivedFlag = true) + val innerCell1 = Cell(null, 1, 1, innerRowLevelAnnotations1) + val innerCell2 = Cell(null, 1, 2, innerRowLevelAnnotations2) + val innerCell3 = Cell(null, 1, 3, innerRowLevelAnnotations3) + val cell = Cell(null, 1, Seq(innerCell1, innerCell2, innerCell3), rowLevelAnnotations) val actual = DomainObject.compatibilityGet(cell) + println(actual.toString()) JSONAssert .assertEquals( - s"""{"value": [{"value": 1}, {"value": 2}, {"value": 3}]}""", + s"""{"value": [{"value":1, "final":true}, {"value":2, "archived":true}, {"value":3}], "final":true, "archived":true}""", actual.toString, JSONCompareMode.STRICT ) From de751e5cc67af57e313fa5d78f123c448517d12a Mon Sep 17 00:00:00 2001 From: Manfred Zingl Date: Wed, 15 May 2024 13:33:07 +0200 Subject: [PATCH 2/7] add more archive tests --- .../api/content/LinkConstraintTest.scala | 129 ++++++++++++++---- 1 file changed, 104 insertions(+), 25 deletions(-) diff --git a/src/test/scala/com/campudus/tableaux/api/content/LinkConstraintTest.scala b/src/test/scala/com/campudus/tableaux/api/content/LinkConstraintTest.scala index 3ef4c8bd..45ccb66f 100644 --- a/src/test/scala/com/campudus/tableaux/api/content/LinkConstraintTest.scala +++ b/src/test/scala/com/campudus/tableaux/api/content/LinkConstraintTest.scala @@ -52,6 +52,31 @@ sealed trait Helper extends LinkTestBase { .map(_.getJsonObject(0)) .map(_.getLong("id")) } + + def createArchiveCascadeLinkColumn( + tableId: TableId, + toTableId: TableId, + columnName: String + ): Future[ColumnId] = { + val columns = Columns( + LinkBiDirectionalCol(columnName, toTableId, Constraint(DefaultCardinality, archiveCascade = true)) + ) + + sendRequest("POST", s"/tables/$tableId/columns", columns) + .map(_.getJsonArray("columns")) + .map(_.getJsonObject(0)) + .map(_.getLong("id")) + } + + def filterArchivedRows(jsonArray: JsonArray): Seq[JsonObject] = { + import scala.collection.JavaConverters._ + + jsonArray.asScala + .collect({ + case obj: JsonObject => obj + }).toSeq + .filter(_.getBoolean("archived")) + } } @RunWith(classOf[VertxUnitRunner]) @@ -1408,31 +1433,6 @@ class LinkArchiveCascadeTest extends LinkTestBase with Helper { } } } - - def filterArchivedRows(jsonArray: JsonArray): Seq[JsonObject] = { - import scala.collection.JavaConverters._ - - jsonArray.asScala - .collect({ - case obj: JsonObject => obj - }).toSeq - .filter(_.getBoolean("archived")) - } - - private def createArchiveCascadeLinkColumn( - tableId: TableId, - toTableId: TableId, - columnName: String - ): Future[ColumnId] = { - val columns = Columns( - LinkBiDirectionalCol(columnName, toTableId, Constraint(DefaultCardinality, archiveCascade = true)) - ) - - sendRequest("POST", s"/tables/$tableId/columns", columns) - .map(_.getJsonArray("columns")) - .map(_.getJsonObject(0)) - .map(_.getLong("id")) - } } @RunWith(classOf[VertxUnitRunner]) @@ -1682,7 +1682,11 @@ class RetrieveFinalAndArchivedRows extends LinkTestBase with Helper { sendRequest("GET", s"/tables/$tableId1/columns/$linkColumnId/rows/$rowId1/foreignRows?final=true") resultForeignRowsArchived <- sendRequest("GET", s"/tables/$tableId1/columns/$linkColumnId/rows/$rowId1/foreignRows?archived=true") + + rrr <- sendRequest("GET", s"/tables/$tableId1/columns/$linkColumnId/rows/$rowId1") } yield { + println(s"rrr: $rrr") + assertEquals(0, resultCell.getJsonArray("value").size()) assertEquals(2, resultForeignRowsFinal.getJsonObject("page").getLong("totalSize").longValue()) @@ -1692,4 +1696,79 @@ class RetrieveFinalAndArchivedRows extends LinkTestBase with Helper { } } } + + @Test + def retrieveSingleArchivedRowAndCell(implicit c: TestContext): Unit = { + okTest { + for { + tableId1 <- createDefaultTable(name = "table1") + _ <- sendRequest("PATCH", s"/tables/1/rows/1/annotations", Json.obj("archived" -> true)) + + row <- sendRequest("GET", s"/tables/1/rows/1").map(_.getBoolean("archived")) + cell <- sendRequest("GET", s"/tables/1/columns/1/rows/1").map(_.getBoolean("archived")) + } yield { + assertEquals(row, true) + assertEquals(cell, true) + } + } + } + + @Test + def retrieveSingleArchivedFirstCellsAndLinkCell(implicit c: TestContext): Unit = { + okTest { + for { + tableId1 <- createDefaultTable(name = "table1") + tableId2 <- createDefaultTable(name = "table2", tableNum = 2) + + linkColumnId <- createArchiveCascadeLinkColumn(tableId1, tableId2, "cardinality") + columns <- sendRequest("GET", s"/tables/$tableId1/columns").map(_.getJsonArray("columns")) + + rowId1 <- + sendRequest("POST", s"/tables/$tableId1/rows", Rows(columns, Json.obj("cardinality" -> Json.arr(1, 2)))) + .map(toRowsArray).map(_.getJsonObject(0)).map(_.getLong("id")) + _ <- sendRequest("PATCH", s"/tables/$tableId2/rows/1/annotations", Json.obj("final" -> true)) + _ <- sendRequest("PATCH", s"/tables/$tableId2/rows/2/annotations", Json.obj("archived" -> true)) + + _ = println(s"#############") + + firstCells <- sendRequest("GET", s"/tables/$tableId2/columns/first/rows").map(_.getJsonArray("rows")) + linkCell <- + sendRequest("GET", s"/tables/$tableId1/columns/$linkColumnId/rows/$rowId1").map(_.getJsonArray("value")) + } yield { + println(s"firstCell: $firstCells") + println(s"linkCells: $linkCell") + + assertJSONEquals( + Json.fromObjectString("""|{ + | "id": 1, + | "values": [ + | "table2row1" + | ], + | "final": true + |} + |""".stripMargin), + firstCells.getJsonObject(0) + ) + + assertJSONEquals( + Json.fromArrayString( + """|[ + | { + | "id": 1, + | "value": "table2row1", + | "final": true + | }, + | { + | "id": 2, + | "value": "table2row2", + | "archived": true + | } + |] + |""".stripMargin + ), + linkCell + ) + } + } + } } From 652355a6373ab2444bba9e7cb63be271d71b75a9 Mon Sep 17 00:00:00 2001 From: Manfred Zingl Date: Wed, 15 May 2024 15:18:03 +0200 Subject: [PATCH 3/7] change query to also select row annotations --- .../database/model/tableaux/RowModel.scala | 29 ++++++++++++------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/src/main/scala/com/campudus/tableaux/database/model/tableaux/RowModel.scala b/src/main/scala/com/campudus/tableaux/database/model/tableaux/RowModel.scala index 48a8dbe7..de8efe22 100644 --- a/src/main/scala/com/campudus/tableaux/database/model/tableaux/RowModel.scala +++ b/src/main/scala/com/campudus/tableaux/database/model/tableaux/RowModel.scala @@ -1435,18 +1435,27 @@ class RetrieveRowModel(val connection: DatabaseConnection)( s"""( |SELECT - | json_agg(sub.value) + | json_agg(sub.value) |FROM |(SELECT - | lt$linkId.${direction.fromSql} AS ${direction.fromSql}, - | json_build_object('id', ut$toTableId.id, 'value', $value) AS value + | lt$linkId.${direction.fromSql} AS ${direction.fromSql}, + | ( + | jsonb_build_object('id', ut$toTableId.id, 'value', $value) + | || + | jsonb_strip_nulls( + | jsonb_build_object( + | 'final', CASE WHEN ut$toTableId.final IS TRUE THEN ut$toTableId.final ELSE NULL END, + | 'archived', CASE WHEN ut$toTableId.archived IS TRUE THEN ut$toTableId.archived ELSE NULL END + | ) + | ) + | ) AS value | FROM - | link_table_$linkId lt$linkId - | JOIN user_table_$toTableId ut$toTableId ON (lt$linkId.${direction.toSql} = ut$toTableId.id) - | LEFT JOIN user_table_lang_$toTableId utl$toTableId ON (ut$toTableId.id = utl$toTableId.id) - | WHERE $column IS NOT NULL - | GROUP BY ut$toTableId.id, lt$linkId.${direction.fromSql}, lt$linkId.${direction.orderingSql} - | ORDER BY lt$linkId.${direction.fromSql}, lt$linkId.${direction.orderingSql} + | link_table_$linkId lt$linkId + | JOIN user_table_$toTableId ut$toTableId ON (lt$linkId.${direction.toSql} = ut$toTableId.id) + | LEFT JOIN user_table_lang_$toTableId utl$toTableId ON (ut$toTableId.id = utl$toTableId.id) + | WHERE $column IS NOT NULL + | GROUP BY ut$toTableId.id, lt$linkId.${direction.fromSql}, lt$linkId.${direction.orderingSql} + | ORDER BY lt$linkId.${direction.fromSql}, lt$linkId.${direction.orderingSql} |) sub |WHERE sub.${direction.fromSql} = ut.id |GROUP BY sub.${direction.fromSql}) AS column_${c.id}""".stripMargin @@ -1456,7 +1465,7 @@ class RetrieveRowModel(val connection: DatabaseConnection)( s"""|ut.final AS final_flag, |ut.archived AS archived_flag, |(SELECT json_agg( - | json_build_object( + | jsonb_build_object( | 'column_id', column_id, | 'uuid', uuid, | 'langtags', langtags::text[], From 041655e8bf2f84c7b7530a8560d92ecb86446c40 Mon Sep 17 00:00:00 2001 From: Manfred Zingl Date: Wed, 15 May 2024 17:07:04 +0200 Subject: [PATCH 4/7] add cache for row level annotations --- .../campudus/tableaux/cache/CacheClient.scala | 47 +++++ .../tableaux/cache/CacheVerticle.scala | 199 ++++++++++++++---- .../database/model/TableauxModel.scala | 24 ++- 3 files changed, 230 insertions(+), 40 deletions(-) diff --git a/src/main/scala/com/campudus/tableaux/cache/CacheClient.scala b/src/main/scala/com/campudus/tableaux/cache/CacheClient.scala index 3a9cae08..b5252be9 100644 --- a/src/main/scala/com/campudus/tableaux/cache/CacheClient.scala +++ b/src/main/scala/com/campudus/tableaux/cache/CacheClient.scala @@ -96,6 +96,53 @@ class CacheClient(vertxAccess: VertxAccess) extends VertxAccess { eventBus.sendFuture(CacheVerticle.ADDRESS_INVALIDATE_ROW_PERMISSIONS, obj) } + def setRowLevelAnnotations( + tableId: TableId, + rowId: RowId, + rowLevelAnnotations: RowLevelAnnotations + ): Future[_] = { + val rowValue = Json.obj("value" -> rowLevelAnnotations.getJson) + val obj = rowKey(tableId, rowId).copy().mergeIn(rowValue) + println(s"set cache value for row level annotations: $obj") + eventBus.sendFuture(CacheVerticle.ADDRESS_SET_ROW_LEVEL_ANNOTATIONS, obj, options) + } + + def retrieveRowLevelAnnotations(tableId: TableId, rowId: RowId): Future[Option[RowLevelAnnotations]] = { + val obj = rowKey(tableId, rowId) + + eventBus + .sendFuture[JsonObject](CacheVerticle.ADDRESS_RETRIEVE_ROW_LEVEL_ANNOTATIONS, obj, options) + .map(value => { + value match { + case v if v.body().containsKey("value") => { + val rawRowLevelAnnotations = v.body().getValue("value").asInstanceOf[JsonObject] + val finalFlag = rawRowLevelAnnotations.getBoolean("final", false) + val archivedFlag = rawRowLevelAnnotations.getBoolean("archived", false) + Some(RowLevelAnnotations(finalFlag, archivedFlag)) + } + case _ => { + None + } + } + }) + .recoverWith({ + case ex: ReplyException if ex.failureCode() == CacheVerticle.NOT_FOUND_FAILURE => + Future.successful(None) + case ex => + Future.failed(ex) + }) + } + + def invalidateRowLevelAnnotations(tableId: TableId, rowId: RowId): Future[_] = { + val obj = Json.obj("tableId" -> tableId, "rowId" -> rowId) + eventBus.sendFuture(CacheVerticle.ADDRESS_INVALIDATE_ROW_LEVEL_ANNOTATIONS, obj) + } + + def invalidateTableRowLevelAnnotations(tableId: TableId): Future[_] = { + val obj = Json.obj("tableId" -> tableId) + eventBus.sendFuture(CacheVerticle.ADDRESS_INVALIDATE_TABLE_ROW_LEVEL_ANNOTATIONS, obj) + } + def invalidateCellValue(tableId: TableId, columnId: ColumnId, rowId: RowId): Future[_] = { val obj = cellKey(tableId, columnId, rowId) eventBus.sendFuture(CacheVerticle.ADDRESS_INVALIDATE_CELL, obj, options) diff --git a/src/main/scala/com/campudus/tableaux/cache/CacheVerticle.scala b/src/main/scala/com/campudus/tableaux/cache/CacheVerticle.scala index 0b0c981d..6e4d8e85 100644 --- a/src/main/scala/com/campudus/tableaux/cache/CacheVerticle.scala +++ b/src/main/scala/com/campudus/tableaux/cache/CacheVerticle.scala @@ -35,18 +35,26 @@ object CacheVerticle { val NOT_FOUND_FAILURE: Int = 404 val INVALID_MESSAGE: Int = 400 + // cell based addresses val ADDRESS_SET_CELL: String = "cache.set.cell" - val ADDRESS_SET_ROW_PERMISSIONS: String = "cache.set.row_permissions" val ADDRESS_RETRIEVE_CELL: String = "cache.retrieve.cell" - val ADDRESS_RETRIEVE_ROW_PERMISSIONS: String = "cache.retrieve.row_permissions" - val ADDRESS_INVALIDATE_CELL: String = "cache.invalidate.cell" val ADDRESS_INVALIDATE_COLUMN: String = "cache.invalidate.column" val ADDRESS_INVALIDATE_ROW: String = "cache.invalidate.row" val ADDRESS_INVALIDATE_TABLE: String = "cache.invalidate.table" val ADDRESS_INVALIDATE_ALL: String = "cache.invalidate.all" + + // row permissions based addresses + val ADDRESS_SET_ROW_PERMISSIONS: String = "cache.set.row_permissions" + val ADDRESS_RETRIEVE_ROW_PERMISSIONS: String = "cache.retrieve.row_permissions" val ADDRESS_INVALIDATE_ROW_PERMISSIONS: String = "cache.invalidate.row_permissions" + // row level annotations based addresses + val ADDRESS_SET_ROW_LEVEL_ANNOTATIONS: String = "cache.set.row_level_annotations" + val ADDRESS_RETRIEVE_ROW_LEVEL_ANNOTATIONS: String = "cache.retrieve.row_level_annotations" + val ADDRESS_INVALIDATE_ROW_LEVEL_ANNOTATIONS: String = "cache.invalidate.row_level_annotations" + val ADDRESS_INVALIDATE_TABLE_ROW_LEVEL_ANNOTATIONS: String = "cache.invalidate.table.row_level_annotations" + val TIMEOUT_AFTER_MILLISECONDS: Int = 400 def apply(tableauxConfig: TableauxConfig): CacheVerticle = { @@ -62,10 +70,11 @@ class CacheVerticle(tableauxConfig: TableauxConfig) extends ScalaVerticle with L type CellCaches = mutable.Map[(TableId, ColumnId), Cache[AnyRef]] type RowPermissionsCaches = mutable.Map[(TableId), Cache[AnyRef]] - type Caches = Either[CellCaches, RowPermissionsCaches] + type RowLevelAnnotationsCache = mutable.Map[(TableId), Cache[AnyRef]] private val cellCaches: CellCaches = mutable.Map.empty private val rowPermissionsCaches: RowPermissionsCaches = mutable.Map.empty + private val rowLevelAnnotationsCache: RowLevelAnnotationsCache = mutable.Map.empty override def startFuture(): Future[_] = { logger.info( @@ -84,16 +93,31 @@ class CacheVerticle(tableauxConfig: TableauxConfig) extends ScalaVerticle with L private def registerOnEventBus(): Future[_] = { Future.sequence( Seq( + // cell registerHandler(eventBus, ADDRESS_SET_CELL, messageHandlerSetCell), registerHandler(eventBus, ADDRESS_RETRIEVE_CELL, messageHandlerRetrieveCell), - registerHandler(eventBus, ADDRESS_SET_ROW_PERMISSIONS, messageHandlerSetRowPermissions), - registerHandler(eventBus, ADDRESS_RETRIEVE_ROW_PERMISSIONS, messageHandlerRetrieveRowPermissions), registerHandler(eventBus, ADDRESS_INVALIDATE_CELL, messageHandlerInvalidateCell), registerHandler(eventBus, ADDRESS_INVALIDATE_COLUMN, messageHandlerInvalidateColumn), registerHandler(eventBus, ADDRESS_INVALIDATE_ROW, messageHandlerInvalidateRow), registerHandler(eventBus, ADDRESS_INVALIDATE_TABLE, messageHandlerInvalidateTable), registerHandler(eventBus, ADDRESS_INVALIDATE_ALL, messageHandlerInvalidateAll), - registerHandler(eventBus, ADDRESS_INVALIDATE_ROW_PERMISSIONS, messageHandlerInvalidateRowPermissions) + // row permissions + registerHandler(eventBus, ADDRESS_SET_ROW_PERMISSIONS, messageHandlerSetRowPermissions), + registerHandler(eventBus, ADDRESS_RETRIEVE_ROW_PERMISSIONS, messageHandlerRetrieveRowPermissions), + registerHandler(eventBus, ADDRESS_INVALIDATE_ROW_PERMISSIONS, messageHandlerInvalidateRowPermissions), + // row level annotations + registerHandler(eventBus, ADDRESS_SET_ROW_LEVEL_ANNOTATIONS, messageHandlerSetRowLevelAnnotations), + registerHandler(eventBus, ADDRESS_RETRIEVE_ROW_LEVEL_ANNOTATIONS, messageHandlerRetrieveRowLevelAnnotations), + registerHandler( + eventBus, + ADDRESS_INVALIDATE_ROW_LEVEL_ANNOTATIONS, + messageHandlerInvalidateRowLevelAnnotations + ), + registerHandler( + eventBus, + ADDRESS_INVALIDATE_TABLE_ROW_LEVEL_ANNOTATIONS, + messageHandlerInvalidateTableRowLevelAnnotations + ) ) ) } @@ -135,6 +159,16 @@ class CacheVerticle(tableauxConfig: TableauxConfig) extends ScalaVerticle with L } } + private def getRowLevelAnnotationsCache(tableId: TableId): Cache[AnyRef] = { + rowLevelAnnotationsCache.get(tableId) match { + case Some(cache) => cache + case None => + val cache: Cache[AnyRef] = GuavaCache(createCache()) + rowLevelAnnotationsCache.put((tableId), cache) + cache + } + } + private def removeCache(tableId: TableId, columnId: ColumnId): Unit = cellCaches.remove((tableId, columnId)) private def extractTableColumnRow(obj: JsonObject): Option[(TableId, ColumnId, RowId)] = { @@ -203,7 +237,6 @@ class CacheVerticle(tableauxConfig: TableauxConfig) extends ScalaVerticle with L extractTableRow(obj) match { case Some((tableId, rowId)) => { implicit val scalaCache: Cache[AnyRef] = getRowPermissionsCache(tableId) - logger.info(s"messageHandlerSetRowPermissions $obj $tableId $rowId $value") put(rowId)(value).map(_ => replyJson(message, tableId, rowId)) } case None => { @@ -247,9 +280,7 @@ class CacheVerticle(tableauxConfig: TableauxConfig) extends ScalaVerticle with L case Some((tableId, columnId, rowId)) => // invalidate cell implicit val scalaCache: Cache[AnyRef] = getCellCache(tableId, columnId) - - remove(rowId) - .map(_ => replyJson(message, tableId, columnId, rowId)) + remove(rowId).map(_ => replyJson(message, tableId, columnId, rowId)) case None => logger.error("Message invalid: Fields (tableId, columnId, rowId) should be a Long") @@ -257,6 +288,109 @@ class CacheVerticle(tableauxConfig: TableauxConfig) extends ScalaVerticle with L } } + private def messageHandlerSetRowLevelAnnotations(message: Message[JsonObject]): Unit = { + val obj = message.body() + val value = obj.getValue("value") + + extractTableRow(obj) match { + case Some((tableId, rowId)) => { + implicit val scalaCache: Cache[AnyRef] = getRowLevelAnnotationsCache(tableId) + put(rowId)(value).map(_ => replyJson(message, tableId, rowId)) + } + case None => { + logger.error("Message invalid: Fields (tableId, rowId) should be a Long") + message.fail(INVALID_MESSAGE, "Message invalid: Fields (tableId, rowId) should be a Long") + } + } + } + + private def messageHandlerRetrieveRowLevelAnnotations(message: Message[JsonObject]): Unit = { + val obj = message.body() + + extractTableRow(obj) match { + case Some((tableId, rowId)) => + implicit val scalaCache: Cache[AnyRef] = getRowLevelAnnotationsCache(tableId) + + get(rowId).map({ + case Some(value) => { + val reply = Json.obj( + "tableId" -> tableId, + "rowId" -> rowId, + "value" -> value + ) + + println(s"hey, ausm cache raus: $value") + message.reply(reply) + } + case None => { + logger.debug(s"messageHandlerRetrieveRowLevelAnnotations $tableId, $rowId not found") + message.fail(NOT_FOUND_FAILURE, "Not found") + } + }) + + case None => + logger.error("Message invalid: Fields (tableId, rowId) should be a Long") + message.fail(INVALID_MESSAGE, "Message invalid: Fields (tableId, rowId) should be a Long") + } + } + + private def messageHandlerInvalidateRowLevelAnnotations(message: Message[JsonObject]): Unit = { + extractTableRow(message.body()) match { + case Some((tableId, rowId)) => + // invalidate cell + implicit val scalaCache: Cache[AnyRef] = getRowLevelAnnotationsCache(tableId) + + remove(rowId).map(_ => replyJson(message, tableId, rowId)) + + case None => + logger.error("Message invalid: Fields (tableId, rowId) should be a Long") + message.fail(INVALID_MESSAGE, "Message invalid: Fields (tableId, rowId) should be a Long") + } + } + + private def messageHandlerInvalidateTableRowLevelAnnotations(message: Message[JsonObject]): Unit = { + val obj = message.body() + println(s"caches clear arrived for all tables") + + (for { + tableId <- Option(obj.getLong("tableId")).map(_.toLong) + } yield tableId) match { + case Some(tableId) => + Future.sequence(filterRowLevelAnnotationsCache(tableId) + .map(implicit cache => removeAll())) + .map(_ => { + val reply = Json.obj("tableId" -> tableId) + message.reply(reply) + }) + + case None => + logger.error("Message invalid: Fields (tableId) should be a Long") + message.fail(INVALID_MESSAGE, "Message invalid: Fields (tableId) should be a Long") + } + + // extractTableRow(message.body()) match { + // case Some((tableId, rowId)) => + // // invalidate cell + // implicit val scalaCache: Cache[AnyRef] = getRowLevelAnnotationsCache(tableId) + + // remove(rowId).map(_ => replyJson(message, tableId, rowId)) + + // case None => + // logger.error("Message invalid: Fields (tableId, rowId) should be a Long") + // message.fail(INVALID_MESSAGE, "Message invalid: Fields (tableId, rowId) should be a Long") + // } + + // Future.sequence(rowLevelAnnotationsCache.map({ + // case ((tableId), cache) => + // implicit val implicitCache: Cache[AnyRef] = implicitly(cache) + + // removeAll().map(_ => rowLevelAnnotationsCache.remove(tableId)) + // })).onComplete(_ => { + // rowLevelAnnotationsCache.clear() + // message.reply(Json.emptyObj()) + // }) + } + private def replyJson(message: Message[JsonObject], tableId: TableId, columnId: ColumnId, rowId: RowId): Unit = { val reply = Json.obj("tableId" -> tableId, "columnId" -> columnId, "rowId" -> rowId) message.reply(reply) @@ -299,11 +433,10 @@ class CacheVerticle(tableauxConfig: TableauxConfig) extends ScalaVerticle with L }).values } - private def filterRowPermissionCaches(tableId: TableId) = { + private def filterRowLevelAnnotationsCache(tableId: TableId) = { // invalidate table - rowPermissionsCaches.filterKeys({ - case (cachedTableId) => - cachedTableId == tableId + rowLevelAnnotationsCache.filterKeys({ + case (cachedTableId) => cachedTableId == tableId }).values } @@ -356,35 +489,25 @@ class CacheVerticle(tableauxConfig: TableauxConfig) extends ScalaVerticle with L } private def messageHandlerInvalidateAll(message: Message[JsonObject]): Unit = { - Future - .sequence(cellCaches.map({ - case ((tableId, columnId), cache) => - implicit val implicitCache: Cache[AnyRef] = implicitly(cache) - - removeAll().map(_ => removeCache(tableId, columnId)) - })) - .onComplete(_ => { - cellCaches.clear() - message.reply(Json.emptyObj()) - }) + Future.sequence(cellCaches.map({ + case ((tableId, columnId), cache) => + implicit val implicitCache: Cache[AnyRef] = implicitly(cache) + + removeAll().map(_ => removeCache(tableId, columnId)) + })).onComplete(_ => { + cellCaches.clear() + message.reply(Json.emptyObj()) + }) } private def messageHandlerInvalidateRowPermissions(message: Message[JsonObject]): Unit = { val obj = message.body() - (extractTableRow(obj)) match { + extractTableRow(message.body()) match { case Some((tableId, rowId)) => - Future - .sequence( - filterRowPermissionCaches(tableId) - .map(implicit cache => { - remove(rowId) - }) - ) - .map(_ => { - val reply = Json.obj("tableId" -> tableId, "rowId" -> rowId) - message.reply(reply) - }) + // invalidate cell + implicit val scalaCache: Cache[AnyRef] = getRowPermissionsCache(tableId) + remove(rowId).map(_ => replyJson(message, tableId, rowId)) case None => logger.error("Message invalid: Fields (tableId, rowId) should be a Long") diff --git a/src/main/scala/com/campudus/tableaux/database/model/TableauxModel.scala b/src/main/scala/com/campudus/tableaux/database/model/TableauxModel.scala index ca7cd26c..0d3e73d6 100644 --- a/src/main/scala/com/campudus/tableaux/database/model/TableauxModel.scala +++ b/src/main/scala/com/campudus/tableaux/database/model/TableauxModel.scala @@ -538,6 +538,7 @@ class TableauxModel( } yield () }) + _ <- CacheClient(this.connection).invalidateRowLevelAnnotations(table.id, rowId) } yield () }; @@ -570,6 +571,7 @@ class TableauxModel( _ <- Future.sequence( rowSeq.map(row => updateRowAnnotations(table, row.id, finalFlagOpt, archivedFlagOpt)) ) + _ <- CacheClient(this.connection).invalidateTableRowLevelAnnotations(table.id) } yield () } @@ -970,6 +972,7 @@ class TableauxModel( } for { + rowLevelAnnotationsCache <- CacheClient(this.connection).retrieveRowLevelAnnotations(column.table.id, rowId) valueCache <- CacheClient(this.connection).retrieveCellValue(column.table.id, column.id, rowId) value <- valueCache match { @@ -1018,8 +1021,25 @@ class TableauxModel( Future.successful(value) } - // TODO use cache for rowLevelAnnotations - (rowLevelAnnotations, _, _) <- retrieveRowModel.retrieveAnnotations(column.table.id, rowId, Seq(column)) + rowLevelAnnotations <- rowLevelAnnotationsCache match { + case Some(annotations) => { + println(s"Cache hit for rowLevelAnnotations for table ${column.table.id} and row $rowId") + Future.successful(annotations) + } + case None => { + for { + (rowLevelAnnotations, _, _) <- retrieveRowModel.retrieveAnnotations(column.table.id, rowId, Seq(column)) + } yield { + println( + s"Cache miss for rowLevelAnnotations for table ${column.table.id} and row $rowId, rowLevelAnnotations: $rowLevelAnnotations" + ) + // fire-and-forget don't need to wait for this to return + CacheClient(this.connection).setRowLevelAnnotations(column.table.id, rowId, rowLevelAnnotations) + rowLevelAnnotations + } + + } + } } yield { Cell(column, rowId, resultValue, rowLevelAnnotations) } From 3098961bd2592503e83b3332bd833e1706032d59 Mon Sep 17 00:00:00 2001 From: Manfred Zingl Date: Wed, 15 May 2024 17:08:15 +0200 Subject: [PATCH 5/7] fix jsonb concat operator it is stipped away if not behind other instructions --- .../campudus/tableaux/database/model/tableaux/RowModel.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/scala/com/campudus/tableaux/database/model/tableaux/RowModel.scala b/src/main/scala/com/campudus/tableaux/database/model/tableaux/RowModel.scala index de8efe22..19eaeb80 100644 --- a/src/main/scala/com/campudus/tableaux/database/model/tableaux/RowModel.scala +++ b/src/main/scala/com/campudus/tableaux/database/model/tableaux/RowModel.scala @@ -1440,8 +1440,7 @@ class RetrieveRowModel(val connection: DatabaseConnection)( |(SELECT | lt$linkId.${direction.fromSql} AS ${direction.fromSql}, | ( - | jsonb_build_object('id', ut$toTableId.id, 'value', $value) - | || + | jsonb_build_object('id', ut$toTableId.id, 'value', $value) || | jsonb_strip_nulls( | jsonb_build_object( | 'final', CASE WHEN ut$toTableId.final IS TRUE THEN ut$toTableId.final ELSE NULL END, From ebd068c57380f9e733f6ca2508d682407be562b5 Mon Sep 17 00:00:00 2001 From: Manfred Zingl Date: Wed, 15 May 2024 17:59:59 +0200 Subject: [PATCH 6/7] fix missing row level annotations in concat column --- src/main/resources/swagger.json | 30 +++++++++++++++---- .../database/model/TableauxModel.scala | 5 +++- .../content/CellLevelAnnotationsTest.scala | 4 +-- .../api/content/LinkConstraintTest.scala | 6 +--- 4 files changed, 32 insertions(+), 13 deletions(-) diff --git a/src/main/resources/swagger.json b/src/main/resources/swagger.json index 604e37f1..a66c7495 100644 --- a/src/main/resources/swagger.json +++ b/src/main/resources/swagger.json @@ -1644,13 +1644,15 @@ "id": 1, "values": [ "Shimano" - ] + ], + "final": true }, { "id": 2, "values": [ "Sram " - ] + ], + "archived": true } ] } @@ -1713,13 +1715,15 @@ "id": 1, "values": [ "Shimano" - ] + ], + "final": true }, { "id": 2, "values": [ - "Sram " - ] + "Sram" + ], + "archived": true } ] } @@ -2476,6 +2480,9 @@ "allOf": [ { "$ref": "#/definitions/Property:Status" + }, + { + "$ref": "#/definitions/Property:RowLevelAnnotations" } ], "properties": { @@ -3956,6 +3963,19 @@ "example": 1 } } + }, + "Property:RowLevelAnnotations": { + "type": "object", + "properties": { + "final": { + "type": "boolean", + "example": true + }, + "archived": { + "type": "boolean", + "example": true + } + } } }, "parameters": { diff --git a/src/main/scala/com/campudus/tableaux/database/model/TableauxModel.scala b/src/main/scala/com/campudus/tableaux/database/model/TableauxModel.scala index 0d3e73d6..4c68ecc9 100644 --- a/src/main/scala/com/campudus/tableaux/database/model/TableauxModel.scala +++ b/src/main/scala/com/campudus/tableaux/database/model/TableauxModel.scala @@ -1440,7 +1440,10 @@ class TableauxModel( for { list <- futureList cell <- retrieveCell(concatenateColumn, rowId, true) - } yield list ++ List(Json.obj("id" -> rowId, "value" -> DomainObject.compatibilityGet(cell.value))) + } yield { + val cellJson = cell.getJson + list ++ List(Json.obj("id" -> rowId).mergeIn(cellJson)) + } } } diff --git a/src/test/scala/com/campudus/tableaux/api/content/CellLevelAnnotationsTest.scala b/src/test/scala/com/campudus/tableaux/api/content/CellLevelAnnotationsTest.scala index 6eae5ad4..0e7929ff 100644 --- a/src/test/scala/com/campudus/tableaux/api/content/CellLevelAnnotationsTest.scala +++ b/src/test/scala/com/campudus/tableaux/api/content/CellLevelAnnotationsTest.scala @@ -559,7 +559,7 @@ class CellLevelAnnotationsTest extends TableauxTestBase { for { tableId <- createEmptyDefaultTable() - // make second column an identifer + // make second column an identifier _ <- sendRequest("POST", s"/tables/$tableId/columns/2", Json.obj("identifier" -> true)) // empty row @@ -592,7 +592,7 @@ class CellLevelAnnotationsTest extends TableauxTestBase { for { tableId <- createEmptyDefaultTable() - // make second column an identifer + // make second column an identifier _ <- sendRequest("POST", s"/tables/$tableId/columns/2", Json.obj("identifier" -> true)) // empty row diff --git a/src/test/scala/com/campudus/tableaux/api/content/LinkConstraintTest.scala b/src/test/scala/com/campudus/tableaux/api/content/LinkConstraintTest.scala index 45ccb66f..b0f10de4 100644 --- a/src/test/scala/com/campudus/tableaux/api/content/LinkConstraintTest.scala +++ b/src/test/scala/com/campudus/tableaux/api/content/LinkConstraintTest.scala @@ -1729,15 +1729,11 @@ class RetrieveFinalAndArchivedRows extends LinkTestBase with Helper { _ <- sendRequest("PATCH", s"/tables/$tableId2/rows/1/annotations", Json.obj("final" -> true)) _ <- sendRequest("PATCH", s"/tables/$tableId2/rows/2/annotations", Json.obj("archived" -> true)) - _ = println(s"#############") - firstCells <- sendRequest("GET", s"/tables/$tableId2/columns/first/rows").map(_.getJsonArray("rows")) linkCell <- sendRequest("GET", s"/tables/$tableId1/columns/$linkColumnId/rows/$rowId1").map(_.getJsonArray("value")) + row <- sendRequest("GET", s"/tables/$tableId1/rows/$rowId1") } yield { - println(s"firstCell: $firstCells") - println(s"linkCells: $linkCell") - assertJSONEquals( Json.fromObjectString("""|{ | "id": 1, From fa99feadbcc58a01ebe690617b2f1cdf92f2b8da Mon Sep 17 00:00:00 2001 From: Manfred Zingl Date: Wed, 15 May 2024 18:10:05 +0200 Subject: [PATCH 7/7] cleanup println and comments --- .../campudus/tableaux/cache/CacheClient.scala | 1 - .../tableaux/cache/CacheVerticle.scala | 24 ------------------- .../database/model/TableauxModel.scala | 4 ---- .../api/content/DuplicateRowTest.scala | 1 - .../api/content/LinkConstraintTest.scala | 4 ---- .../api/content/TranslationStatusTest.scala | 1 - .../database/domain/domainobjectTest.scala | 1 - 7 files changed, 36 deletions(-) diff --git a/src/main/scala/com/campudus/tableaux/cache/CacheClient.scala b/src/main/scala/com/campudus/tableaux/cache/CacheClient.scala index b5252be9..89728d6b 100644 --- a/src/main/scala/com/campudus/tableaux/cache/CacheClient.scala +++ b/src/main/scala/com/campudus/tableaux/cache/CacheClient.scala @@ -103,7 +103,6 @@ class CacheClient(vertxAccess: VertxAccess) extends VertxAccess { ): Future[_] = { val rowValue = Json.obj("value" -> rowLevelAnnotations.getJson) val obj = rowKey(tableId, rowId).copy().mergeIn(rowValue) - println(s"set cache value for row level annotations: $obj") eventBus.sendFuture(CacheVerticle.ADDRESS_SET_ROW_LEVEL_ANNOTATIONS, obj, options) } diff --git a/src/main/scala/com/campudus/tableaux/cache/CacheVerticle.scala b/src/main/scala/com/campudus/tableaux/cache/CacheVerticle.scala index 6e4d8e85..a576824a 100644 --- a/src/main/scala/com/campudus/tableaux/cache/CacheVerticle.scala +++ b/src/main/scala/com/campudus/tableaux/cache/CacheVerticle.scala @@ -319,7 +319,6 @@ class CacheVerticle(tableauxConfig: TableauxConfig) extends ScalaVerticle with L "value" -> value ) - println(s"hey, ausm cache raus: $value") message.reply(reply) } case None => { @@ -350,7 +349,6 @@ class CacheVerticle(tableauxConfig: TableauxConfig) extends ScalaVerticle with L private def messageHandlerInvalidateTableRowLevelAnnotations(message: Message[JsonObject]): Unit = { val obj = message.body() - println(s"caches clear arrived for all tables") (for { tableId <- Option(obj.getLong("tableId")).map(_.toLong) @@ -367,28 +365,6 @@ class CacheVerticle(tableauxConfig: TableauxConfig) extends ScalaVerticle with L logger.error("Message invalid: Fields (tableId) should be a Long") message.fail(INVALID_MESSAGE, "Message invalid: Fields (tableId) should be a Long") } - - // extractTableRow(message.body()) match { - // case Some((tableId, rowId)) => - // // invalidate cell - // implicit val scalaCache: Cache[AnyRef] = getRowLevelAnnotationsCache(tableId) - - // remove(rowId).map(_ => replyJson(message, tableId, rowId)) - - // case None => - // logger.error("Message invalid: Fields (tableId, rowId) should be a Long") - // message.fail(INVALID_MESSAGE, "Message invalid: Fields (tableId, rowId) should be a Long") - // } - - // Future.sequence(rowLevelAnnotationsCache.map({ - // case ((tableId), cache) => - // implicit val implicitCache: Cache[AnyRef] = implicitly(cache) - - // removeAll().map(_ => rowLevelAnnotationsCache.remove(tableId)) - // })).onComplete(_ => { - // rowLevelAnnotationsCache.clear() - // message.reply(Json.emptyObj()) - // }) } private def replyJson(message: Message[JsonObject], tableId: TableId, columnId: ColumnId, rowId: RowId): Unit = { diff --git a/src/main/scala/com/campudus/tableaux/database/model/TableauxModel.scala b/src/main/scala/com/campudus/tableaux/database/model/TableauxModel.scala index 4c68ecc9..48c77d1c 100644 --- a/src/main/scala/com/campudus/tableaux/database/model/TableauxModel.scala +++ b/src/main/scala/com/campudus/tableaux/database/model/TableauxModel.scala @@ -1023,16 +1023,12 @@ class TableauxModel( rowLevelAnnotations <- rowLevelAnnotationsCache match { case Some(annotations) => { - println(s"Cache hit for rowLevelAnnotations for table ${column.table.id} and row $rowId") Future.successful(annotations) } case None => { for { (rowLevelAnnotations, _, _) <- retrieveRowModel.retrieveAnnotations(column.table.id, rowId, Seq(column)) } yield { - println( - s"Cache miss for rowLevelAnnotations for table ${column.table.id} and row $rowId, rowLevelAnnotations: $rowLevelAnnotations" - ) // fire-and-forget don't need to wait for this to return CacheClient(this.connection).setRowLevelAnnotations(column.table.id, rowId, rowLevelAnnotations) rowLevelAnnotations diff --git a/src/test/scala/com/campudus/tableaux/api/content/DuplicateRowTest.scala b/src/test/scala/com/campudus/tableaux/api/content/DuplicateRowTest.scala index 75d431e0..6cca3657 100644 --- a/src/test/scala/com/campudus/tableaux/api/content/DuplicateRowTest.scala +++ b/src/test/scala/com/campudus/tableaux/api/content/DuplicateRowTest.scala @@ -243,7 +243,6 @@ class DuplicateRowTest extends TableauxTestBase { modelTableId = tableIds(0) _ <- sendRequest("POST", s"/tables/$modelTableId/rows/1/duplicate?skipConstrainedLinks=true&annotateSkipped=true") res <- sendRequest("GET", s"/tables/$modelTableId/columns/3/rows/3/annotations") - _ = println(res) } yield { val checkMeAnnotation = res.getJsonArray("annotations").getJsonArray(0).getJsonObject(0) assertEquals(checkMeAnnotation.getString("type"), CellAnnotationType.FLAG) diff --git a/src/test/scala/com/campudus/tableaux/api/content/LinkConstraintTest.scala b/src/test/scala/com/campudus/tableaux/api/content/LinkConstraintTest.scala index b0f10de4..2df04e1e 100644 --- a/src/test/scala/com/campudus/tableaux/api/content/LinkConstraintTest.scala +++ b/src/test/scala/com/campudus/tableaux/api/content/LinkConstraintTest.scala @@ -1682,11 +1682,7 @@ class RetrieveFinalAndArchivedRows extends LinkTestBase with Helper { sendRequest("GET", s"/tables/$tableId1/columns/$linkColumnId/rows/$rowId1/foreignRows?final=true") resultForeignRowsArchived <- sendRequest("GET", s"/tables/$tableId1/columns/$linkColumnId/rows/$rowId1/foreignRows?archived=true") - - rrr <- sendRequest("GET", s"/tables/$tableId1/columns/$linkColumnId/rows/$rowId1") } yield { - println(s"rrr: $rrr") - assertEquals(0, resultCell.getJsonArray("value").size()) assertEquals(2, resultForeignRowsFinal.getJsonObject("page").getLong("totalSize").longValue()) diff --git a/src/test/scala/com/campudus/tableaux/api/content/TranslationStatusTest.scala b/src/test/scala/com/campudus/tableaux/api/content/TranslationStatusTest.scala index c213daea..48fefcc9 100644 --- a/src/test/scala/com/campudus/tableaux/api/content/TranslationStatusTest.scala +++ b/src/test/scala/com/campudus/tableaux/api/content/TranslationStatusTest.scala @@ -193,7 +193,6 @@ class TranslationStatusTest extends TableauxTestBase { translationStatus <- sendRequest("GET", s"/tables/translationStatus") } yield { - println(s"translationStatus: $translationStatus") val expectedTranslationStatus = Json.obj( "tables" -> Json.emptyArr(), "translationStatus" -> Json.obj( diff --git a/src/test/scala/com/campudus/tableaux/database/domain/domainobjectTest.scala b/src/test/scala/com/campudus/tableaux/database/domain/domainobjectTest.scala index 2950629a..ad52319f 100644 --- a/src/test/scala/com/campudus/tableaux/database/domain/domainobjectTest.scala +++ b/src/test/scala/com/campudus/tableaux/database/domain/domainobjectTest.scala @@ -53,7 +53,6 @@ class DomainObjectTest { val actual = DomainObject.compatibilityGet(cell) - println(actual.toString()) JSONAssert .assertEquals( s"""{"value": [{"value":1, "final":true}, {"value":2, "archived":true}, {"value":3}], "final":true, "archived":true}""",