diff --git a/.gitignore b/.gitignore index edd4982b..7b7293cc 100644 --- a/.gitignore +++ b/.gitignore @@ -61,3 +61,5 @@ project/plugins/project/ # local log properties local_logging.properties + +*.local* diff --git a/GETTING_STARTED.md b/GETTING_STARTED.md index f6c95cdb..a066fdf1 100644 --- a/GETTING_STARTED.md +++ b/GETTING_STARTED.md @@ -11,7 +11,7 @@ - [Retrieving a specific column](#retrieving-a-specific-column) - [Retrieving a specific row](#retrieving-a-specific-row) - [Retrieving a specific cell](#retrieving-a-specific-cell) - - [2.2. Data types & column kinds](#22-data-types--column-kinds) + - [2.2. Data types \& column kinds](#22-data-types--column-kinds) - [Primitive data types](#primitive-data-types) - [Examples](#examples) - [Complex data types](#complex-data-types) @@ -155,7 +155,7 @@ Call this endpoint to retrieve all rows of a specific table. Most important part "page": { // result can be paged with two query parameters offset and limit "offset": null, "limit": null, - "totalSize": 18 // total size of this table + "totalSize": 18 // total number of rows depending on filter query parameters. If no filters are set, totalSize is the total number of rows in the table }, "rows": [ { // row object diff --git a/Jenkinsfile b/Jenkinsfile index e8e6b620..9dba6492 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,11 +1,18 @@ @Library('campudus-jenkins-shared-lib') _ -IMAGE_NAME = "campudus/grud-backend" -DEPLOY_DIR = 'build/libs' -LEGACY_ARCHIVE_FILENAME="grud-backend-docker.tar.gz" -DOCKER_BASE_IMAGE_TAG = "build-${BUILD_NUMBER}" +final String BRANCH = params.BRANCH +final boolean NOTIFY_SLACK_ON_FAILURE = params.NOTIFY_SLACK_ON_FAILURE +final boolean NOTIFY_SLACK_ON_SUCCESS = params.NOTIFY_SLACK_ON_SUCCESS -SLACK_CHANNEL = "#grud" +final String CLEAN_GIT_BRANCH = BRANCH ? BRANCH.replaceAll("[\\.\\_\\#]", "-").tokenize('/').last() : "" + +final String IMAGE_NAME = "campudus/grud-backend" +final String IMAGE_TAG = CLEAN_GIT_BRANCH && CLEAN_GIT_BRANCH != "master" ? CLEAN_GIT_BRANCH : "latest" +final String DEPLOY_DIR = 'build/libs' +final String LEGACY_ARCHIVE_FILENAME="grud-backend-docker.tar.gz" +final GString DOCKER_BASE_IMAGE_TAG = "build-${BUILD_NUMBER}" + +final String SLACK_CHANNEL = "#grud" // flag deactivate tests for fast redeployment from jenkins frontend shouldTest = true @@ -16,12 +23,11 @@ pipeline { environment { BUILD_DATE = sh(returnStdout: true, script: 'date \"+%Y-%m-%d %H:%M:%S\"').trim() GIT_COMMIT_DATE = sh(returnStdout: true, script: "git show -s --format=%ci").trim() - CLEAN_GIT_BRANCH = sh(returnStdout: true, script: "echo $GIT_BRANCH | sed 's/[\\.\\/\\_\\#]/-/g'").trim() COMPOSE_PROJECT_NAME = "${IMAGE_NAME}-${CLEAN_GIT_BRANCH}" } triggers { - pollSCM('H/5 * * * *') + githubPush() } options { @@ -29,6 +35,11 @@ pipeline { copyArtifactPermission('*'); } + parameters { + booleanParam(name: 'NOTIFY_SLACK_ON_FAILURE', defaultValue: true, description: '') + booleanParam(name: 'NOTIFY_SLACK_ON_SUCCESS', defaultValue: false, description: '') + } + stages { stage('Cleanup & Build base docker image') { steps { @@ -45,7 +56,7 @@ pipeline { groovyVars = [:] << getBinding().getVariables() groovyVars.each {k,v -> print "$k = $v"} } - + sh "docker build -t ${IMAGE_NAME}-cacher --target=cacher ." } } @@ -85,12 +96,13 @@ pipeline { --label "GIT_COMMIT_DATE=${GIT_COMMIT_DATE}" \ --label "BUILD_DATE=${BUILD_DATE}" \ -t ${IMAGE_NAME}:${DOCKER_BASE_IMAGE_TAG}-${GIT_COMMIT} \ - -t ${IMAGE_NAME}:latest \ + -t ${IMAGE_NAME}:${IMAGE_TAG} \ -f Dockerfile \ --rm --target=prod . """ + // Legacy, but needed for some project deployments - sh "docker save ${IMAGE_NAME}:latest | gzip -c > ${DEPLOY_DIR}/${LEGACY_ARCHIVE_FILENAME}" + sh "docker save ${IMAGE_NAME}:${IMAGE_TAG} | gzip -c > ${DEPLOY_DIR}/${LEGACY_ARCHIVE_FILENAME}" } } @@ -105,7 +117,7 @@ pipeline { steps { withDockerRegistry([ credentialsId: "dockerhub", url: "" ]) { sh "docker push ${IMAGE_NAME}:${DOCKER_BASE_IMAGE_TAG}-${GIT_COMMIT}" - sh "docker push ${IMAGE_NAME}:latest" + sh "docker push ${IMAGE_NAME}:${IMAGE_TAG}" } } } @@ -115,8 +127,14 @@ pipeline { success { wrap([$class: 'BuildUser']) { script { - sh "echo successful" - slackOk(channel: SLACK_CHANNEL, message: "Image pushed to docker registry: ${IMAGE_NAME}:${DOCKER_BASE_IMAGE_TAG}-${GIT_COMMIT}") + if (NOTIFY_SLACK_ON_SUCCESS) { + final String logParams = [ + BRANCH ? "BRANCH=${BRANCH}" : null, + "image: ${IMAGE_NAME}:${IMAGE_TAG}", + ].minus(null).join(' ') + + slackOk(channel: SLACK_CHANNEL, message: "${logParams}") + } } } } @@ -124,8 +142,13 @@ pipeline { failure { wrap([$class: 'BuildUser']) { script { - sh "echo failed" - slackError(channel: SLACK_CHANNEL) + if (NOTIFY_SLACK_ON_FAILURE) { + final String logParams = [ + BRANCH ? "BRANCH=${BRANCH}" : null, + ].minus(null).join(' ') + + slackError(channel: SLACK_CHANNEL, message: "${logParams}") + } } } } diff --git a/src/main/resources/swagger.json b/src/main/resources/swagger.json index a66c7495..617d165c 100644 --- a/src/main/resources/swagger.json +++ b/src/main/resources/swagger.json @@ -2238,6 +2238,7 @@ "example": 100 }, "totalSize": { + "description": "Total number of rows depending on filter query parameters. If no filters are set, totalSize is the total number of rows in the table", "type": "integer", "example": 315 } diff --git a/src/main/scala/com/campudus/tableaux/database/model/TableauxModel.scala b/src/main/scala/com/campudus/tableaux/database/model/TableauxModel.scala index 48c77d1c..c1107e88 100644 --- a/src/main/scala/com/campudus/tableaux/database/model/TableauxModel.scala +++ b/src/main/scala/com/campudus/tableaux/database/model/TableauxModel.scala @@ -1235,7 +1235,7 @@ class TableauxModel( }) (_, linkDirection, _) <- structureModel.columnStruc.retrieveLinkInformation(table, linkColumn.id) - totalSize <- retrieveRowModel.sizeForeign(linkColumn, rowId, linkDirection) + totalSize <- retrieveRowModel.sizeForeign(linkColumn, rowId, linkDirection, finalFlagOpt, archivedFlagOpt) rawRows <- retrieveRowModel.retrieveForeign( linkColumn, rowId, @@ -1327,7 +1327,7 @@ class TableauxModel( implicit user: TableauxUser ): Future[RowSeq] = { for { - totalSize <- retrieveRowModel.size(table.id) + totalSize <- retrieveRowModel.size(table.id, finalFlagOpt, archivedFlagOpt) rawRows <- retrieveRowModel.retrieveAll(table.id, columns, finalFlagOpt, archivedFlagOpt, pagination) rowSeq <- mapRawRows(table, columns, rawRows) } yield { @@ -1575,8 +1575,12 @@ class TableauxModel( } yield values } - def retrieveTotalSize(table: Table): Future[Long] = { - retrieveRowModel.size(table.id) + def retrieveTotalSize( + table: Table, + finalFlagOpt: Option[Boolean] = None, + archivedFlagOpt: Option[Boolean] = None + ): Future[Long] = { + retrieveRowModel.size(table.id, finalFlagOpt, archivedFlagOpt) } def retrieveCellHistory( diff --git a/src/main/scala/com/campudus/tableaux/database/model/tableaux/RowModel.scala b/src/main/scala/com/campudus/tableaux/database/model/tableaux/RowModel.scala index 19eaeb80..38d87e87 100644 --- a/src/main/scala/com/campudus/tableaux/database/model/tableaux/RowModel.scala +++ b/src/main/scala/com/campudus/tableaux/database/model/tableaux/RowModel.scala @@ -1192,12 +1192,11 @@ class RetrieveRowModel(val connection: DatabaseConnection)( val projection = generateProjection(foreignTableId, foreignColumns) val fromClause = generateFromClause(foreignTableId) - val whereClause = generateRowAnnotationWhereClause(finalFlagOpt, archivedFlagOpt) - val cardinalityFilter = generateCardinalityFilter(linkColumn) + val rowAnnotationFilter = generateRowAnnotationFilter(finalFlagOpt, archivedFlagOpt) val shouldCheckCardinality = !linkDirection.isManyToMany - val maybeCardinalityFilter = shouldCheckCardinality match { + val cardinalityFilter = shouldCheckCardinality match { case false => "" - case true => s"AND $cardinalityFilter" + case true => generateCardinalityFilter(linkColumn) } val binds = shouldCheckCardinality match { case false => Json.arr() @@ -1208,7 +1207,7 @@ class RetrieveRowModel(val connection: DatabaseConnection)( result <- connection.query( s"""|SELECT $projection |FROM $fromClause - |WHERE TRUE $maybeCardinalityFilter $whereClause + |WHERE TRUE $cardinalityFilter $rowAnnotationFilter |GROUP BY ut.id ORDER BY ut.id $pagination""".stripMargin, binds ) @@ -1226,14 +1225,14 @@ class RetrieveRowModel(val connection: DatabaseConnection)( ): Future[Seq[RawRow]] = { val projection = generateProjection(tableId, columns) val fromClause = generateFromClause(tableId) - val whereClause = generateRowAnnotationWhereClause(finalFlagOpt, archivedFlagOpt) + val rowAnnotationFilter = generateRowAnnotationFilter(finalFlagOpt, archivedFlagOpt) for { result <- connection.query( s"""|SELECT $projection |FROM $fromClause - |WHERE TRUE $whereClause + |WHERE TRUE $rowAnnotationFilter |GROUP BY ut.id ORDER BY ut.id $pagination""".stripMargin ) } yield { @@ -1279,31 +1278,36 @@ class RetrieveRowModel(val connection: DatabaseConnection)( } } - def size(tableId: TableId): Future[Long] = { - val select = s"SELECT COUNT(*) FROM user_table_$tableId" + def size(tableId: TableId, finalFlagOpt: Option[Boolean], archivedFlagOpt: Option[Boolean]): Future[Long] = { + val rowAnnotationFilter = generateRowAnnotationFilter(finalFlagOpt, archivedFlagOpt) + val query = s"SELECT COUNT(*) FROM user_table_$tableId WHERE TRUE $rowAnnotationFilter" - connection.selectSingleValue(select) + connection.selectSingleValue(query) } - def sizeForeign(linkColumn: LinkColumn, rowId: RowId, linkDirection: LinkDirection): Future[Long] = { + def sizeForeign( + linkColumn: LinkColumn, + rowId: RowId, + linkDirection: LinkDirection, + finalFlagOpt: Option[Boolean], + archivedFlagOpt: Option[Boolean] + ): Future[Long] = { val foreignTableId = linkColumn.to.table.id - val cardinalityFilter = generateCardinalityFilter(linkColumn) - val shouldNotCheckCardinality = linkDirection.isManyToMany + val shouldCheckCardinality = !linkDirection.isManyToMany + val rowAnnotationFilter = generateRowAnnotationFilter(finalFlagOpt, archivedFlagOpt) + val cardinalityFilter = shouldCheckCardinality match { + case false => "" + case true => generateCardinalityFilter(linkColumn) + } + val binds = shouldCheckCardinality match { + case false => Json.arr() + case true => Json.arr(rowId, linkColumn.linkId, rowId, linkColumn.linkId) + } for { - maybeCardinalityFilter <- - if (shouldNotCheckCardinality) { - Future.successful("") - } else { - Future.successful(s"WHERE $cardinalityFilter") - } result <- connection.selectSingleValue[Long]( - s"SELECT COUNT(*) FROM user_table_$foreignTableId ut $maybeCardinalityFilter", - if (shouldNotCheckCardinality) { - Json.arr() - } else { - Json.arr(rowId, linkColumn.linkId, rowId, linkColumn.linkId) - } + s"SELECT COUNT(*) FROM user_table_$foreignTableId ut WHERE TRUE $cardinalityFilter $rowAnnotationFilter", + binds ) } yield { result } } @@ -1484,13 +1488,14 @@ class RetrieveRowModel(val connection: DatabaseConnection)( // linkColumn is from origin tables point of view // ... so we need to swap toSql and fromSql s""" + | AND |(SELECT COUNT(*) = 0 FROM $linkTable WHERE ${linkDirection.toSql} = ut.id AND ${linkDirection.fromSql} = ?) AND |(SELECT COUNT(*) FROM $linkTable WHERE ${linkDirection.toSql} = ut.id) < (SELECT ${linkDirection.fromCardinality} FROM system_link_table WHERE link_id = ?) AND |(SELECT COUNT(*) FROM $linkTable WHERE ${linkDirection.fromSql} = ?) < (SELECT ${linkDirection.toCardinality} FROM system_link_table WHERE link_id = ?) """.stripMargin } - private def generateRowAnnotationWhereClause( + private def generateRowAnnotationFilter( finalFlagOpt: Option[Boolean], archivedFlagOpt: Option[Boolean] ): String = { diff --git a/src/test/scala/com/campudus/tableaux/api/content/LinkConstraintTest.scala b/src/test/scala/com/campudus/tableaux/api/content/LinkConstraintTest.scala index 2df04e1e..75a58a71 100644 --- a/src/test/scala/com/campudus/tableaux/api/content/LinkConstraintTest.scala +++ b/src/test/scala/com/campudus/tableaux/api/content/LinkConstraintTest.scala @@ -36,6 +36,10 @@ sealed trait Helper extends LinkTestBase { obj.getJsonArray("rows") } + def toRowsArrayAndTotalSize(obj: JsonObject): (JsonArray, Long) = { + (obj.getJsonArray("rows"), obj.getJsonObject("page").getLong("totalSize")) + } + def createCardinalityLinkColumn( tableId: TableId, toTableId: TableId, @@ -1639,26 +1643,48 @@ class RetrieveFinalAndArchivedRows extends LinkTestBase with Helper { _ <- sendRequest("PATCH", s"/tables/1/rows/6/annotations", Json.obj("final" -> true, "archived" -> false)) _ <- sendRequest("PATCH", s"/tables/1/rows/7/annotations", Json.obj("final" -> true, "archived" -> true)) - finalFalse <- sendRequest("GET", s"/tables/1/rows?final=false").map(toRowsArray) - finalTrue <- sendRequest("GET", s"/tables/1/rows?final=true").map(toRowsArray) - archivedFalse <- sendRequest("GET", s"/tables/1/rows?archived=false").map(toRowsArray) - archivedTrue <- sendRequest("GET", s"/tables/1/rows?archived=true").map(toRowsArray) - allRows <- sendRequest("GET", s"/tables/1/rows").map(toRowsArray) - finalFalseAndArchivedFalse <- sendRequest("GET", s"/tables/1/rows?final=false&archived=false").map(toRowsArray) - finalTrueAndArchivedTrue <- sendRequest("GET", s"/tables/1/rows?final=true&archived=true").map(toRowsArray) - finalFalseAndArchivedTrue <- sendRequest("GET", s"/tables/1/rows?final=false&archived=true").map(toRowsArray) - finalTrueAndArchivedFalse <- sendRequest("GET", s"/tables/1/rows?final=true&archived=false").map(toRowsArray) + (allRows, totalSize1) <- sendRequest("GET", s"/tables/1/rows").map(toRowsArrayAndTotalSize) + + (finalFalse, totalSize2) <- sendRequest("GET", s"/tables/1/rows?final=false").map(toRowsArrayAndTotalSize) + (finalTrue, totalSize3) <- sendRequest("GET", s"/tables/1/rows?final=true").map(toRowsArrayAndTotalSize) + (archivedFalse, totalSize4) <- sendRequest("GET", s"/tables/1/rows?archived=false").map(toRowsArrayAndTotalSize) + (archivedTrue, totalSize5) <- sendRequest("GET", s"/tables/1/rows?archived=true").map(toRowsArrayAndTotalSize) + + (finalFalseAndArchivedFalse, totalSize6) <- + sendRequest("GET", s"/tables/1/rows?final=false&archived=false").map(toRowsArrayAndTotalSize) + (finalTrueAndArchivedTrue, totalSize7) <- + sendRequest("GET", s"/tables/1/rows?final=true&archived=true").map(toRowsArrayAndTotalSize) + (finalFalseAndArchivedTrue, totalSize8) <- + sendRequest("GET", s"/tables/1/rows?final=false&archived=true").map(toRowsArrayAndTotalSize) + (finalTrueAndArchivedFalse, totalSize9) <- + sendRequest("GET", s"/tables/1/rows?final=true&archived=false").map(toRowsArrayAndTotalSize) + + (finalTrueWithPagination, totalSize10) <- + sendRequest("GET", s"/tables/1/rows?final=true&limit=4&offset=0").map(toRowsArrayAndTotalSize) } yield { + assertEquals(7, allRows.size()) + assertEquals(7, totalSize1) + assertEquals(2, finalFalse.size()) + assertEquals(2, totalSize2) assertEquals(5, finalTrue.size()) + assertEquals(5, totalSize3) assertEquals(4, archivedFalse.size()) + assertEquals(4, totalSize4) assertEquals(3, archivedTrue.size()) + assertEquals(3, totalSize5) - assertEquals(7, allRows.size()) assertEquals(1, finalFalseAndArchivedFalse.size()) + assertEquals(1, totalSize6) assertEquals(2, finalTrueAndArchivedTrue.size()) + assertEquals(2, totalSize7) assertEquals(1, finalFalseAndArchivedTrue.size()) + assertEquals(1, totalSize8) assertEquals(3, finalTrueAndArchivedFalse.size()) + assertEquals(3, totalSize9) + + assertEquals(4, finalTrueWithPagination.size()) + assertEquals(5, totalSize10) } } } @@ -1678,6 +1704,7 @@ class RetrieveFinalAndArchivedRows extends LinkTestBase with Helper { resultCell <- sendRequest("GET", s"/tables/$tableId1/columns/$linkColumnId/rows/$rowId1") + resultAllForeignRows <- sendRequest("GET", s"/tables/$tableId1/columns/$linkColumnId/rows/$rowId1/foreignRows") resultForeignRowsFinal <- sendRequest("GET", s"/tables/$tableId1/columns/$linkColumnId/rows/$rowId1/foreignRows?final=true") resultForeignRowsArchived <- @@ -1685,8 +1712,11 @@ class RetrieveFinalAndArchivedRows extends LinkTestBase with Helper { } yield { assertEquals(0, resultCell.getJsonArray("value").size()) - assertEquals(2, resultForeignRowsFinal.getJsonObject("page").getLong("totalSize").longValue()) - assertEquals(2, resultForeignRowsArchived.getJsonObject("page").getLong("totalSize").longValue()) + assertEquals(2, resultAllForeignRows.getJsonObject("page").getLong("totalSize").longValue()) + assertJSONEquals(Json.arr(Json.obj("id" -> 1), Json.obj("id" -> 2)), resultAllForeignRows.getJsonArray("rows")) + + assertEquals(1, resultForeignRowsFinal.getJsonObject("page").getLong("totalSize").longValue()) + assertEquals(1, resultForeignRowsArchived.getJsonObject("page").getLong("totalSize").longValue()) assertJSONEquals(Json.arr(Json.obj("id" -> 1)), resultForeignRowsFinal.getJsonArray("rows")) assertJSONEquals(Json.arr(Json.obj("id" -> 2)), resultForeignRowsArchived.getJsonArray("rows")) } @@ -1731,14 +1761,16 @@ class RetrieveFinalAndArchivedRows extends LinkTestBase with Helper { row <- sendRequest("GET", s"/tables/$tableId1/rows/$rowId1") } yield { assertJSONEquals( - Json.fromObjectString("""|{ - | "id": 1, - | "values": [ - | "table2row1" - | ], - | "final": true - |} - |""".stripMargin), + Json.fromObjectString( + """|{ + | "id": 1, + | "values": [ + | "table2row1" + | ], + | "final": true + |} + |""".stripMargin + ), firstCells.getJsonObject(0) ) diff --git a/src/test/scala/com/campudus/tableaux/api/content/LinkTest.scala b/src/test/scala/com/campudus/tableaux/api/content/LinkTest.scala index a2992f49..360af73c 100644 --- a/src/test/scala/com/campudus/tableaux/api/content/LinkTest.scala +++ b/src/test/scala/com/campudus/tableaux/api/content/LinkTest.scala @@ -1178,21 +1178,23 @@ class LinkTest extends LinkTestBase { js.getJsonArray("columns") } } yield { - val expectedJson = Json.fromObjectString(s""" - |{ - | "status": "ok", - | "page": { - | "offset": null, - | "limit": null, - | "totalSize": 3 - | }, - | "rows": [ - | {"id": 1,"values": [[{"id": 2,"value": "blub 2"}]]}, - | {"id": 2,"values": [[{"id": 3,"value": "blub 3"}]]}, - | {"id": 3,"values": [[{"id": 1,"value": "blub 1"}]]} - | ] - |} - """.stripMargin) + val expectedJson = Json.fromObjectString( + s""" + |{ + | "status": "ok", + | "page": { + | "offset": null, + | "limit": null, + | "totalSize": 3 + | }, + | "rows": [ + | {"id": 1,"values": [[{"id": 2,"value": "blub 2"}]]}, + | {"id": 2,"values": [[{"id": 3,"value": "blub 3"}]]}, + | {"id": 3,"values": [[{"id": 1,"value": "blub 1"}]]} + | ] + |} + """.stripMargin + ) assertJSONEquals(expectedJson, rows)