fix/filter-total-size-queries

.gitignore

@@ -61,3 +61,5 @@ project/plugins/project/
 
 # local log properties
 local_logging.properties
+
+*.local*

GETTING_STARTED.md

@@ -11,7 +11,7 @@
     - [Retrieving a specific column](#retrieving-a-specific-column)
     - [Retrieving a specific row](#retrieving-a-specific-row)
     - [Retrieving a specific cell](#retrieving-a-specific-cell)
-  - [2.2. Data types & column kinds](#22-data-types--column-kinds)
+  - [2.2. Data types \& column kinds](#22-data-types--column-kinds)
     - [Primitive data types](#primitive-data-types)
       - [Examples](#examples)
     - [Complex data types](#complex-data-types)
@@ -155,7 +155,7 @@ Call this endpoint to retrieve all rows of a specific table. Most important part
   "page": { // result can be paged with two query parameters offset and limit
     "offset": null,
     "limit": null,
-    "totalSize": 18 // total size of this table
+    "totalSize": 18 // total number of rows depending on filter query parameters. If no filters are set, totalSize is the total number of rows in the table
   },
   "rows": [
     { // row object

Jenkinsfile

@@ -1,11 +1,18 @@
 @Library('campudus-jenkins-shared-lib') _
 
-IMAGE_NAME = "campudus/grud-backend"
-DEPLOY_DIR = 'build/libs'
-LEGACY_ARCHIVE_FILENAME="grud-backend-docker.tar.gz"
-DOCKER_BASE_IMAGE_TAG = "build-${BUILD_NUMBER}"
+final String BRANCH = params.BRANCH
+final boolean NOTIFY_SLACK_ON_FAILURE = params.NOTIFY_SLACK_ON_FAILURE
+final boolean NOTIFY_SLACK_ON_SUCCESS = params.NOTIFY_SLACK_ON_SUCCESS
 
-SLACK_CHANNEL = "#grud"
+final String CLEAN_GIT_BRANCH = BRANCH ? BRANCH.replaceAll("[\\.\\_\\#]", "-").tokenize('/').last() : ""
+
+final String IMAGE_NAME = "campudus/grud-backend"
+final String IMAGE_TAG = CLEAN_GIT_BRANCH && CLEAN_GIT_BRANCH != "master" ? CLEAN_GIT_BRANCH : "latest"
+final String DEPLOY_DIR = 'build/libs'
+final String LEGACY_ARCHIVE_FILENAME="grud-backend-docker.tar.gz"
+final GString DOCKER_BASE_IMAGE_TAG = "build-${BUILD_NUMBER}"
+
+final String SLACK_CHANNEL = "#grud"
 
 // flag deactivate tests for fast redeployment from jenkins frontend
 shouldTest = true
@@ -16,19 +23,23 @@ pipeline {
   environment {
     BUILD_DATE = sh(returnStdout: true, script: 'date \"+%Y-%m-%d %H:%M:%S\"').trim()
     GIT_COMMIT_DATE = sh(returnStdout: true, script: "git show -s --format=%ci").trim()
-    CLEAN_GIT_BRANCH = sh(returnStdout: true, script: "echo $GIT_BRANCH | sed 's/[\\.\\/\\_\\#]/-/g'").trim()
     COMPOSE_PROJECT_NAME = "${IMAGE_NAME}-${CLEAN_GIT_BRANCH}"
   }
 
   triggers {
-    pollSCM('H/5 * * * *')
+    githubPush()
   }
 
   options {
     timestamps()
     copyArtifactPermission('*');
   }
 
+  parameters {
+    booleanParam(name: 'NOTIFY_SLACK_ON_FAILURE', defaultValue: true, description: '')
+    booleanParam(name: 'NOTIFY_SLACK_ON_SUCCESS', defaultValue: false, description: '')
+  }
+
   stages {
     stage('Cleanup & Build base docker image') {
       steps {
@@ -45,7 +56,7 @@ pipeline {
           groovyVars = [:] << getBinding().getVariables()
           groovyVars.each  {k,v -> print "$k = $v"}
         }
-        
+
         sh "docker build -t ${IMAGE_NAME}-cacher --target=cacher ."
       }
     }
@@ -85,12 +96,13 @@ pipeline {
           --label "GIT_COMMIT_DATE=${GIT_COMMIT_DATE}" \
           --label "BUILD_DATE=${BUILD_DATE}" \
           -t ${IMAGE_NAME}:${DOCKER_BASE_IMAGE_TAG}-${GIT_COMMIT} \
-          -t ${IMAGE_NAME}:latest \
+          -t ${IMAGE_NAME}:${IMAGE_TAG} \
           -f Dockerfile \
           --rm --target=prod .
         """
+
         // Legacy, but needed for some project deployments
-        sh "docker save ${IMAGE_NAME}:latest | gzip -c > ${DEPLOY_DIR}/${LEGACY_ARCHIVE_FILENAME}"
+        sh "docker save ${IMAGE_NAME}:${IMAGE_TAG} | gzip -c > ${DEPLOY_DIR}/${LEGACY_ARCHIVE_FILENAME}"
       }
     }
 
@@ -105,7 +117,7 @@ pipeline {
       steps {
         withDockerRegistry([ credentialsId: "dockerhub", url: "" ]) {
           sh "docker push ${IMAGE_NAME}:${DOCKER_BASE_IMAGE_TAG}-${GIT_COMMIT}"
-          sh "docker push ${IMAGE_NAME}:latest"
+          sh "docker push ${IMAGE_NAME}:${IMAGE_TAG}"
         }
       }
     }
@@ -115,17 +127,28 @@ pipeline {
     success {
       wrap([$class: 'BuildUser']) {
         script {
-          sh "echo successful"
-          slackOk(channel: SLACK_CHANNEL, message: "Image pushed to docker registry: ${IMAGE_NAME}:${DOCKER_BASE_IMAGE_TAG}-${GIT_COMMIT}")
+          if (NOTIFY_SLACK_ON_SUCCESS) {
+            final String logParams = [
+                BRANCH ? "BRANCH=${BRANCH}" : null,
+                "image: ${IMAGE_NAME}:${IMAGE_TAG}",
+            ].minus(null).join(' ')
+
+            slackOk(channel: SLACK_CHANNEL, message: "${logParams}")
+          }
         }
       }
     }
 
     failure {
       wrap([$class: 'BuildUser']) {
         script {
-          sh "echo failed"
-          slackError(channel: SLACK_CHANNEL)
+          if (NOTIFY_SLACK_ON_FAILURE) {
+            final String logParams = [
+                BRANCH ? "BRANCH=${BRANCH}" : null,
+            ].minus(null).join(' ')
+
+            slackError(channel: SLACK_CHANNEL, message: "${logParams}")
+          }
         }
       }
     }

src/main/resources/swagger.json

@@ -2238,6 +2238,7 @@
               "example": 100
             },
             "totalSize": {
+              "description": "Total number of rows depending on filter query parameters. If no filters are set, totalSize is the total number of rows in the table",
               "type": "integer",
               "example": 315
             }

src/main/scala/com/campudus/tableaux/database/model/TableauxModel.scala

@@ -1235,7 +1235,7 @@ class TableauxModel(
         })
 
       (_, linkDirection, _) <- structureModel.columnStruc.retrieveLinkInformation(table, linkColumn.id)
-      totalSize <- retrieveRowModel.sizeForeign(linkColumn, rowId, linkDirection)
+      totalSize <- retrieveRowModel.sizeForeign(linkColumn, rowId, linkDirection, finalFlagOpt, archivedFlagOpt)
       rawRows <- retrieveRowModel.retrieveForeign(
         linkColumn,
         rowId,
@@ -1327,7 +1327,7 @@ class TableauxModel(
       implicit user: TableauxUser
   ): Future[RowSeq] = {
     for {
-      totalSize <- retrieveRowModel.size(table.id)
+      totalSize <- retrieveRowModel.size(table.id, finalFlagOpt, archivedFlagOpt)
       rawRows <- retrieveRowModel.retrieveAll(table.id, columns, finalFlagOpt, archivedFlagOpt, pagination)
       rowSeq <- mapRawRows(table, columns, rawRows)
     } yield {
@@ -1575,8 +1575,12 @@ class TableauxModel(
     } yield values
   }
 
-  def retrieveTotalSize(table: Table): Future[Long] = {
-    retrieveRowModel.size(table.id)
+  def retrieveTotalSize(
+      table: Table,
+      finalFlagOpt: Option[Boolean] = None,
+      archivedFlagOpt: Option[Boolean] = None
+  ): Future[Long] = {
+    retrieveRowModel.size(table.id, finalFlagOpt, archivedFlagOpt)
   }
 
   def retrieveCellHistory(

src/main/scala/com/campudus/tableaux/database/model/tableaux/RowModel.scala

@@ -1192,12 +1192,11 @@ class RetrieveRowModel(val connection: DatabaseConnection)(
 
     val projection = generateProjection(foreignTableId, foreignColumns)
     val fromClause = generateFromClause(foreignTableId)
-    val whereClause = generateRowAnnotationWhereClause(finalFlagOpt, archivedFlagOpt)
-    val cardinalityFilter = generateCardinalityFilter(linkColumn)
+    val rowAnnotationFilter = generateRowAnnotationFilter(finalFlagOpt, archivedFlagOpt)
     val shouldCheckCardinality = !linkDirection.isManyToMany
-    val maybeCardinalityFilter = shouldCheckCardinality match {
+    val cardinalityFilter = shouldCheckCardinality match {
       case false => ""
-      case true => s"AND $cardinalityFilter"
+      case true => generateCardinalityFilter(linkColumn)
     }
     val binds = shouldCheckCardinality match {
       case false => Json.arr()
@@ -1208,7 +1207,7 @@ class RetrieveRowModel(val connection: DatabaseConnection)(
       result <- connection.query(
         s"""|SELECT $projection
             |FROM $fromClause
-            |WHERE TRUE $maybeCardinalityFilter $whereClause
+            |WHERE TRUE $cardinalityFilter $rowAnnotationFilter
             |GROUP BY ut.id ORDER BY ut.id $pagination""".stripMargin,
         binds
       )
@@ -1226,14 +1225,14 @@ class RetrieveRowModel(val connection: DatabaseConnection)(
   ): Future[Seq[RawRow]] = {
     val projection = generateProjection(tableId, columns)
     val fromClause = generateFromClause(tableId)
-    val whereClause = generateRowAnnotationWhereClause(finalFlagOpt, archivedFlagOpt)
+    val rowAnnotationFilter = generateRowAnnotationFilter(finalFlagOpt, archivedFlagOpt)
 
     for {
       result <-
         connection.query(
           s"""|SELECT $projection
               |FROM $fromClause
-              |WHERE TRUE $whereClause
+              |WHERE TRUE $rowAnnotationFilter
               |GROUP BY ut.id ORDER BY ut.id $pagination""".stripMargin
         )
     } yield {
@@ -1279,31 +1278,36 @@ class RetrieveRowModel(val connection: DatabaseConnection)(
     }
   }
 
-  def size(tableId: TableId): Future[Long] = {
-    val select = s"SELECT COUNT(*) FROM user_table_$tableId"
+  def size(tableId: TableId, finalFlagOpt: Option[Boolean], archivedFlagOpt: Option[Boolean]): Future[Long] = {
+    val rowAnnotationFilter = generateRowAnnotationFilter(finalFlagOpt, archivedFlagOpt)
+    val query = s"SELECT COUNT(*) FROM user_table_$tableId WHERE TRUE $rowAnnotationFilter"
 
-    connection.selectSingleValue(select)
+    connection.selectSingleValue(query)
   }
 
-  def sizeForeign(linkColumn: LinkColumn, rowId: RowId, linkDirection: LinkDirection): Future[Long] = {
+  def sizeForeign(
+      linkColumn: LinkColumn,
+      rowId: RowId,
+      linkDirection: LinkDirection,
+      finalFlagOpt: Option[Boolean],
+      archivedFlagOpt: Option[Boolean]
+  ): Future[Long] = {
     val foreignTableId = linkColumn.to.table.id
-    val cardinalityFilter = generateCardinalityFilter(linkColumn)
-    val shouldNotCheckCardinality = linkDirection.isManyToMany
+    val shouldCheckCardinality = !linkDirection.isManyToMany
+    val rowAnnotationFilter = generateRowAnnotationFilter(finalFlagOpt, archivedFlagOpt)
+    val cardinalityFilter = shouldCheckCardinality match {
+      case false => ""
+      case true => generateCardinalityFilter(linkColumn)
+    }
+    val binds = shouldCheckCardinality match {
+      case false => Json.arr()
+      case true => Json.arr(rowId, linkColumn.linkId, rowId, linkColumn.linkId)
+    }
 
     for {
-      maybeCardinalityFilter <-
-        if (shouldNotCheckCardinality) {
-          Future.successful("")
-        } else {
-          Future.successful(s"WHERE $cardinalityFilter")
-        }
       result <- connection.selectSingleValue[Long](
-        s"SELECT COUNT(*) FROM user_table_$foreignTableId ut $maybeCardinalityFilter",
-        if (shouldNotCheckCardinality) {
-          Json.arr()
-        } else {
-          Json.arr(rowId, linkColumn.linkId, rowId, linkColumn.linkId)
-        }
+        s"SELECT COUNT(*) FROM user_table_$foreignTableId ut WHERE TRUE $cardinalityFilter $rowAnnotationFilter",
+        binds
       )
     } yield { result }
   }
@@ -1484,13 +1488,14 @@ class RetrieveRowModel(val connection: DatabaseConnection)(
     // linkColumn is from origin tables point of view
     // ... so we need to swap toSql and fromSql
     s"""
+       | AND
        |(SELECT COUNT(*) = 0 FROM $linkTable WHERE ${linkDirection.toSql} = ut.id AND ${linkDirection.fromSql} = ?) AND
        |(SELECT COUNT(*) FROM $linkTable WHERE ${linkDirection.toSql} = ut.id) < (SELECT ${linkDirection.fromCardinality} FROM system_link_table WHERE link_id = ?) AND
        |(SELECT COUNT(*) FROM $linkTable WHERE ${linkDirection.fromSql} = ?) < (SELECT ${linkDirection.toCardinality} FROM system_link_table WHERE link_id = ?)
        """.stripMargin
   }
 
-  private def generateRowAnnotationWhereClause(
+  private def generateRowAnnotationFilter(
       finalFlagOpt: Option[Boolean],
       archivedFlagOpt: Option[Boolean]
   ): String = {