From 69a4e4c117f0098143374f424537efda35b82fb9 Mon Sep 17 00:00:00 2001
From: Tom Herold <tom.herold@scalableminds.com>
Date: Thu, 24 Oct 2024 16:00:18 +0200
Subject: [PATCH 01/39] convert circle ci config to github actions

---
 .github/not-on-master.sh                |   8 +
 .github/workflows/build_test_deploy.yml | 199 +++++++++++++++++++++++-
 2 files changed, 205 insertions(+), 2 deletions(-)
 create mode 100644 .github/not-on-master.sh

diff --git a/.github/not-on-master.sh b/.github/not-on-master.sh
new file mode 100644
index 0000000000..fc9ccc8fc0
--- /dev/null
+++ b/.github/not-on-master.sh
@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+if [ "${GITHUB_REF}" == "master" ]; then
+  echo "Skipping this step on master..."
+else
+  exec "$@"
+fi
diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml
index 599e2b22e0..17f93d24d9 100644
--- a/.github/workflows/build_test_deploy.yml
+++ b/.github/workflows/build_test_deploy.yml
@@ -1,13 +1,208 @@
 name: CI Pipeline
 
 on:
+  push:
+    branches:
+      - '*'
+  pull_request:
+    branches:
+      - '*'
   workflow_dispatch:
 
+env:
+  USER_NAME: circleci
+  USER_UID: 1000
+  USER_GID: 1000
+  TZ: Europe/Berlin
+  DOCKER_USER: ${{ secrets.DOCKER_USER }}
+  DOCKER_PASS: ${{ secrets.DOCKER_PASS }}
+
 jobs:
-  foo:
+  static_frontent_code_checks:
     runs-on: ubuntu-20.04
     steps:
     - name: Checkout code
       uses: actions/checkout@v3
       with:
-        fetch-depth: 5
\ No newline at end of file
+        fetch-depth: 5
+
+    - uses: actions/setup-node@v4
+      with:
+          node-version: 18
+
+    - name: Install frontend dependencies
+      run: corepack enable && yarn install --immutable
+
+    - name: Lint frontend code and check formatting
+      run: yarn run check-frontend
+
+    - name: Typecheck frontend code
+      run: yarn typecheck
+
+    - name: Check for cyclic dependencies in frontend
+      run: yarn check-cyclic-dependencies
+
+
+
+
+  build_test_deploy:
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 5
+
+      - name: "Custom environment variables"
+        run: |
+          if [[ ${{ github.ref_type }} == "branch" ]]; then
+            NORMALIZED_BRANCH=$(echo ${{ github.ref_name }} | sed 's/[\/-]/_/g')
+            echo "NORMALIZED_BRANCH=$NORMALIZED_BRANCH" >> $GITHUB_ENV
+            DOCKER_TAG="${NORMALIZED_BRANCH}__${{ github.run_number }}"
+            echo "DOCKER_TAG=$DOCKER_TAG" >> $GITHUB_ENV
+          else
+            echo "NORMALIZED_BRANCH=master" >> $GITHUB_ENV
+            echo "DOCKER_TAG=${{ github.ref_name }}" >> $GITHUB_ENV
+          fi
+
+      - name: Set up Docker
+        uses: docker/setup-buildx-action@v2
+
+      - name: Build webknossos-dev docker image
+        run: |
+          docker pull scalableminds/webknossos-dev:$NORMALIZED_BRANCH || true
+          DEV_CACHE=$NORMALIZED_BRANCH docker-compose build base
+
+      - name: Prepare dependency folders
+        run: mkdir -p project/target target ~/.ivy2 ~/.cache/coursier
+
+      - name: Install frontend dependencies
+        run: docker-compose run -e PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true base yarn install --immutable
+
+      - name: Assert unique evolution numbers
+        run: docker-compose run base tools/postgres/dbtool.js assert-unique-evolution-numbers
+
+      - name: Assert schema.sql and evolutions are equal
+        run: |
+          docker-compose up -d postgres
+          sleep 3
+          docker-compose run compile tools/postgres/dbtool.js check-evolutions-schema
+
+      - name: Build frontend documentation
+        run: |
+          WK_VERSION=${{ github.event.release.tag_name || github.run_number || 'dev' }}
+          docker-compose run base yarn run docs --project-version $WK_VERSION
+
+      - name: Build webknossos (webpack)
+        run: docker-compose run base yarn build
+
+      - name: Build webknossos (sbt)
+        run: |
+          if [ "${{ github.ref }}" == "refs/heads/master" ]; then
+            docker-compose run compile sbt -no-colors clean compile stage
+          else
+            docker-compose run compile sbt -no-colors -DfailOnWarning compile stage
+          fi
+
+      - name: Build webknossos-datastore (sbt)
+        run: docker-compose run base sbt -no-colors -DfailOnWarning "project webknossosDatastore" copyMessages compile stage
+
+      - name: Build webknossos-tracingstore (sbt)
+        run: docker-compose run base sbt -no-colors -DfailOnWarning "project webknossosTracingstore" copyMessages compile stage
+
+      - name: Checksum App Dirs
+        run: find app webknossos-datastore/app webknossos-tracingstore/app -type f -exec md5sum {} \; | sort -k 2 | md5sum > app_checksum.txt
+
+      - name: Build webknossos docker image
+        run: |
+          docker pull scalableminds/webknossos:$NORMALIZED_BRANCH || true
+          DEV_CACHE=$NORMALIZED_BRANCH docker-compose build --pull webknossos
+
+      - name: Build webknossos-datastore docker image
+        run: docker-compose build --pull webknossos-datastore
+
+      - name: Build webknossos-tracingstore docker image
+        run: docker-compose build --pull webknossos-tracingstore
+
+      - name: Run frontend tests
+        run: .github/not-on-master.sh docker-compose run base yarn test-verbose
+
+      - name: Lint backend code and check formatting
+        run: .github/not-on-master.sh docker-compose run backend-lint-format
+
+      - name: Run backend tests
+        run: .github/not-on-master.sh docker-compose run backend-tests
+
+      - name: Run end-to-end tests
+        run: |
+          for i in {1..3}; do # retry
+            .github/not-on-master.sh docker-compose run e2e-tests && s=0 && break || s=$?
+          done
+          (exit $s)
+
+      - name: Validate frontend types
+        run: .github/not-on-master.sh docker-compose run base yarn typecheck
+
+      - name: Start webknossos, datastore, and tracingstore
+        run: |
+          docker-compose up -d webknossos
+          docker-compose up -d webknossos-datastore
+          docker-compose up -d webknossos-tracingstore
+
+      - name: Run webknossos smoke test
+        run: |
+          for i in {1..20}; do # retry
+            curl --fail -v http://localhost:9000/api/health && s=0 && break || s=$?
+            sleep 5
+          done
+          (exit $s)
+
+      - name: Run webknossos-datastore smoke test
+        run: |
+          for i in {1..20}; do # retry
+            curl --fail -v http://localhost:9090/data/health && s=0 && break || s=$?
+            sleep 5
+          done
+          (exit $s)
+
+      - name: Run webknossos-tracingstore smoke test
+        run: |
+          for i in {1..20}; do # retry
+            curl --fail -v http://localhost:9050/tracings/health && s=0 && break || s=$?
+            sleep 5
+          done
+          (exit $s)
+
+      - name: Stop webknossos, datastore, and tracingstore
+        run: docker-compose down --volumes --remove-orphans
+
+      - name: Push docker images
+        run: |
+          function retry() {
+            for i in {1..5}; do
+              "$@" && s=0 && break || s=$?
+              sleep 10
+            done
+            return $s
+          }
+          retry docker login -u $DOCKER_USER -p $DOCKER_PASS
+          retry docker-compose push webknossos
+          retry docker-compose push webknossos-datastore
+          retry docker-compose push webknossos-tracingstore
+          if [[ ${{ github.ref_type }} == "branch" ]]; then
+            docker tag scalableminds/webknossos:${DOCKER_TAG} scalableminds/webknossos:${NORMALIZED_BRANCH}
+            retry docker push scalableminds/webknossos:${NORMALIZED_BRANCH}
+            docker tag scalableminds/webknossos-datastore:${DOCKER_TAG} scalableminds/webknossos-datastore:${NORMALIZED_BRANCH}
+            retry docker push scalableminds/webknossos-datastore:${NORMALIZED_BRANCH}
+            docker tag scalableminds/webknossos-tracingstore:${DOCKER_TAG} scalableminds/webknossos-tracingstore:${NORMALIZED_BRANCH}
+            retry docker push scalableminds/webknossos-tracingstore:${NORMALIZED_BRANCH}
+            docker tag scalableminds/webknossos-dev scalableminds/webknossos-dev:${NORMALIZED_BRANCH}
+            retry docker push scalableminds/webknossos-dev:${NORMALIZED_BRANCH}
+          fi
+          docker logout
+
+      - name: Report coverage
+        run: .github/not-on-master.sh docker-compose run base yarn coverage || true
+
+      # - name: Send Slack notification (master only)
+      #   run: .circleci/slack-notification.sh

From 4a9811eff7c14db0c0b99cbd77d3bd6459892cfc Mon Sep 17 00:00:00 2001
From: Tom Herold <tom.herold@scalableminds.com>
Date: Thu, 24 Oct 2024 16:03:40 +0200
Subject: [PATCH 02/39] use new docker compose syntax

---
 .github/workflows/build_test_deploy.yml | 54 ++++++++++++-------------
 1 file changed, 27 insertions(+), 27 deletions(-)

diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml
index 17f93d24d9..506cf1ad82 100644
--- a/.github/workflows/build_test_deploy.yml
+++ b/.github/workflows/build_test_deploy.yml
@@ -71,44 +71,44 @@ jobs:
       - name: Build webknossos-dev docker image
         run: |
           docker pull scalableminds/webknossos-dev:$NORMALIZED_BRANCH || true
-          DEV_CACHE=$NORMALIZED_BRANCH docker-compose build base
+          DEV_CACHE=$NORMALIZED_BRANCH docker compose build base
 
       - name: Prepare dependency folders
         run: mkdir -p project/target target ~/.ivy2 ~/.cache/coursier
 
       - name: Install frontend dependencies
-        run: docker-compose run -e PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true base yarn install --immutable
+        run: docker compose run -e PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true base yarn install --immutable
 
       - name: Assert unique evolution numbers
-        run: docker-compose run base tools/postgres/dbtool.js assert-unique-evolution-numbers
+        run: docker compose run base tools/postgres/dbtool.js assert-unique-evolution-numbers
 
       - name: Assert schema.sql and evolutions are equal
         run: |
-          docker-compose up -d postgres
+          docker compose up -d postgres
           sleep 3
-          docker-compose run compile tools/postgres/dbtool.js check-evolutions-schema
+          docker compose run compile tools/postgres/dbtool.js check-evolutions-schema
 
       - name: Build frontend documentation
         run: |
           WK_VERSION=${{ github.event.release.tag_name || github.run_number || 'dev' }}
-          docker-compose run base yarn run docs --project-version $WK_VERSION
+          docker compose run base yarn run docs --project-version $WK_VERSION
 
       - name: Build webknossos (webpack)
-        run: docker-compose run base yarn build
+        run: docker compose run base yarn build
 
       - name: Build webknossos (sbt)
         run: |
           if [ "${{ github.ref }}" == "refs/heads/master" ]; then
-            docker-compose run compile sbt -no-colors clean compile stage
+            docker compose run compile sbt -no-colors clean compile stage
           else
-            docker-compose run compile sbt -no-colors -DfailOnWarning compile stage
+            docker compose run compile sbt -no-colors -DfailOnWarning compile stage
           fi
 
       - name: Build webknossos-datastore (sbt)
-        run: docker-compose run base sbt -no-colors -DfailOnWarning "project webknossosDatastore" copyMessages compile stage
+        run: docker compose run base sbt -no-colors -DfailOnWarning "project webknossosDatastore" copyMessages compile stage
 
       - name: Build webknossos-tracingstore (sbt)
-        run: docker-compose run base sbt -no-colors -DfailOnWarning "project webknossosTracingstore" copyMessages compile stage
+        run: docker compose run base sbt -no-colors -DfailOnWarning "project webknossosTracingstore" copyMessages compile stage
 
       - name: Checksum App Dirs
         run: find app webknossos-datastore/app webknossos-tracingstore/app -type f -exec md5sum {} \; | sort -k 2 | md5sum > app_checksum.txt
@@ -116,38 +116,38 @@ jobs:
       - name: Build webknossos docker image
         run: |
           docker pull scalableminds/webknossos:$NORMALIZED_BRANCH || true
-          DEV_CACHE=$NORMALIZED_BRANCH docker-compose build --pull webknossos
+          DEV_CACHE=$NORMALIZED_BRANCH docker compose build --pull webknossos
 
       - name: Build webknossos-datastore docker image
-        run: docker-compose build --pull webknossos-datastore
+        run: docker compose build --pull webknossos-datastore
 
       - name: Build webknossos-tracingstore docker image
-        run: docker-compose build --pull webknossos-tracingstore
+        run: docker compose build --pull webknossos-tracingstore
 
       - name: Run frontend tests
-        run: .github/not-on-master.sh docker-compose run base yarn test-verbose
+        run: .github/not-on-master.sh docker compose run base yarn test-verbose
 
       - name: Lint backend code and check formatting
-        run: .github/not-on-master.sh docker-compose run backend-lint-format
+        run: .github/not-on-master.sh docker compose run backend-lint-format
 
       - name: Run backend tests
-        run: .github/not-on-master.sh docker-compose run backend-tests
+        run: .github/not-on-master.sh docker compose run backend-tests
 
       - name: Run end-to-end tests
         run: |
           for i in {1..3}; do # retry
-            .github/not-on-master.sh docker-compose run e2e-tests && s=0 && break || s=$?
+            .github/not-on-master.sh docker compose run e2e-tests && s=0 && break || s=$?
           done
           (exit $s)
 
       - name: Validate frontend types
-        run: .github/not-on-master.sh docker-compose run base yarn typecheck
+        run: .github/not-on-master.sh docker compose run base yarn typecheck
 
       - name: Start webknossos, datastore, and tracingstore
         run: |
-          docker-compose up -d webknossos
-          docker-compose up -d webknossos-datastore
-          docker-compose up -d webknossos-tracingstore
+          docker compose up -d webknossos
+          docker compose up -d webknossos-datastore
+          docker compose up -d webknossos-tracingstore
 
       - name: Run webknossos smoke test
         run: |
@@ -174,7 +174,7 @@ jobs:
           (exit $s)
 
       - name: Stop webknossos, datastore, and tracingstore
-        run: docker-compose down --volumes --remove-orphans
+        run: docker compose down --volumes --remove-orphans
 
       - name: Push docker images
         run: |
@@ -186,9 +186,9 @@ jobs:
             return $s
           }
           retry docker login -u $DOCKER_USER -p $DOCKER_PASS
-          retry docker-compose push webknossos
-          retry docker-compose push webknossos-datastore
-          retry docker-compose push webknossos-tracingstore
+          retry docker compose push webknossos
+          retry docker compose push webknossos-datastore
+          retry docker compose push webknossos-tracingstore
           if [[ ${{ github.ref_type }} == "branch" ]]; then
             docker tag scalableminds/webknossos:${DOCKER_TAG} scalableminds/webknossos:${NORMALIZED_BRANCH}
             retry docker push scalableminds/webknossos:${NORMALIZED_BRANCH}
@@ -202,7 +202,7 @@ jobs:
           docker logout
 
       - name: Report coverage
-        run: .github/not-on-master.sh docker-compose run base yarn coverage || true
+        run: .github/not-on-master.sh docker compose run base yarn coverage || true
 
       # - name: Send Slack notification (master only)
       #   run: .circleci/slack-notification.sh

From d0eed33509de93c2e96cb5dfbf86c612a3053347 Mon Sep 17 00:00:00 2001
From: Tom Herold <tom.herold@scalableminds.com>
Date: Thu, 24 Oct 2024 16:09:27 +0200
Subject: [PATCH 03/39] only run github actions on PRs and master branch

---
 .github/workflows/build_test_deploy.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml
index 506cf1ad82..57e90dfd66 100644
--- a/.github/workflows/build_test_deploy.yml
+++ b/.github/workflows/build_test_deploy.yml
@@ -3,14 +3,14 @@ name: CI Pipeline
 on:
   push:
     branches:
-      - '*'
+      - master
   pull_request:
     branches:
       - '*'
   workflow_dispatch:
 
 env:
-  USER_NAME: circleci
+  USER_NAME: root
   USER_UID: 1000
   USER_GID: 1000
   TZ: Europe/Berlin
@@ -18,7 +18,7 @@ env:
   DOCKER_PASS: ${{ secrets.DOCKER_PASS }}
 
 jobs:
-  static_frontent_code_checks:
+  static_frontend_code_checks:
     runs-on: ubuntu-20.04
     steps:
     - name: Checkout code

From 53e35c463c9ae9ed03eefd3f398afb29ddfd3893 Mon Sep 17 00:00:00 2001
From: Tom Herold <tom.herold@scalableminds.com>
Date: Mon, 28 Oct 2024 08:31:45 +0100
Subject: [PATCH 04/39] CI

---
 .github/actions/health_check_action.yml | 16 ++++++++++++++
 .github/workflows/build_test_deploy.yml | 29 +++++++++----------------
 docker-compose.yml                      |  4 ++--
 project/BuildInfoSettings.scala         |  4 ++--
 4 files changed, 30 insertions(+), 23 deletions(-)
 create mode 100644 .github/actions/health_check_action.yml

diff --git a/.github/actions/health_check_action.yml b/.github/actions/health_check_action.yml
new file mode 100644
index 0000000000..36d180d96e
--- /dev/null
+++ b/.github/actions/health_check_action.yml
@@ -0,0 +1,16 @@
+name: 'URL Health Check'
+description: 'Performs URL health check with retries'
+inputs:
+  url:
+    description: 'Health check URL'
+    required: true
+runs:
+  using: 'composite'
+  steps:
+    - shell: bash
+      run: |
+        for i in {1..20}; do
+          curl --fail -v "${{ inputs.url }}" && exit 0
+          sleep 5
+        done
+        exit 1
\ No newline at end of file
diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml
index 57e90dfd66..1e4a0614f8 100644
--- a/.github/workflows/build_test_deploy.yml
+++ b/.github/workflows/build_test_deploy.yml
@@ -13,7 +13,6 @@ env:
   USER_NAME: root
   USER_UID: 1000
   USER_GID: 1000
-  TZ: Europe/Berlin
   DOCKER_USER: ${{ secrets.DOCKER_USER }}
   DOCKER_PASS: ${{ secrets.DOCKER_PASS }}
 
@@ -150,28 +149,20 @@ jobs:
           docker compose up -d webknossos-tracingstore
 
       - name: Run webknossos smoke test
-        run: |
-          for i in {1..20}; do # retry
-            curl --fail -v http://localhost:9000/api/health && s=0 && break || s=$?
-            sleep 5
-          done
-          (exit $s)
+        uses: ./.github/actions/health_check_action
+        with: 
+          url: http://localhost:9000/api/health
+          
 
       - name: Run webknossos-datastore smoke test
-        run: |
-          for i in {1..20}; do # retry
-            curl --fail -v http://localhost:9090/data/health && s=0 && break || s=$?
-            sleep 5
-          done
-          (exit $s)
+        uses: ./.github/actions/health_check_action
+        with: 
+          url: http://localhost:9090/data/health
 
       - name: Run webknossos-tracingstore smoke test
-        run: |
-          for i in {1..20}; do # retry
-            curl --fail -v http://localhost:9050/tracings/health && s=0 && break || s=$?
-            sleep 5
-          done
-          (exit $s)
+        uses: ./.github/actions/health_check_action
+        with: 
+          url: http://localhost:9050/tracings/health
 
       - name: Stop webknossos, datastore, and tracingstore
         run: docker compose down --volumes --remove-orphans
diff --git a/docker-compose.yml b/docker-compose.yml
index 76f802426b..70232a2356 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -105,8 +105,8 @@ services:
       - POSTGRES_USER=webknossos_user
       - POSTGRES_PASSWORD=secret_password
       - HOME=/root
-      - CIRCLE_TAG=${CIRCLE_TAG}
-      - CIRCLE_BUILD_NUM=${CIRCLE_BUILD_NUM}
+      - CI_TAG=${GITHUB_REF}
+      - CI_BUILD_NUM=${GITHUB_JOB}
       - NODE_OPTIONS=--max_old_space_size=2048
       # This will be picked up by ava so that the tests fail if a snapshot doesn't exist during CI, see https://github.com/avajs/ava/issues/1585
       - CI
diff --git a/project/BuildInfoSettings.scala b/project/BuildInfoSettings.scala
index a16d0223fe..229e55b575 100644
--- a/project/BuildInfoSettings.scala
+++ b/project/BuildInfoSettings.scala
@@ -13,8 +13,8 @@ object BuildInfoSettings {
     }
 
   val ciBuild: String =
-    if (System.getenv().containsKey("CIRCLE_BUILD_NUM")) System.getenv().get("CIRCLE_BUILD_NUM") else ""
-  val ciTag: String = if (System.getenv().containsKey("CIRCLE_TAG")) System.getenv().get("CIRCLE_TAG") else ""
+    if (System.getenv().containsKey("CI_BUILD_NUM")) System.getenv().get("CI_BUILD_NUM") else ""
+  val ciTag: String = if (System.getenv().containsKey("CI_TAG")) System.getenv().get("CI_TAG") else ""
 
   def commitHash: String = getStdoutFromCommand("git rev-parse HEAD", "<getting commit hash failed>")
   def commitDate: String = getStdoutFromCommand("git log -1 --format=%cd ", "<getting git date failed>")

From 53d1c23a65ca7f3c2f78567bfdb3e33738edccc1 Mon Sep 17 00:00:00 2001
From: Tom Herold <tom.herold@scalableminds.com>
Date: Mon, 28 Oct 2024 08:41:17 +0100
Subject: [PATCH 05/39] fix yarn lockfile

---
 yarn.lock | 22 ++++++----------------
 1 file changed, 6 insertions(+), 16 deletions(-)

diff --git a/yarn.lock b/yarn.lock
index f1575bb6d8..addd51b547 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -8109,7 +8109,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"lodash-es@npm:^4.17.21, lodash-es@npm:^4.2.1":
+"lodash-es@npm:^4.17.21":
   version: 4.17.21
   resolution: "lodash-es@npm:4.17.21"
   checksum: 10c0/fb407355f7e6cd523a9383e76e6b455321f0f153a6c9625e21a8827d10c54c2a2341bd2ae8d034358b60e07325e1330c14c224ff582d04612a46a4f0479ff2f2
@@ -8144,7 +8144,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"lodash@npm:^4.17.14, lodash@npm:^4.17.15, lodash@npm:^4.17.21, lodash@npm:^4.2.1":
+"lodash@npm:^4.17.14, lodash@npm:^4.17.15, lodash@npm:^4.17.21":
   version: 4.17.21
   resolution: "lodash@npm:4.17.21"
   checksum: 10c0/d8cbea072bb08655bb4c989da418994b073a608dffa608b09ac04b43a791b12aeae7cd7ad919aa4c925f33b48490b5cfe6c1f71d827956071dae2e7bb3a6b74c
@@ -11587,14 +11587,11 @@ __metadata:
   linkType: hard
 
 "redux@npm:^4.2.0":
-  version: 3.7.2
-  resolution: "redux@npm:3.7.2"
+  version: 4.2.1
+  resolution: "redux@npm:4.2.1"
   dependencies:
-    lodash: "npm:^4.2.1"
-    lodash-es: "npm:^4.2.1"
-    loose-envify: "npm:^1.1.0"
-    symbol-observable: "npm:^1.0.3"
-  checksum: 10c0/544456f95734de33326637b370894addb57d9de2524edf36a20e4a326d0a36a0e223979d027545c5aa8a8d7a2859363981f63d1146401b72df0d16f373dd09cb
+    "@babel/runtime": "npm:^7.9.2"
+  checksum: 10c0/136d98b3d5dbed1cd6279c8c18a6a74c416db98b8a432a46836bdd668475de6279a2d4fd9d1363f63904e00f0678a8a3e7fa532c897163340baf1e71bb42c742
   languageName: node
   linkType: hard
 
@@ -12835,13 +12832,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"symbol-observable@npm:^1.0.3":
-  version: 1.2.0
-  resolution: "symbol-observable@npm:1.2.0"
-  checksum: 10c0/009fee50798ef80ed4b8195048288f108b03de162db07493f2e1fd993b33fafa72d659e832b584da5a2427daa78e5a738fb2a9ab027ee9454252e0bedbcd1fdc
-  languageName: node
-  linkType: hard
-
 "symbol-tree@npm:^3.2.4":
   version: 3.2.4
   resolution: "symbol-tree@npm:3.2.4"

From eb8af0bf2c7ed41d0571475a369c2a8c05f3d378 Mon Sep 17 00:00:00 2001
From: Tom Herold <tom.herold@scalableminds.com>
Date: Mon, 28 Oct 2024 09:00:09 +0100
Subject: [PATCH 06/39] CI

---
 .github/workflows/build_test_deploy.yml | 4 ++--
 docker-compose.yml                      | 2 --
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml
index 1e4a0614f8..8afc349cd7 100644
--- a/.github/workflows/build_test_deploy.yml
+++ b/.github/workflows/build_test_deploy.yml
@@ -10,7 +10,7 @@ on:
   workflow_dispatch:
 
 env:
-  USER_NAME: root
+  USER_NAME: webknossos
   USER_UID: 1000
   USER_GID: 1000
   DOCKER_USER: ${{ secrets.DOCKER_USER }}
@@ -76,7 +76,7 @@ jobs:
         run: mkdir -p project/target target ~/.ivy2 ~/.cache/coursier
 
       - name: Install frontend dependencies
-        run: docker compose run -e PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true base yarn install --immutable
+        run: docker compose run -e PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true base yarn install --immutable && yarn cache clean
 
       - name: Assert unique evolution numbers
         run: docker compose run base tools/postgres/dbtool.js assert-unique-evolution-numbers
diff --git a/docker-compose.yml b/docker-compose.yml
index 70232a2356..131b190dbf 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,5 +1,3 @@
-version: "2.2"
-
 services:
   # Production
   webknossos:

From 1d77e3c8048cb4ca7455196d80358f0c5d2f10df Mon Sep 17 00:00:00 2001
From: Tom Herold <tom.herold@scalableminds.com>
Date: Mon, 28 Oct 2024 09:07:37 +0100
Subject: [PATCH 07/39] CI

---
 .github/workflows/build_test_deploy.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml
index 8afc349cd7..b6e79a27a0 100644
--- a/.github/workflows/build_test_deploy.yml
+++ b/.github/workflows/build_test_deploy.yml
@@ -10,9 +10,9 @@ on:
   workflow_dispatch:
 
 env:
-  USER_NAME: webknossos
-  USER_UID: 1000
-  USER_GID: 1000
+  # USER_NAME: root
+  # USER_UID: 1000
+  # USER_GID: 1000
   DOCKER_USER: ${{ secrets.DOCKER_USER }}
   DOCKER_PASS: ${{ secrets.DOCKER_PASS }}
 

From 03c1cbfa47b83b3bc6d95dabc53cfb9c5dabb0b1 Mon Sep 17 00:00:00 2001
From: Tom Herold <tom.herold@scalableminds.com>
Date: Mon, 28 Oct 2024 09:31:09 +0100
Subject: [PATCH 08/39] Ci

---
 .github/workflows/build_test_deploy.yml | 8 +++-----
 Dockerfile                              | 1 +
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml
index b6e79a27a0..b99fa61d6e 100644
--- a/.github/workflows/build_test_deploy.yml
+++ b/.github/workflows/build_test_deploy.yml
@@ -10,9 +10,9 @@ on:
   workflow_dispatch:
 
 env:
-  # USER_NAME: root
-  # USER_UID: 1000
-  # USER_GID: 1000
+  USER_NAME: root
+  USER_UID: 1000
+  USER_GID: 1000
   DOCKER_USER: ${{ secrets.DOCKER_USER }}
   DOCKER_PASS: ${{ secrets.DOCKER_PASS }}
 
@@ -42,8 +42,6 @@ jobs:
       run: yarn check-cyclic-dependencies
 
 
-
-
   build_test_deploy:
     runs-on: ubuntu-20.04
     steps:
diff --git a/Dockerfile b/Dockerfile
index 2a553862cc..d3e12dc63c 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -8,6 +8,7 @@ RUN curl -sL "https://deb.nodesource.com/setup_${VERSION_NODE}" | bash - \
 RUN mkdir -p /webknossos
 WORKDIR /webknossos
 
+# Copy compiled Scala output from a previous build step, e.g. output of the Docker-dev image
 COPY target/universal/stage .
 
 RUN addgroup --system --gid 999 webknossos \

From 7acf3a953f5474cb0ffa54045c897185990d12c6 Mon Sep 17 00:00:00 2001
From: Tom Herold <tom.herold@scalableminds.com>
Date: Mon, 28 Oct 2024 13:53:49 +0100
Subject: [PATCH 09/39] CI

---
 .github/workflows/build_test_deploy.yml | 92 +++++++++++++++----------
 1 file changed, 56 insertions(+), 36 deletions(-)

diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml
index b99fa61d6e..16ef68853f 100644
--- a/.github/workflows/build_test_deploy.yml
+++ b/.github/workflows/build_test_deploy.yml
@@ -27,7 +27,7 @@ jobs:
 
     - uses: actions/setup-node@v4
       with:
-          node-version: 18
+        node-version: 18
 
     - name: Install frontend dependencies
       run: corepack enable && yarn install --immutable
@@ -61,51 +61,68 @@ jobs:
             echo "NORMALIZED_BRANCH=master" >> $GITHUB_ENV
             echo "DOCKER_TAG=${{ github.ref_name }}" >> $GITHUB_ENV
           fi
-
-      - name: Set up Docker
-        uses: docker/setup-buildx-action@v2
-
-      - name: Build webknossos-dev docker image
-        run: |
-          docker pull scalableminds/webknossos-dev:$NORMALIZED_BRANCH || true
-          DEV_CACHE=$NORMALIZED_BRANCH docker compose build base
-
-      - name: Prepare dependency folders
-        run: mkdir -p project/target target ~/.ivy2 ~/.cache/coursier
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 18
 
       - name: Install frontend dependencies
-        run: docker compose run -e PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true base yarn install --immutable && yarn cache clean
-
-      - name: Assert unique evolution numbers
-        run: docker compose run base tools/postgres/dbtool.js assert-unique-evolution-numbers
-
-      - name: Assert schema.sql and evolutions are equal
-        run: |
-          docker compose up -d postgres
-          sleep 3
-          docker compose run compile tools/postgres/dbtool.js check-evolutions-schema
-
-      - name: Build frontend documentation
-        run: |
-          WK_VERSION=${{ github.event.release.tag_name || github.run_number || 'dev' }}
-          docker compose run base yarn run docs --project-version $WK_VERSION
-
+        run: corepack enable && yarn install --immutable
+      
       - name: Build webknossos (webpack)
-        run: docker compose run base yarn build
+        run: yarn build
+
+      - name: Set up Java / SBT
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin' # See 'Supported distributions' for available options
+          java-version: '21'
+          cache: 'sbt'
+      
+      - name: Install dependencies
+        run: apt-get update \
+          && apt-get install -y \
+          findutils \
+          libdraco-dev \
+          libblosc1
+
+                    # build-essential \
+          # cmake \
+      # - name: Set up Docker
+      #   uses: docker/setup-buildx-action@v2
+
+      # - name: Build webknossos-dev docker image
+      #   run: |
+      #     docker pull scalableminds/webknossos-dev:$NORMALIZED_BRANCH || true
+      #     DEV_CACHE=$NORMALIZED_BRANCH docker compose build base
+
+      # - name: Prepare dependency folders
+      #   run: mkdir -p project/target target ~/.ivy2 ~/.cache/coursier
+
+      # - name: Install frontend dependencies
+      #   run: docker compose run -e PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true base yarn install --immutable && yarn cache clean
+
+      # - name: Assert unique evolution numbers
+      #   run: docker compose run base tools/postgres/dbtool.js assert-unique-evolution-numbers
+
+      # - name: Assert schema.sql and evolutions are equal
+      #   run: |
+      #     docker compose up -d postgres
+      #     sleep 3
+      #     docker compose run compile tools/postgres/dbtool.js check-evolutions-schema
 
       - name: Build webknossos (sbt)
         run: |
           if [ "${{ github.ref }}" == "refs/heads/master" ]; then
-            docker compose run compile sbt -no-colors clean compile stage
+            sbt -no-colors clean compile stage
           else
-            docker compose run compile sbt -no-colors -DfailOnWarning compile stage
+            sbt -no-colors -DfailOnWarning compile stage
           fi
 
       - name: Build webknossos-datastore (sbt)
-        run: docker compose run base sbt -no-colors -DfailOnWarning "project webknossosDatastore" copyMessages compile stage
+        run: sbt -no-colors -DfailOnWarning "project webknossosDatastore" copyMessages compile stage
 
       - name: Build webknossos-tracingstore (sbt)
-        run: docker compose run base sbt -no-colors -DfailOnWarning "project webknossosTracingstore" copyMessages compile stage
+        run: sbt -no-colors -DfailOnWarning "project webknossosTracingstore" copyMessages compile stage
 
       - name: Checksum App Dirs
         run: find app webknossos-datastore/app webknossos-tracingstore/app -type f -exec md5sum {} \; | sort -k 2 | md5sum > app_checksum.txt
@@ -137,9 +154,6 @@ jobs:
           done
           (exit $s)
 
-      - name: Validate frontend types
-        run: .github/not-on-master.sh docker compose run base yarn typecheck
-
       - name: Start webknossos, datastore, and tracingstore
         run: |
           docker compose up -d webknossos
@@ -190,6 +204,12 @@ jobs:
           fi
           docker logout
 
+      - name: Build frontend documentation
+        run: |
+          WK_VERSION=${{ github.event.release.tag_name || github.run_number || 'dev' }}
+          yarn run docs --project-version $WK_VERSION
+
+
       - name: Report coverage
         run: .github/not-on-master.sh docker compose run base yarn coverage || true
 

From 633b9bf20caea7538c575ac43b5a7b105211b8c5 Mon Sep 17 00:00:00 2001
From: Tom Herold <tom.herold@scalableminds.com>
Date: Mon, 28 Oct 2024 14:03:54 +0100
Subject: [PATCH 10/39] CI

---
 .github/workflows/build_test_deploy.yml | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml
index 16ef68853f..af2e0a6cba 100644
--- a/.github/workflows/build_test_deploy.yml
+++ b/.github/workflows/build_test_deploy.yml
@@ -78,9 +78,8 @@ jobs:
           java-version: '21'
           cache: 'sbt'
       
-      - name: Install dependencies
-        run: apt-get update \
-          && apt-get install -y \
+      - name: Install OS dependencies / libs
+        run: apt-get update && apt-get install -y \
           findutils \
           libdraco-dev \
           libblosc1

From 267d6468d7dd7979932f8de4087330f419624e47 Mon Sep 17 00:00:00 2001
From: Tom Herold <tom.herold@scalableminds.com>
Date: Mon, 28 Oct 2024 14:06:16 +0100
Subject: [PATCH 11/39] sudo!

---
 .github/workflows/build_test_deploy.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml
index af2e0a6cba..90941d0e3a 100644
--- a/.github/workflows/build_test_deploy.yml
+++ b/.github/workflows/build_test_deploy.yml
@@ -79,7 +79,7 @@ jobs:
           cache: 'sbt'
       
       - name: Install OS dependencies / libs
-        run: apt-get update && apt-get install -y \
+        run: sudo apt-get update && apt-get install -y \
           findutils \
           libdraco-dev \
           libblosc1

From 188753d532d4acbae4d9edd8fcf527616472bb54 Mon Sep 17 00:00:00 2001
From: Tom Herold <tom.herold@scalableminds.com>
Date: Mon, 28 Oct 2024 14:08:03 +0100
Subject: [PATCH 12/39] sudo

---
 .github/workflows/build_test_deploy.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml
index 90941d0e3a..3f38d6c38e 100644
--- a/.github/workflows/build_test_deploy.yml
+++ b/.github/workflows/build_test_deploy.yml
@@ -79,7 +79,7 @@ jobs:
           cache: 'sbt'
       
       - name: Install OS dependencies / libs
-        run: sudo apt-get update && apt-get install -y \
+        run: sudo apt-get update && sudo apt-get install -y \
           findutils \
           libdraco-dev \
           libblosc1

From 47e66d6adf1a01b21201ea275bf4da2087ed3e69 Mon Sep 17 00:00:00 2001
From: Tom Herold <tom.herold@scalableminds.com>
Date: Mon, 28 Oct 2024 14:13:08 +0100
Subject: [PATCH 13/39] bump ubuntu version

---
 .github/workflows/build_test_deploy.yml | 19 ++++---------------
 1 file changed, 4 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml
index 3f38d6c38e..5c13f4c59a 100644
--- a/.github/workflows/build_test_deploy.yml
+++ b/.github/workflows/build_test_deploy.yml
@@ -18,7 +18,7 @@ env:
 
 jobs:
   static_frontend_code_checks:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     steps:
     - name: Checkout code
       uses: actions/checkout@v3
@@ -86,20 +86,10 @@ jobs:
 
                     # build-essential \
           # cmake \
-      # - name: Set up Docker
-      #   uses: docker/setup-buildx-action@v2
-
-      # - name: Build webknossos-dev docker image
-      #   run: |
-      #     docker pull scalableminds/webknossos-dev:$NORMALIZED_BRANCH || true
-      #     DEV_CACHE=$NORMALIZED_BRANCH docker compose build base
 
       # - name: Prepare dependency folders
       #   run: mkdir -p project/target target ~/.ivy2 ~/.cache/coursier
 
-      # - name: Install frontend dependencies
-      #   run: docker compose run -e PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true base yarn install --immutable && yarn cache clean
-
       # - name: Assert unique evolution numbers
       #   run: docker compose run base tools/postgres/dbtool.js assert-unique-evolution-numbers
 
@@ -126,6 +116,9 @@ jobs:
       - name: Checksum App Dirs
         run: find app webknossos-datastore/app webknossos-tracingstore/app -type f -exec md5sum {} \; | sort -k 2 | md5sum > app_checksum.txt
 
+      - name: Set up Docker
+        uses: docker/setup-buildx-action@v2
+
       - name: Build webknossos docker image
         run: |
           docker pull scalableminds/webknossos:$NORMALIZED_BRANCH || true
@@ -137,9 +130,6 @@ jobs:
       - name: Build webknossos-tracingstore docker image
         run: docker compose build --pull webknossos-tracingstore
 
-      - name: Run frontend tests
-        run: .github/not-on-master.sh docker compose run base yarn test-verbose
-
       - name: Lint backend code and check formatting
         run: .github/not-on-master.sh docker compose run backend-lint-format
 
@@ -164,7 +154,6 @@ jobs:
         with: 
           url: http://localhost:9000/api/health
           
-
       - name: Run webknossos-datastore smoke test
         uses: ./.github/actions/health_check_action
         with: 

From f69e1c9a5281f635e92aceadac37c1a85ef0d904 Mon Sep 17 00:00:00 2001
From: Tom Herold <tom.herold@scalableminds.com>
Date: Mon, 28 Oct 2024 14:13:54 +0100
Subject: [PATCH 14/39] CI

---
 .github/workflows/build_test_deploy.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml
index 5c13f4c59a..e7d13d4668 100644
--- a/.github/workflows/build_test_deploy.yml
+++ b/.github/workflows/build_test_deploy.yml
@@ -43,7 +43,7 @@ jobs:
 
 
   build_test_deploy:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout code
         uses: actions/checkout@v3

From 98337a47fa1ea6925dcf6e179340bea2f8d2abcf Mon Sep 17 00:00:00 2001
From: Tom Herold <tom.herold@scalableminds.com>
Date: Mon, 28 Oct 2024 14:17:48 +0100
Subject: [PATCH 15/39] use different package names for ubuntu

---
 .github/workflows/build_test_deploy.yml | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml
index e7d13d4668..e9ff84b86a 100644
--- a/.github/workflows/build_test_deploy.yml
+++ b/.github/workflows/build_test_deploy.yml
@@ -80,9 +80,8 @@ jobs:
       
       - name: Install OS dependencies / libs
         run: sudo apt-get update && sudo apt-get install -y \
-          findutils \
-          libdraco-dev \
-          libblosc1
+          draco \
+          c-blosc
 
                     # build-essential \
           # cmake \

From da87a2a0b5df6b7cb769d6a15fcb3cda27e3b6c4 Mon Sep 17 00:00:00 2001
From: Tom Herold <tom.herold@scalableminds.com>
Date: Mon, 28 Oct 2024 14:27:45 +0100
Subject: [PATCH 16/39] CI

---
 .github/workflows/build_test_deploy.yml | 23 +++++++++++++++++++++--
 1 file changed, 21 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml
index e9ff84b86a..d1a9acf4b7 100644
--- a/.github/workflows/build_test_deploy.yml
+++ b/.github/workflows/build_test_deploy.yml
@@ -44,6 +44,25 @@ jobs:
 
   build_test_deploy:
     runs-on: ubuntu-24.04
+    env:
+      # The hostname used to communicate with the PostgreSQL service container
+      POSTGRES_HOST: localhost
+      POSTGRES_PORT: 5432
+    services:
+        postgres:
+          image: postgres:10-alpine
+          env:
+            POSTGRES_DB: webknossos
+            POSTGRES_USER: webknossos_user
+            POSTGRES_PASSWORD: secret_password
+          # Set health checks to wait until postgres has started
+          options: >-
+            --health-cmd pg_isready
+            --health-interval 2s
+            --health-timeout 5s
+            --health-retries 30
+          ports:
+            - 5432:5432
     steps:
       - name: Checkout code
         uses: actions/checkout@v3
@@ -80,8 +99,8 @@ jobs:
       
       - name: Install OS dependencies / libs
         run: sudo apt-get update && sudo apt-get install -y \
-          draco \
-          c-blosc
+          libdraco-dev \
+          libblosc-dev
 
                     # build-essential \
           # cmake \

From a15f95c9cc0f09710432bcbc282a489b053c3cb4 Mon Sep 17 00:00:00 2001
From: Tom Herold <tom.herold@scalableminds.com>
Date: Mon, 28 Oct 2024 14:34:41 +0100
Subject: [PATCH 17/39] enable ssh debug

---
 .github/workflows/build_test_deploy.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml
index d1a9acf4b7..9071aa8645 100644
--- a/.github/workflows/build_test_deploy.yml
+++ b/.github/workflows/build_test_deploy.yml
@@ -64,6 +64,10 @@ jobs:
           ports:
             - 5432:5432
     steps:
+      # Enable tmate debugging of manually-triggered workflows if the input option was provided
+      - name: Setup tmate session
+        uses: mxschmitt/action-tmate@v3
+        if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }}
       - name: Checkout code
         uses: actions/checkout@v3
         with:

From 5a2766fc1390f62898150c8b784ae8ced6842966 Mon Sep 17 00:00:00 2001
From: Tom Herold <tom.herold@scalableminds.com>
Date: Mon, 28 Oct 2024 14:36:25 +0100
Subject: [PATCH 18/39] CI

---
 .github/workflows/build_test_deploy.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml
index 9071aa8645..8b59f73c99 100644
--- a/.github/workflows/build_test_deploy.yml
+++ b/.github/workflows/build_test_deploy.yml
@@ -8,6 +8,12 @@ on:
     branches:
       - '*'
   workflow_dispatch:
+    inputs:
+      debug_enabled:
+        type: boolean
+        description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
+        required: false
+        default: false
 
 env:
   USER_NAME: root

From 35e7381300e90a4ec789a84a10605a55e2d843fc Mon Sep 17 00:00:00 2001
From: Tom Herold <tom.herold@scalableminds.com>
Date: Mon, 28 Oct 2024 14:40:54 +0100
Subject: [PATCH 19/39] CI

---
 .github/workflows/build_test_deploy.yml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml
index 8b59f73c99..ebba6f6139 100644
--- a/.github/workflows/build_test_deploy.yml
+++ b/.github/workflows/build_test_deploy.yml
@@ -108,9 +108,7 @@ jobs:
           cache: 'sbt'
       
       - name: Install OS dependencies / libs
-        run: sudo apt-get update && sudo apt-get install -y \
-          libdraco-dev \
-          libblosc-dev
+        run: sudo apt-get update && sudo apt-get install -y libdraco-dev libblosc-dev
 
                     # build-essential \
           # cmake \

From e81a452fe2c4d21dc27c7f9253428c43af6bc76d Mon Sep 17 00:00:00 2001
From: Tom Herold <tom.herold@scalableminds.com>
Date: Mon, 28 Oct 2024 14:48:53 +0100
Subject: [PATCH 20/39] add sbt action

---
 .github/workflows/build_test_deploy.yml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml
index ebba6f6139..567124714e 100644
--- a/.github/workflows/build_test_deploy.yml
+++ b/.github/workflows/build_test_deploy.yml
@@ -100,13 +100,15 @@ jobs:
       - name: Build webknossos (webpack)
         run: yarn build
 
-      - name: Set up Java / SBT
+      - name: Set up Java
         uses: actions/setup-java@v4
         with:
           distribution: 'temurin' # See 'Supported distributions' for available options
           java-version: '21'
           cache: 'sbt'
-      
+      - name: Set up sbt
+        uses: sbt/setup-sbt@v1
+        
       - name: Install OS dependencies / libs
         run: sudo apt-get update && sudo apt-get install -y libdraco-dev libblosc-dev
 

From e3bd80464a00fde82ce1768f878e14df36f5c8bc Mon Sep 17 00:00:00 2001
From: Tom Herold <tom.herold@scalableminds.com>
Date: Mon, 28 Oct 2024 14:54:36 +0100
Subject: [PATCH 21/39] ci

---
 .github/workflows/build_test_deploy.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml
index 567124714e..b045589435 100644
--- a/.github/workflows/build_test_deploy.yml
+++ b/.github/workflows/build_test_deploy.yml
@@ -59,7 +59,7 @@ jobs:
           image: postgres:10-alpine
           env:
             POSTGRES_DB: webknossos
-            POSTGRES_USER: webknossos_user
+            POSTGRES_USER: postgres
             POSTGRES_PASSWORD: secret_password
           # Set health checks to wait until postgres has started
           options: >-
@@ -108,7 +108,7 @@ jobs:
           cache: 'sbt'
       - name: Set up sbt
         uses: sbt/setup-sbt@v1
-        
+
       - name: Install OS dependencies / libs
         run: sudo apt-get update && sudo apt-get install -y libdraco-dev libblosc-dev
 

From d12c11004f1a98bbffb15cb4749d417eed92fe60 Mon Sep 17 00:00:00 2001
From: Tom Herold <tom.herold@scalableminds.com>
Date: Mon, 28 Oct 2024 15:01:14 +0100
Subject: [PATCH 22/39] ci

---
 .github/workflows/build_test_deploy.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml
index b045589435..5fa31e45bc 100644
--- a/.github/workflows/build_test_deploy.yml
+++ b/.github/workflows/build_test_deploy.yml
@@ -52,8 +52,9 @@ jobs:
     runs-on: ubuntu-24.04
     env:
       # The hostname used to communicate with the PostgreSQL service container
-      POSTGRES_HOST: localhost
-      POSTGRES_PORT: 5432
+      # POSTGRES_HOST: localhost
+      # POSTGRES_PORT: 5432
+      PGPASSWORD: secret_password
     services:
         postgres:
           image: postgres:10-alpine

From b91701de2f3e3b84f71c7f543b16466b3308ca74 Mon Sep 17 00:00:00 2001
From: Tom Herold <tom.herold@scalableminds.com>
Date: Wed, 30 Oct 2024 15:05:08 +0100
Subject: [PATCH 23/39] CI

---
 .github/workflows/build_test_deploy.yml | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml
index 5fa31e45bc..9deb2d2eb7 100644
--- a/.github/workflows/build_test_deploy.yml
+++ b/.github/workflows/build_test_deploy.yml
@@ -59,8 +59,8 @@ jobs:
         postgres:
           image: postgres:10-alpine
           env:
-            POSTGRES_DB: webknossos
-            POSTGRES_USER: postgres
+            POSTGRES_URL: jdbc:postgresql://postgres/webknossos
+            POSTGRES_USER: webknossos_user
             POSTGRES_PASSWORD: secret_password
           # Set health checks to wait until postgres has started
           options: >-
@@ -152,6 +152,12 @@ jobs:
         run: |
           docker pull scalableminds/webknossos:$NORMALIZED_BRANCH || true
           DEV_CACHE=$NORMALIZED_BRANCH docker compose build --pull webknossos
+        env:
+          POSTGRES_URL: jdbc:postgresql://postgres/webknossos
+          POSTGRES_USER: webknossos_user
+          PGUSER: webknossos_user
+          POSTGRES_PASSWORD: secret_password
+          PGPASSWORD: secret_password
 
       - name: Build webknossos-datastore docker image
         run: docker compose build --pull webknossos-datastore

From c15cc6be6e749c95f1846c9119351bca5a44b81f Mon Sep 17 00:00:00 2001
From: Tom Herold <tom.herold@scalableminds.com>
Date: Wed, 30 Oct 2024 15:13:07 +0100
Subject: [PATCH 24/39] CI

---
 docker-compose.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index 131b190dbf..60978e4ad4 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -99,7 +99,7 @@ services:
       - USER_GID
       - COVERALLS_REPO_TOKEN
       - TZ=${TZ:-Europe/Berlin}
-      - POSTGRES_URL=jdbc:postgresql://postgres/webknossos
+      - POSTGRES_URL=jdbc:postgresql://postgres@localhost:5432/webknossos
       - POSTGRES_USER=webknossos_user
       - POSTGRES_PASSWORD=secret_password
       - HOME=/root

From e3c6b4d84a8151676a944cdd0a44f94ab6f9b1c4 Mon Sep 17 00:00:00 2001
From: Tom Herold <tom.herold@scalableminds.com>
Date: Wed, 30 Oct 2024 15:20:10 +0100
Subject: [PATCH 25/39] CI

---
 docker-compose.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index 60978e4ad4..96bb979361 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -99,7 +99,7 @@ services:
       - USER_GID
       - COVERALLS_REPO_TOKEN
       - TZ=${TZ:-Europe/Berlin}
-      - POSTGRES_URL=jdbc:postgresql://postgres@localhost:5432/webknossos
+      - POSTGRES_URL=jdbc:postgresql://webknossos_user@localhost:5432/webknossos
       - POSTGRES_USER=webknossos_user
       - POSTGRES_PASSWORD=secret_password
       - HOME=/root

From 543bba102e518c5218cde7f96cc5abdddebe6757 Mon Sep 17 00:00:00 2001
From: Tom Herold <tom.herold@scalableminds.com>
Date: Wed, 30 Oct 2024 15:21:23 +0100
Subject: [PATCH 26/39] CI

---
 .github/workflows/build_test_deploy.yml | 2 +-
 docker-compose.yml                      | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml
index 9deb2d2eb7..ccd54642fe 100644
--- a/.github/workflows/build_test_deploy.yml
+++ b/.github/workflows/build_test_deploy.yml
@@ -153,7 +153,7 @@ jobs:
           docker pull scalableminds/webknossos:$NORMALIZED_BRANCH || true
           DEV_CACHE=$NORMALIZED_BRANCH docker compose build --pull webknossos
         env:
-          POSTGRES_URL: jdbc:postgresql://postgres/webknossos
+          POSTGRES_URL: jdbc:postgresql://webknossos_user@localhost:5432/webknossos
           POSTGRES_USER: webknossos_user
           PGUSER: webknossos_user
           POSTGRES_PASSWORD: secret_password
diff --git a/docker-compose.yml b/docker-compose.yml
index 96bb979361..131b190dbf 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -99,7 +99,7 @@ services:
       - USER_GID
       - COVERALLS_REPO_TOKEN
       - TZ=${TZ:-Europe/Berlin}
-      - POSTGRES_URL=jdbc:postgresql://webknossos_user@localhost:5432/webknossos
+      - POSTGRES_URL=jdbc:postgresql://postgres/webknossos
       - POSTGRES_USER=webknossos_user
       - POSTGRES_PASSWORD=secret_password
       - HOME=/root

From f25cca044d8951cb5ec8d3cd8b6c152f33a7d5da Mon Sep 17 00:00:00 2001
From: Tom Herold <tom.herold@scalableminds.com>
Date: Mon, 11 Nov 2024 15:38:03 +0100
Subject: [PATCH 27/39] enable ssh debugging

---
 .github/workflows/build_test_deploy.yml | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml
index ccd54642fe..c7bcaa4ba6 100644
--- a/.github/workflows/build_test_deploy.yml
+++ b/.github/workflows/build_test_deploy.yml
@@ -72,9 +72,7 @@ jobs:
             - 5432:5432
     steps:
       # Enable tmate debugging of manually-triggered workflows if the input option was provided
-      - name: Setup tmate session
-        uses: mxschmitt/action-tmate@v3
-        if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }}
+      
       - name: Checkout code
         uses: actions/checkout@v3
         with:
@@ -127,6 +125,9 @@ jobs:
       #     docker compose up -d postgres
       #     sleep 3
       #     docker compose run compile tools/postgres/dbtool.js check-evolutions-schema
+      - name: Setup tmate session
+        uses: mxschmitt/action-tmate@v3
+        if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }}
 
       - name: Build webknossos (sbt)
         run: |

From e6de5cb1e79dbb7189c489e83775d70a645294d3 Mon Sep 17 00:00:00 2001
From: Tom Herold <tom.herold@scalableminds.com>
Date: Mon, 11 Nov 2024 16:00:38 +0100
Subject: [PATCH 28/39] set postgres url

---
 .github/workflows/build_test_deploy.yml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml
index c7bcaa4ba6..92824a8d97 100644
--- a/.github/workflows/build_test_deploy.yml
+++ b/.github/workflows/build_test_deploy.yml
@@ -118,7 +118,7 @@ jobs:
       #   run: mkdir -p project/target target ~/.ivy2 ~/.cache/coursier
 
       # - name: Assert unique evolution numbers
-      #   run: docker compose run base tools/postgres/dbtool.js assert-unique-evolution-numbers
+      #   run: node tools/postgres/dbtool.js assert-unique-evolution-numbers
 
       # - name: Assert schema.sql and evolutions are equal
       #   run: |
@@ -136,6 +136,9 @@ jobs:
           else
             sbt -no-colors -DfailOnWarning compile stage
           fi
+        env:
+          POSTGRES_URL: postgres://webknossos_user:secret_password@localhost:5432/webknossos
+
 
       - name: Build webknossos-datastore (sbt)
         run: sbt -no-colors -DfailOnWarning "project webknossosDatastore" copyMessages compile stage

From 816989dfa8911a86d6e0113979d6186373a5179e Mon Sep 17 00:00:00 2001
From: Tom Herold <tom.herold@scalableminds.com>
Date: Mon, 11 Nov 2024 16:05:28 +0100
Subject: [PATCH 29/39] CI

---
 .github/workflows/build_test_deploy.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml
index 92824a8d97..29128f4bf0 100644
--- a/.github/workflows/build_test_deploy.yml
+++ b/.github/workflows/build_test_deploy.yml
@@ -137,7 +137,7 @@ jobs:
             sbt -no-colors -DfailOnWarning compile stage
           fi
         env:
-          POSTGRES_URL: postgres://webknossos_user:secret_password@localhost:5432/webknossos
+          POSTGRES_URL: jdbc:postgres://webknossos_user:secret_password@localhost:5432/webknossos
 
 
       - name: Build webknossos-datastore (sbt)

From 495296ea3001ee475d9adc5a36f5c64edc76758b Mon Sep 17 00:00:00 2001
From: Tom Herold <tom.herold@scalableminds.com>
Date: Mon, 11 Nov 2024 16:22:53 +0100
Subject: [PATCH 30/39] CI

---
 .github/workflows/build_test_deploy.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml
index 29128f4bf0..d2109c7f52 100644
--- a/.github/workflows/build_test_deploy.yml
+++ b/.github/workflows/build_test_deploy.yml
@@ -137,8 +137,7 @@ jobs:
             sbt -no-colors -DfailOnWarning compile stage
           fi
         env:
-          POSTGRES_URL: jdbc:postgres://webknossos_user:secret_password@localhost:5432/webknossos
-
+          POSTGRES_URL: jdbc:postgresql://webknossos_user:secret_password@localhost:5432/webknossos
 
       - name: Build webknossos-datastore (sbt)
         run: sbt -no-colors -DfailOnWarning "project webknossosDatastore" copyMessages compile stage

From d9322ce5946f69c6abf3444195120b64ecbb1b29 Mon Sep 17 00:00:00 2001
From: Tom Herold <tom.herold@scalableminds.com>
Date: Mon, 11 Nov 2024 16:42:02 +0100
Subject: [PATCH 31/39] ci

---
 .github/workflows/build_test_deploy.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml
index d2109c7f52..639759e29f 100644
--- a/.github/workflows/build_test_deploy.yml
+++ b/.github/workflows/build_test_deploy.yml
@@ -138,6 +138,8 @@ jobs:
           fi
         env:
           POSTGRES_URL: jdbc:postgresql://webknossos_user:secret_password@localhost:5432/webknossos
+          POSTGRES_USER: webknossos_user
+          POSTGRES_PASSWORD: secret_password
 
       - name: Build webknossos-datastore (sbt)
         run: sbt -no-colors -DfailOnWarning "project webknossosDatastore" copyMessages compile stage

From 035f5a3a532e23592f75a773aa62c5c7eb97a699 Mon Sep 17 00:00:00 2001
From: Tom Herold <tom.herold@scalableminds.com>
Date: Mon, 11 Nov 2024 16:59:35 +0100
Subject: [PATCH 32/39] ci

---
 .github/workflows/build_test_deploy.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml
index 639759e29f..73523aa945 100644
--- a/.github/workflows/build_test_deploy.yml
+++ b/.github/workflows/build_test_deploy.yml
@@ -137,7 +137,7 @@ jobs:
             sbt -no-colors -DfailOnWarning compile stage
           fi
         env:
-          POSTGRES_URL: jdbc:postgresql://webknossos_user:secret_password@localhost:5432/webknossos
+          POSTGRES_URL: jdbc:postgresql://localhost:5432/webknossos
           POSTGRES_USER: webknossos_user
           POSTGRES_PASSWORD: secret_password
 

From b162be248cb1d5be7abbabf72cf2dc48d29a6e10 Mon Sep 17 00:00:00 2001
From: Tom Herold <tom.herold@scalableminds.com>
Date: Mon, 11 Nov 2024 17:18:29 +0100
Subject: [PATCH 33/39] ci

---
 .github/workflows/build_test_deploy.yml | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml
index 73523aa945..5176540efa 100644
--- a/.github/workflows/build_test_deploy.yml
+++ b/.github/workflows/build_test_deploy.yml
@@ -71,7 +71,6 @@ jobs:
           ports:
             - 5432:5432
     steps:
-      # Enable tmate debugging of manually-triggered workflows if the input option was provided
       
       - name: Checkout code
         uses: actions/checkout@v3
@@ -125,10 +124,8 @@ jobs:
       #     docker compose up -d postgres
       #     sleep 3
       #     docker compose run compile tools/postgres/dbtool.js check-evolutions-schema
-      - name: Setup tmate session
-        uses: mxschmitt/action-tmate@v3
-        if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }}
-
+      
+      
       - name: Build webknossos (sbt)
         run: |
           if [ "${{ github.ref }}" == "refs/heads/master" ]; then
@@ -158,11 +155,9 @@ jobs:
           docker pull scalableminds/webknossos:$NORMALIZED_BRANCH || true
           DEV_CACHE=$NORMALIZED_BRANCH docker compose build --pull webknossos
         env:
-          POSTGRES_URL: jdbc:postgresql://webknossos_user@localhost:5432/webknossos
+          POSTGRES_URL: jdbc:postgresql://localhost:5432/webknossos
           POSTGRES_USER: webknossos_user
-          PGUSER: webknossos_user
           POSTGRES_PASSWORD: secret_password
-          PGPASSWORD: secret_password
 
       - name: Build webknossos-datastore docker image
         run: docker compose build --pull webknossos-datastore
@@ -170,6 +165,11 @@ jobs:
       - name: Build webknossos-tracingstore docker image
         run: docker compose build --pull webknossos-tracingstore
 
+        # Enable tmate debugging of manually-triggered workflows if the input option was provided
+      - name: Setup tmate session
+        uses: mxschmitt/action-tmate@v3
+        if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }}
+
       - name: Lint backend code and check formatting
         run: .github/not-on-master.sh docker compose run backend-lint-format
 

From 6c06ca31ffd0a785c390b50128cf4ed62aad888b Mon Sep 17 00:00:00 2001
From: Tom Herold <tom.herold@scalableminds.com>
Date: Mon, 11 Nov 2024 17:27:27 +0100
Subject: [PATCH 34/39] CI

---
 .github/not-on-master.sh | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 mode change 100644 => 100755 .github/not-on-master.sh

diff --git a/.github/not-on-master.sh b/.github/not-on-master.sh
old mode 100644
new mode 100755

From 18b7cb42f482bd66c0e24d029d1e9d1a183c63f9 Mon Sep 17 00:00:00 2001
From: Tom Herold <tom.herold@scalableminds.com>
Date: Mon, 11 Nov 2024 17:29:23 +0100
Subject: [PATCH 35/39] ci

---
 .github/not-on-master.sh                | 2 +-
 .github/workflows/build_test_deploy.yml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/not-on-master.sh b/.github/not-on-master.sh
index fc9ccc8fc0..b2bd8536de 100755
--- a/.github/not-on-master.sh
+++ b/.github/not-on-master.sh
@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 set -Eeuo pipefail
 
-if [ "${GITHUB_REF}" == "master" ]; then
+if [ "${GITHUB_REF_NAME}" == "master" ]; then
   echo "Skipping this step on master..."
 else
   exec "$@"
diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml
index 5176540efa..b7be8e1923 100644
--- a/.github/workflows/build_test_deploy.yml
+++ b/.github/workflows/build_test_deploy.yml
@@ -128,7 +128,7 @@ jobs:
       
       - name: Build webknossos (sbt)
         run: |
-          if [ "${{ github.ref }}" == "refs/heads/master" ]; then
+          if [ "${{ github.ref_name }}" == "master" ]; then
             sbt -no-colors clean compile stage
           else
             sbt -no-colors -DfailOnWarning compile stage

From 77d54be8b55f49e59abcd3a7f7fa39cf66906bcf Mon Sep 17 00:00:00 2001
From: Tom Herold <tom.herold@scalableminds.com>
Date: Tue, 12 Nov 2024 11:58:44 +0100
Subject: [PATCH 36/39] ci

---
 .github/workflows/build_test_deploy.yml | 66 +++++++++++--------------
 1 file changed, 29 insertions(+), 37 deletions(-)

diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml
index b7be8e1923..1ae1770440 100644
--- a/.github/workflows/build_test_deploy.yml
+++ b/.github/workflows/build_test_deploy.yml
@@ -144,6 +144,9 @@ jobs:
       - name: Build webknossos-tracingstore (sbt)
         run: sbt -no-colors -DfailOnWarning "project webknossosTracingstore" copyMessages compile stage
 
+      - name: Lint backend code and check formatting
+        run: .github/not-on-master.sh sbt ";scapegoat; scalafmtCheck; util/scalafmtCheck; webknossosTracingstore/scalafmtCheck; webknossosDatastore/scalafmtCheck"
+      
       - name: Checksum App Dirs
         run: find app webknossos-datastore/app webknossos-tracingstore/app -type f -exec md5sum {} \; | sort -k 2 | md5sum > app_checksum.txt
 
@@ -170,18 +173,16 @@ jobs:
         uses: mxschmitt/action-tmate@v3
         if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }}
 
-      - name: Lint backend code and check formatting
-        run: .github/not-on-master.sh docker compose run backend-lint-format
 
       - name: Run backend tests
         run: .github/not-on-master.sh docker compose run backend-tests
 
       - name: Run end-to-end tests
-        run: |
-          for i in {1..3}; do # retry
-            .github/not-on-master.sh docker compose run e2e-tests && s=0 && break || s=$?
-          done
-          (exit $s)
+        uses: nick-invision/retry@v2
+        with:
+          timeout_minutes: 10
+          max_attempts: 3
+          command: .github/not-on-master.sh docker compose run e2e-tests
 
       - name: Start webknossos, datastore, and tracingstore
         run: |
@@ -208,38 +209,29 @@ jobs:
         run: docker compose down --volumes --remove-orphans
 
       - name: Push docker images
-        run: |
-          function retry() {
-            for i in {1..5}; do
-              "$@" && s=0 && break || s=$?
-              sleep 10
-            done
-            return $s
-          }
-          retry docker login -u $DOCKER_USER -p $DOCKER_PASS
-          retry docker compose push webknossos
-          retry docker compose push webknossos-datastore
-          retry docker compose push webknossos-tracingstore
-          if [[ ${{ github.ref_type }} == "branch" ]]; then
-            docker tag scalableminds/webknossos:${DOCKER_TAG} scalableminds/webknossos:${NORMALIZED_BRANCH}
-            retry docker push scalableminds/webknossos:${NORMALIZED_BRANCH}
-            docker tag scalableminds/webknossos-datastore:${DOCKER_TAG} scalableminds/webknossos-datastore:${NORMALIZED_BRANCH}
-            retry docker push scalableminds/webknossos-datastore:${NORMALIZED_BRANCH}
-            docker tag scalableminds/webknossos-tracingstore:${DOCKER_TAG} scalableminds/webknossos-tracingstore:${NORMALIZED_BRANCH}
-            retry docker push scalableminds/webknossos-tracingstore:${NORMALIZED_BRANCH}
-            docker tag scalableminds/webknossos-dev scalableminds/webknossos-dev:${NORMALIZED_BRANCH}
-            retry docker push scalableminds/webknossos-dev:${NORMALIZED_BRANCH}
-          fi
-          docker logout
-
-      - name: Build frontend documentation
-        run: |
-          WK_VERSION=${{ github.event.release.tag_name || github.run_number || 'dev' }}
-          yarn run docs --project-version $WK_VERSION
-
+        uses: nick-invision/retry@v2
+        with:
+          timeout_minutes: 10
+          max_attempts: 3
+          command: |
+            docker compose push webknossos
+            docker compose push webknossos-datastore
+            docker compose push webknossos-tracingstore
+            
+            if [[ ${{ github.ref_type }} == "branch" ]]; then
+              docker tag scalableminds/webknossos:${DOCKER_TAG} scalableminds/webknossos:${NORMALIZED_BRANCH}
+              retry docker push scalableminds/webknossos:${NORMALIZED_BRANCH}
+              docker tag scalableminds/webknossos-datastore:${DOCKER_TAG} scalableminds/webknossos-datastore:${NORMALIZED_BRANCH}
+              retry docker push scalableminds/webknossos-datastore:${NORMALIZED_BRANCH}
+              docker tag scalableminds/webknossos-tracingstore:${DOCKER_TAG} scalableminds/webknossos-tracingstore:${NORMALIZED_BRANCH}
+              retry docker push scalableminds/webknossos-tracingstore:${NORMALIZED_BRANCH}
+              docker tag scalableminds/webknossos-dev scalableminds/webknossos-dev:${NORMALIZED_BRANCH}
+              retry docker push scalableminds/webknossos-dev:${NORMALIZED_BRANCH}
+            fi
+            docker logout
 
       - name: Report coverage
-        run: .github/not-on-master.sh docker compose run base yarn coverage || true
+        run: .github/not-on-master.sh yarn coverage || true
 
       # - name: Send Slack notification (master only)
       #   run: .circleci/slack-notification.sh

From 46f66962503c0efeac609335bf3d6d25cc7a0caf Mon Sep 17 00:00:00 2001
From: Tom Herold <tom.herold@scalableminds.com>
Date: Tue, 12 Nov 2024 13:35:55 +0100
Subject: [PATCH 37/39] ci

---
 .github/workflows/build_test_deploy.yml |  23 +++--
 docker-compose.yml                      | 126 ++++++++++++------------
 2 files changed, 74 insertions(+), 75 deletions(-)

diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml
index 1ae1770440..b779699eb8 100644
--- a/.github/workflows/build_test_deploy.yml
+++ b/.github/workflows/build_test_deploy.yml
@@ -147,6 +147,9 @@ jobs:
       - name: Lint backend code and check formatting
         run: .github/not-on-master.sh sbt ";scapegoat; scalafmtCheck; util/scalafmtCheck; webknossosTracingstore/scalafmtCheck; webknossosDatastore/scalafmtCheck"
       
+      - name: Run backend tests
+        run: .github/not-on-master.sh sbt -v "testOnly backend.*"
+
       - name: Checksum App Dirs
         run: find app webknossos-datastore/app webknossos-tracingstore/app -type f -exec md5sum {} \; | sort -k 2 | md5sum > app_checksum.txt
 
@@ -172,18 +175,7 @@ jobs:
       - name: Setup tmate session
         uses: mxschmitt/action-tmate@v3
         if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }}
-
-
-      - name: Run backend tests
-        run: .github/not-on-master.sh docker compose run backend-tests
-
-      - name: Run end-to-end tests
-        uses: nick-invision/retry@v2
-        with:
-          timeout_minutes: 10
-          max_attempts: 3
-          command: .github/not-on-master.sh docker compose run e2e-tests
-
+      
       - name: Start webknossos, datastore, and tracingstore
         run: |
           docker compose up -d webknossos
@@ -204,6 +196,13 @@ jobs:
         uses: ./.github/actions/health_check_action
         with: 
           url: http://localhost:9050/tracings/health
+      
+      - name: Run end-to-end tests
+        uses: nick-invision/retry@v2
+        with:
+          timeout_minutes: 10
+          max_attempts: 3
+          command: .github/not-on-master.sh yarn run test-e2e
 
       - name: Stop webknossos, datastore, and tracingstore
         run: docker compose down --volumes --remove-orphans
diff --git a/docker-compose.yml b/docker-compose.yml
index 131b190dbf..00d39873b0 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -116,62 +116,62 @@ services:
       - "${DOCKER_CACHE_PREFIX:-~}/.cache/yarn:/root/.cache/yarn"
     user: ${USER_UID:-1000}:${USER_GID:-1000}
 
-  compile:
-    extends:
-      service: base
-    depends_on:
-      postgres:
-        condition: service_healthy
+  # compile:
+  #   extends:
+  #     service: base
+  #   depends_on:
+  #     postgres:
+  #       condition: service_healthy
 
-  dev:
-    extends:
-      service: base
-    ports:
-      - "5005:5005"
-      - "9000:9000"
-    links:
-      - "fossildb-dev:fossildb"
-      - "postgres-dev:postgres"
-    depends_on:
-      redis:
-        condition: service_healthy
-      postgres-dev:
-        condition: service_healthy
-      fossildb-dev:
-        condition: service_healthy
-    environment:
-      POSTGRES_URL: jdbc:postgresql://postgres/webknossos
-      POSTGRES_USER: webknossos_user
-      POSTGRES_PASSWORD: secret_password
-    command:
-      - bash
-      - -c
-      - >
-        sbt
-        -v -d -jvm-debug 5005
-        "run
-          -Djava.net.preferIPv4Stack=true
-          -Dtracingstore.fossildb.address=fossildb
-          -Dtracingstore.redis.address=redis"
-          -Ddatastore.redis.address=redis"
-    stdin_open: true
+  # dev:
+  #   extends:
+  #     service: base
+  #   ports:
+  #     - "5005:5005"
+  #     - "9000:9000"
+  #   links:
+  #     - "fossildb-dev:fossildb"
+  #     - "postgres-dev:postgres"
+  #   depends_on:
+  #     redis:
+  #       condition: service_healthy
+  #     postgres-dev:
+  #       condition: service_healthy
+  #     fossildb-dev:
+  #       condition: service_healthy
+  #   environment:
+  #     POSTGRES_URL: jdbc:postgresql://postgres/webknossos
+  #     POSTGRES_USER: webknossos_user
+  #     POSTGRES_PASSWORD: secret_password
+  #   command:
+  #     - bash
+  #     - -c
+  #     - >
+  #       sbt
+  #       -v -d -jvm-debug 5005
+  #       "run
+  #         -Djava.net.preferIPv4Stack=true
+  #         -Dtracingstore.fossildb.address=fossildb
+  #         -Dtracingstore.redis.address=redis"
+  #         -Ddatastore.redis.address=redis"
+  #   stdin_open: true
 
-  # Tests
-  backend-tests:
-    extends:
-      service: base
-    ports:
-      - "5005:5005"
-      - "9000:9000"
-    command: sbt -v "testOnly backend.*"
+  # # Tests
+  # backend-tests:
+  #   extends:
+  #     service: base
+  #   ports:
+  #     - "5005:5005"
+  #     - "9000:9000"
+  #   command: sbt -v "testOnly backend.*"
 
-  backend-lint-format:
-    extends:
-      service: base
-    ports:
-      - "5005:5005"
-      - "9000:9000"
-    command: sbt ";scapegoat; scalafmtCheck; util/scalafmtCheck; webknossosTracingstore/scalafmtCheck; webknossosDatastore/scalafmtCheck"
+  # backend-lint-format:
+  #   extends:
+  #     service: base
+  #   ports:
+  #     - "5005:5005"
+  #     - "9000:9000"
+  #   command: sbt ";scapegoat; scalafmtCheck; util/scalafmtCheck; webknossosTracingstore/scalafmtCheck; webknossosDatastore/scalafmtCheck"
 
   e2e-tests:
     extends:
@@ -201,16 +201,16 @@ services:
     volumes:
       - ./binaryData/Connectomics department:/home/${USER_NAME:-sbt-user}/webknossos/binaryData/Organization_X
 
-  screenshot-tests:
-    image: scalableminds/puppeteer:master
-    environment:
-      - URL
-      - WK_AUTH_TOKEN
-    working_dir: /home/pptruser/webknossos
-    command: bash -c 'for i in {1..3}; do yarn test-screenshot && break; done'
-    volumes:
-      - ".:/home/pptruser/webknossos"
-    user: ${USER_UID:-1000}:${USER_GID:-1000}
+  # screenshot-tests:
+  #   image: scalableminds/puppeteer:master
+  #   environment:
+  #     - URL
+  #     - WK_AUTH_TOKEN
+  #   working_dir: /home/pptruser/webknossos
+  #   command: bash -c 'for i in {1..3}; do yarn test-screenshot && break; done'
+  #   volumes:
+  #     - ".:/home/pptruser/webknossos"
+  #   user: ${USER_UID:-1000}:${USER_GID:-1000}
 
   # Postgres
   postgres:

From e864dbb2d72cab2b6c03fd0e6c0ff8a3af6c2ba2 Mon Sep 17 00:00:00 2001
From: Tom Herold <tom.herold@scalableminds.com>
Date: Tue, 12 Nov 2024 13:44:42 +0100
Subject: [PATCH 38/39] ci

---
 docker-compose.yml | 54 +++++++++++++++++++++++-----------------------
 1 file changed, 27 insertions(+), 27 deletions(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index 00d39873b0..2c8f336700 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -173,33 +173,33 @@ services:
   #     - "9000:9000"
   #   command: sbt ";scapegoat; scalafmtCheck; util/scalafmtCheck; webknossosTracingstore/scalafmtCheck; webknossosDatastore/scalafmtCheck"
 
-  e2e-tests:
-    extends:
-      service: backend-tests
-    depends_on:
-      redis:
-        condition: service_healthy
-      postgres:
-        condition: service_healthy
-      fossildb:
-        condition: service_healthy
-    environment:
-      POSTGRES_URL: jdbc:postgresql://postgres/webknossos_testing
-      POSTGRES_USER: webknossos_user
-      POSTGRES_PASSWORD: secret_password
-    command:
-      - bash
-      - -c
-      - >
-        sbt
-        -v
-        "testOnly e2e.* --
-          -Dtracingstore.fossildb.address=fossildb
-          -Dtracingstore.redis.address=redis
-          -Ddatastore.redis.address=redis
-          -Ddatastore.watchFileSystem.enabled=false"
-    volumes:
-      - ./binaryData/Connectomics department:/home/${USER_NAME:-sbt-user}/webknossos/binaryData/Organization_X
+  # e2e-tests:
+  #   extends:
+  #     service: backend-tests
+  #   depends_on:
+  #     redis:
+  #       condition: service_healthy
+  #     postgres:
+  #       condition: service_healthy
+  #     fossildb:
+  #       condition: service_healthy
+  #   environment:
+  #     POSTGRES_URL: jdbc:postgresql://postgres/webknossos_testing
+  #     POSTGRES_USER: webknossos_user
+  #     POSTGRES_PASSWORD: secret_password
+  #   command:
+  #     - bash
+  #     - -c
+  #     - >
+  #       sbt
+  #       -v
+  #       "testOnly e2e.* --
+  #         -Dtracingstore.fossildb.address=fossildb
+  #         -Dtracingstore.redis.address=redis
+  #         -Ddatastore.redis.address=redis
+  #         -Ddatastore.watchFileSystem.enabled=false"
+  #   volumes:
+  #     - ./binaryData/Connectomics department:/home/${USER_NAME:-sbt-user}/webknossos/binaryData/Organization_X
 
   # screenshot-tests:
   #   image: scalableminds/puppeteer:master

From c9f8868a209c47e906e867934213dfbd35839068 Mon Sep 17 00:00:00 2001
From: Tom Herold <tom.herold@scalableminds.com>
Date: Tue, 12 Nov 2024 14:07:31 +0100
Subject: [PATCH 39/39] ci

---
 docker-compose.yml | 180 ++++++++++++++++++++++-----------------------
 1 file changed, 90 insertions(+), 90 deletions(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index 2c8f336700..131b190dbf 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -116,101 +116,101 @@ services:
       - "${DOCKER_CACHE_PREFIX:-~}/.cache/yarn:/root/.cache/yarn"
     user: ${USER_UID:-1000}:${USER_GID:-1000}
 
-  # compile:
-  #   extends:
-  #     service: base
-  #   depends_on:
-  #     postgres:
-  #       condition: service_healthy
+  compile:
+    extends:
+      service: base
+    depends_on:
+      postgres:
+        condition: service_healthy
 
-  # dev:
-  #   extends:
-  #     service: base
-  #   ports:
-  #     - "5005:5005"
-  #     - "9000:9000"
-  #   links:
-  #     - "fossildb-dev:fossildb"
-  #     - "postgres-dev:postgres"
-  #   depends_on:
-  #     redis:
-  #       condition: service_healthy
-  #     postgres-dev:
-  #       condition: service_healthy
-  #     fossildb-dev:
-  #       condition: service_healthy
-  #   environment:
-  #     POSTGRES_URL: jdbc:postgresql://postgres/webknossos
-  #     POSTGRES_USER: webknossos_user
-  #     POSTGRES_PASSWORD: secret_password
-  #   command:
-  #     - bash
-  #     - -c
-  #     - >
-  #       sbt
-  #       -v -d -jvm-debug 5005
-  #       "run
-  #         -Djava.net.preferIPv4Stack=true
-  #         -Dtracingstore.fossildb.address=fossildb
-  #         -Dtracingstore.redis.address=redis"
-  #         -Ddatastore.redis.address=redis"
-  #   stdin_open: true
+  dev:
+    extends:
+      service: base
+    ports:
+      - "5005:5005"
+      - "9000:9000"
+    links:
+      - "fossildb-dev:fossildb"
+      - "postgres-dev:postgres"
+    depends_on:
+      redis:
+        condition: service_healthy
+      postgres-dev:
+        condition: service_healthy
+      fossildb-dev:
+        condition: service_healthy
+    environment:
+      POSTGRES_URL: jdbc:postgresql://postgres/webknossos
+      POSTGRES_USER: webknossos_user
+      POSTGRES_PASSWORD: secret_password
+    command:
+      - bash
+      - -c
+      - >
+        sbt
+        -v -d -jvm-debug 5005
+        "run
+          -Djava.net.preferIPv4Stack=true
+          -Dtracingstore.fossildb.address=fossildb
+          -Dtracingstore.redis.address=redis"
+          -Ddatastore.redis.address=redis"
+    stdin_open: true
 
-  # # Tests
-  # backend-tests:
-  #   extends:
-  #     service: base
-  #   ports:
-  #     - "5005:5005"
-  #     - "9000:9000"
-  #   command: sbt -v "testOnly backend.*"
+  # Tests
+  backend-tests:
+    extends:
+      service: base
+    ports:
+      - "5005:5005"
+      - "9000:9000"
+    command: sbt -v "testOnly backend.*"
 
-  # backend-lint-format:
-  #   extends:
-  #     service: base
-  #   ports:
-  #     - "5005:5005"
-  #     - "9000:9000"
-  #   command: sbt ";scapegoat; scalafmtCheck; util/scalafmtCheck; webknossosTracingstore/scalafmtCheck; webknossosDatastore/scalafmtCheck"
+  backend-lint-format:
+    extends:
+      service: base
+    ports:
+      - "5005:5005"
+      - "9000:9000"
+    command: sbt ";scapegoat; scalafmtCheck; util/scalafmtCheck; webknossosTracingstore/scalafmtCheck; webknossosDatastore/scalafmtCheck"
 
-  # e2e-tests:
-  #   extends:
-  #     service: backend-tests
-  #   depends_on:
-  #     redis:
-  #       condition: service_healthy
-  #     postgres:
-  #       condition: service_healthy
-  #     fossildb:
-  #       condition: service_healthy
-  #   environment:
-  #     POSTGRES_URL: jdbc:postgresql://postgres/webknossos_testing
-  #     POSTGRES_USER: webknossos_user
-  #     POSTGRES_PASSWORD: secret_password
-  #   command:
-  #     - bash
-  #     - -c
-  #     - >
-  #       sbt
-  #       -v
-  #       "testOnly e2e.* --
-  #         -Dtracingstore.fossildb.address=fossildb
-  #         -Dtracingstore.redis.address=redis
-  #         -Ddatastore.redis.address=redis
-  #         -Ddatastore.watchFileSystem.enabled=false"
-  #   volumes:
-  #     - ./binaryData/Connectomics department:/home/${USER_NAME:-sbt-user}/webknossos/binaryData/Organization_X
+  e2e-tests:
+    extends:
+      service: backend-tests
+    depends_on:
+      redis:
+        condition: service_healthy
+      postgres:
+        condition: service_healthy
+      fossildb:
+        condition: service_healthy
+    environment:
+      POSTGRES_URL: jdbc:postgresql://postgres/webknossos_testing
+      POSTGRES_USER: webknossos_user
+      POSTGRES_PASSWORD: secret_password
+    command:
+      - bash
+      - -c
+      - >
+        sbt
+        -v
+        "testOnly e2e.* --
+          -Dtracingstore.fossildb.address=fossildb
+          -Dtracingstore.redis.address=redis
+          -Ddatastore.redis.address=redis
+          -Ddatastore.watchFileSystem.enabled=false"
+    volumes:
+      - ./binaryData/Connectomics department:/home/${USER_NAME:-sbt-user}/webknossos/binaryData/Organization_X
 
-  # screenshot-tests:
-  #   image: scalableminds/puppeteer:master
-  #   environment:
-  #     - URL
-  #     - WK_AUTH_TOKEN
-  #   working_dir: /home/pptruser/webknossos
-  #   command: bash -c 'for i in {1..3}; do yarn test-screenshot && break; done'
-  #   volumes:
-  #     - ".:/home/pptruser/webknossos"
-  #   user: ${USER_UID:-1000}:${USER_GID:-1000}
+  screenshot-tests:
+    image: scalableminds/puppeteer:master
+    environment:
+      - URL
+      - WK_AUTH_TOKEN
+    working_dir: /home/pptruser/webknossos
+    command: bash -c 'for i in {1..3}; do yarn test-screenshot && break; done'
+    volumes:
+      - ".:/home/pptruser/webknossos"
+    user: ${USER_UID:-1000}:${USER_GID:-1000}
 
   # Postgres
   postgres: