From b66292bf427cae9dc6a081b0a7cc3992c951b952 Mon Sep 17 00:00:00 2001
From: Razvan-Liviu Varzaru <razvan@mariadb.org>
Date: Wed, 27 Nov 2024 13:04:06 +0200
Subject: [PATCH 01/10] Push DEV_ tags for master container

For now, pushes in DEV will trigger the creation of a dev_ tag.
The workflow is not covering the replacement of the production tag because this should be a manual operation a few weeks/months after migration.

To be more precise, we should automate this flow after we bump buildbot version.
---
 .github/workflows/bbm_build_container.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/bbm_build_container.yml b/.github/workflows/bbm_build_container.yml
index 38dceb82..a6e4f4bb 100644
--- a/.github/workflows/bbm_build_container.yml
+++ b/.github/workflows/bbm_build_container.yml
@@ -81,7 +81,7 @@ jobs:
           for image in master master-web; do
             skopeo copy --all --src-tls-verify=0 \
             docker://localhost:5000/${{ env.REPO }}:${image} \
-            docker://ghcr.io/mariadb/buildbot:${image}
+            docker://ghcr.io/mariadb/buildbot:dev_${image}
           done
       - name: Login to quay.io
         if: ${{ env.DEPLOY_IMAGES == 'true' }}
@@ -99,5 +99,5 @@ jobs:
           for image in master master-web; do
             skopeo copy --all --src-tls-verify=0 \
             docker://localhost:5000/${{ env.REPO }}:${image} \
-            docker://quay.io/mariadb-foundation/${{ env.REPO }}:${image}
+            docker://quay.io/mariadb-foundation/${{ env.REPO }}:dev_${image}
           done

From eb8965584eb49d408b0f597259946d139e819cc6 Mon Sep 17 00:00:00 2001
From: Razvan-Liviu Varzaru <razvan@mariadb.org>
Date: Wed, 27 Nov 2024 13:57:57 +0200
Subject: [PATCH 02/10] Consider running config validation on both tags

When master container is under development we must ensure  that configuration changes are valid on both buildbot master versions, PROD/DEV.
Let's say one will bring a new feature to Production independently of a master upgrade.
---
 .github/workflows/bbm_deploy.yml | 13 ++++++++--
 validate_master_cfg.sh           | 43 +++++++++++++++++++++++++++++---
 2 files changed, 51 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/bbm_deploy.yml b/.github/workflows/bbm_deploy.yml
index 35cc6fb3..c1923219 100644
--- a/.github/workflows/bbm_deploy.yml
+++ b/.github/workflows/bbm_deploy.yml
@@ -41,8 +41,17 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v4
-      - name: Check master.cfg files
-        run: ./validate_master_cfg.sh
+
+      - name: Check master.cfg files for DEV container
+        run: ./validate_master_cfg.sh -e DEV
+
+      # When master container is under development we must ensure
+      # that configuration changes are valid on both buildbot master
+      # versions, PROD/DEV. Let's say one will bring a new feature
+      # to Production independently of a master upgrade.
+      - name: Check master.cfg files for PROD container
+        run: ./validate_master_cfg.sh -e PROD
+
       - name: Check get_ssh_cnx_num.py
         run: |
           cd master-libvirt
diff --git a/validate_master_cfg.sh b/validate_master_cfg.sh
index 0349698e..699ed77c 100755
--- a/validate_master_cfg.sh
+++ b/validate_master_cfg.sh
@@ -10,6 +10,43 @@ err() {
   exit 1
 }
 
+usage() {
+  echo "Usage: $0 -e <DEV|PROD>"
+  exit 1
+}
+
+ENVIRONMENT=""
+
+while getopts ":e:" opt; do
+  case ${opt} in
+    e )
+      ENVIRONMENT=$OPTARG
+      ;;
+    \? )
+      usage
+      ;;
+    : )
+      usage
+      ;;
+  esac
+done
+
+if [[ -z "$ENVIRONMENT" ]]; then
+  usage
+fi
+
+case $ENVIRONMENT in
+  DEV)
+    IMAGE="quay.io/mariadb-foundation/bb-master:dev_master"
+    ;;
+  PROD)
+    IMAGE="quay.io/mariadb-foundation/bb-master:master"
+    ;;
+  *)
+    err "Unknown environment: $ENVIRONMENT. Use DEV or PROD."
+    ;;
+esac
+
 mkdir -p master-credential-provider
 [[ -f master-private.cfg ]] ||
   ln -s master-private.cfg-sample master-private.cfg
@@ -33,7 +70,7 @@ python3 define_masters.py
 echo "Checking master.cfg"
 $RUNC run -i -v "$(pwd):/srv/buildbot/master" \
   -w /srv/buildbot/master \
-  quay.io/mariadb-foundation/bb-master:master \
+  $IMAGE \
   buildbot checkconfig master.cfg
 echo -e "done\n"
 # not checking libvirt config file (//TEMP we need to find a solution
@@ -49,7 +86,7 @@ for dir in autogen/* \
   echo "Checking $dir/master.cfg"
   $RUNC run -i -v "$(pwd):/srv/buildbot/master" \
     -w /srv/buildbot/master \
-    quay.io/mariadb-foundation/bb-master:master \
+    $IMAGE \
     bash -c "cd $dir && buildbot checkconfig master.cfg"
   echo -e "done\n"
-done
+done
\ No newline at end of file

From 03621ad3a238deb6c5ede1b1fa6e43ea812f2241 Mon Sep 17 00:00:00 2001
From: Razvan-Liviu Varzaru <razvan@mariadb.org>
Date: Wed, 27 Nov 2024 14:08:29 +0200
Subject: [PATCH 03/10] Deploy job to Production

- no automation for now, only copy the configuration files, prepare docker-compose and Autogen/ masters
- will work only on manual dispatch from the Main branch
- renamed secrets for better env classification
---
 .github/workflows/bbm_deploy.yml | 43 +++++++++++++++++++++++++-------
 1 file changed, 34 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/bbm_deploy.yml b/.github/workflows/bbm_deploy.yml
index c1923219..ad525435 100644
--- a/.github/workflows/bbm_deploy.yml
+++ b/.github/workflows/bbm_deploy.yml
@@ -57,7 +57,7 @@ jobs:
           cd master-libvirt
           python get_ssh_cnx_num.py
 
-  deploy:
+  development-deploy:
     runs-on: ubuntu-22.04
     needs: check
     if: >
@@ -69,22 +69,47 @@ jobs:
         run: |
           install -m 600 -D /dev/null ~/.ssh/id_ed25519
           install -m 600 -D /dev/null ~/.ssh/known_hosts
-          echo "${{ secrets.SSH_PRIVATE_KEY }}" >~/.ssh/id_ed25519
-          echo "${{ secrets.SSH_KNOWN_HOSTS }}" >~/.ssh/known_hosts
+          echo "${{ secrets.BBM_DEV_SSH_PRIVATE_KEY }}" >~/.ssh/id_ed25519
+          echo "${{ secrets.BBM_DEV_SSH_KNOWN_HOSTS }}" >~/.ssh/known_hosts
       - name: shutdown stack
         run: |
-          ssh -p ${{ secrets.SERVER_PORT }} ${{ secrets.USER }}@${{ secrets.SERVER_IP }} "if [[ -f /srv/dev/docker-compose/docker-compose.yaml ]]; then docker-compose -f /srv/dev/docker-compose/docker-compose.yaml down; fi"
+          ssh -p ${{ secrets.BBM_DEV_SERVER_PORT }} ${{ secrets.BBM_DEV_USER }}@${{ secrets.BBM_DEV_SERVER_IP }} "if [[ -f /srv/dev/docker-compose/docker-compose.yaml ]]; then docker-compose -f /srv/dev/docker-compose/docker-compose.yaml down; fi"
       - name: deploy
         run: |
           # temporary fix of jade templating
           sed -i 's#https://ci.mariadb.org#https://ci.dev.mariadb.org#g' master-web/templates/home.jade
-          rsync -a --progress --delete --exclude-from=rsync.exclude -e "ssh -p ${{ secrets.SERVER_PORT }}" ./ ${{ secrets.USER }}@${{ secrets.SERVER_IP }}:/srv/dev/
-          ssh -p ${{ secrets.SERVER_PORT }} ${{ secrets.USER }}@${{ secrets.SERVER_IP }} "cd /srv/dev/docker-compose/ && ./generate-config.py --env=dev"
-          ssh -p ${{ secrets.SERVER_PORT }} ${{ secrets.USER }}@${{ secrets.SERVER_IP }} "cd /srv/dev/ && ./define_masters.py"
-          ssh -p ${{ secrets.SERVER_PORT }} ${{ secrets.USER }}@${{ secrets.SERVER_IP }} "cd /srv/dev/docker-compose/ && ln -sf .. buildbot"
+          rsync -a --progress --delete --exclude-from=rsync.exclude -e "ssh -p ${{ secrets.BBM_DEV_SERVER_PORT }}" ./ ${{ secrets.BBM_DEV_USER }}@${{ secrets.BBM_DEV_SERVER_IP }}:/srv/dev/
+          ssh -p ${{ secrets.BBM_DEV_SERVER_PORT }} ${{ secrets.BBM_DEV_USER }}@${{ secrets.BBM_DEV_SERVER_IP }} "cd /srv/dev/docker-compose/ && ./generate-config.py --env=dev"
+          ssh -p ${{ secrets.BBM_DEV_SERVER_PORT }} ${{ secrets.BBM_DEV_USER }}@${{ secrets.BBM_DEV_SERVER_IP }} "cd /srv/dev/ && ./define_masters.py"
+          ssh -p ${{ secrets.BBM_DEV_SERVER_PORT }} ${{ secrets.BBM_DEV_USER }}@${{ secrets.BBM_DEV_SERVER_IP }} "cd /srv/dev/docker-compose/ && ln -sf .. buildbot"
       - name: start stack
         run: |
-          ssh -p ${{ secrets.SERVER_PORT }} ${{ secrets.USER }}@${{ secrets.SERVER_IP }} "cd /srv/dev/docker-compose && docker-compose pull && docker-compose --env-file .env.dev up -d"
+          ssh -p ${{ secrets.BBM_DEV_SERVER_PORT }} ${{ secrets.BBM_DEV_USER }}@${{ secrets.BBM_DEV_SERVER_IP }} "cd /srv/dev/docker-compose && docker-compose pull && docker-compose --env-file .env.dev up -d"
+      - name: clean
+        run: |
+          rm ~/.ssh/id_ed25519
+
+  production-deploy:
+    runs-on: ubuntu-22.04
+    needs: check
+    if: > 
+      github.event_name == 'workflow_dispatch' && 
+      github.ref == 'refs/heads/main' &&
+      github.repository == 'MariaDB/buildbot'
+    steps:
+      - uses: actions/checkout@v4
+      - name: prepare
+        run: |
+          install -m 600 -D /dev/null ~/.ssh/id_ed25519
+          install -m 600 -D /dev/null ~/.ssh/known_hosts
+          echo "${{ secrets.BBM_PROD_SSH_PRIVATE_KEY }}" >~/.ssh/id_ed25519
+          echo "${{ secrets.BBM_PROD_SSH_KNOWN_HOSTS }}" >~/.ssh/known_hosts
+      - name: deploy
+        run: |
+          rsync -a --progress --delete --exclude-from=rsync.exclude -e "ssh -p ${{ secrets.BBM_PROD_SERVER_PORT }}" ./ ${{ secrets.BBM_PROD_USER }}@${{ secrets.BBM_PROD_SERVER_IP }}:/srv/prod/
+          ssh -p ${{ secrets.BBM_PROD_SERVER_PORT }} ${{ secrets.BBM_PROD_USER }}@${{ secrets.BBM_PROD_SERVER_IP }} "cd /srv/prod/docker-compose/ && ./generate-config.py --env=prod"
+          ssh -p ${{ secrets.BBM_PROD_SERVER_PORT }} ${{ secrets.BBM_PROD_USER }}@${{ secrets.BBM_PROD_SERVER_IP }} "cd /srv/prod/ && ./define_masters.py"
+          ssh -p ${{ secrets.BBM_PROD_SERVER_PORT }} ${{ secrets.BBM_PROD_USER }}@${{ secrets.BBM_PROD_SERVER_IP }} "cd /srv/prod/docker-compose/ && ln -sf .. buildbot"
       - name: clean
         run: |
           rm ~/.ssh/id_ed25519

From d0a113998639b57ed740282c5f220bd0d0e0c0a9 Mon Sep 17 00:00:00 2001
From: Razvan-Liviu Varzaru <razvan@mariadb.org>
Date: Wed, 27 Nov 2024 14:25:09 +0200
Subject: [PATCH 04/10] Generate docker-compose config should use dev_ tags

---
 docker-compose/docker-compose.yaml | 28 ++++++++++++++--------------
 docker-compose/generate-config.py  | 11 +++++++----
 2 files changed, 21 insertions(+), 18 deletions(-)

diff --git a/docker-compose/docker-compose.yaml b/docker-compose/docker-compose.yaml
index 2ba99904..3bfe2489 100644
--- a/docker-compose/docker-compose.yaml
+++ b/docker-compose/docker-compose.yaml
@@ -60,7 +60,7 @@ services:
         tag: "bb-nginx"
 
   master-web:
-    image: quay.io/mariadb-foundation/bb-master:master-web
+    image: quay.io/mariadb-foundation/bb-master:dev_master-web
     restart: unless-stopped
     container_name: master-web
     environment:
@@ -96,7 +96,7 @@ services:
         condition: service_started
 
   master-nonlatent:
-    image: quay.io/mariadb-foundation/bb-master:master
+    image: quay.io/mariadb-foundation/bb-master:dev_master
     restart: unless-stopped
     container_name: master-nonlatent
     environment:
@@ -136,7 +136,7 @@ services:
         condition: service_started
 
   master-libvirt:
-    image: quay.io/mariadb-foundation/bb-master:master
+    image: quay.io/mariadb-foundation/bb-master:dev_master
     restart: unless-stopped
     container_name: master-libvirt
     environment:
@@ -175,7 +175,7 @@ services:
         condition: service_started
 
   autogen_aarch64-master-0:
-    image: quay.io/mariadb-foundation/bb-master:master
+    image: quay.io/mariadb-foundation/bb-master:dev_master
     restart: unless-stopped
     container_name: autogen_aarch64-master-0
     environment:
@@ -214,7 +214,7 @@ services:
         condition: service_started
 
   autogen_amd64-master-0:
-    image: quay.io/mariadb-foundation/bb-master:master
+    image: quay.io/mariadb-foundation/bb-master:dev_master
     restart: unless-stopped
     container_name: autogen_amd64-master-0
     environment:
@@ -253,7 +253,7 @@ services:
         condition: service_started
 
   autogen_amd64-master-1:
-    image: quay.io/mariadb-foundation/bb-master:master
+    image: quay.io/mariadb-foundation/bb-master:dev_master
     restart: unless-stopped
     container_name: autogen_amd64-master-1
     environment:
@@ -292,7 +292,7 @@ services:
         condition: service_started
 
   autogen_ppc64le-master-0:
-    image: quay.io/mariadb-foundation/bb-master:master
+    image: quay.io/mariadb-foundation/bb-master:dev_master
     restart: unless-stopped
     container_name: autogen_ppc64le-master-0
     environment:
@@ -331,7 +331,7 @@ services:
         condition: service_started
 
   autogen_s390x-master-0:
-    image: quay.io/mariadb-foundation/bb-master:master
+    image: quay.io/mariadb-foundation/bb-master:dev_master
     restart: unless-stopped
     container_name: autogen_s390x-master-0
     environment:
@@ -370,7 +370,7 @@ services:
         condition: service_started
 
   autogen_x86-master-0:
-    image: quay.io/mariadb-foundation/bb-master:master
+    image: quay.io/mariadb-foundation/bb-master:dev_master
     restart: unless-stopped
     container_name: autogen_x86-master-0
     environment:
@@ -409,7 +409,7 @@ services:
         condition: service_started
 
   master-docker-nonstandard:
-    image: quay.io/mariadb-foundation/bb-master:master
+    image: quay.io/mariadb-foundation/bb-master:dev_master
     restart: unless-stopped
     container_name: master-docker-nonstandard
     environment:
@@ -448,7 +448,7 @@ services:
         condition: service_started
 
   master-galera:
-    image: quay.io/mariadb-foundation/bb-master:master
+    image: quay.io/mariadb-foundation/bb-master:dev_master
     restart: unless-stopped
     container_name: master-galera
     environment:
@@ -487,7 +487,7 @@ services:
         condition: service_started
 
   master-protected-branches:
-    image: quay.io/mariadb-foundation/bb-master:master
+    image: quay.io/mariadb-foundation/bb-master:dev_master
     restart: unless-stopped
     container_name: master-protected-branches
     environment:
@@ -526,7 +526,7 @@ services:
         condition: service_started
 
   master-docker-nonstandard-2:
-    image: quay.io/mariadb-foundation/bb-master:master
+    image: quay.io/mariadb-foundation/bb-master:dev_master
     restart: unless-stopped
     container_name: master-docker-nonstandard-2
     environment:
@@ -565,7 +565,7 @@ services:
         condition: service_started
 
   master-bintars:
-    image: quay.io/mariadb-foundation/bb-master:master
+    image: quay.io/mariadb-foundation/bb-master:dev_master
     restart: unless-stopped
     container_name: master-bintars
     environment:
diff --git a/docker-compose/generate-config.py b/docker-compose/generate-config.py
index 0d42bf40..0d56d853 100755
--- a/docker-compose/generate-config.py
+++ b/docker-compose/generate-config.py
@@ -86,7 +86,7 @@
         tag: "bb-nginx"
 
   master-web:
-    image: quay.io/mariadb-foundation/bb-master:master-web
+    image: quay.io/mariadb-foundation/bb-master:{environment}master-web
     restart: unless-stopped
     container_name: master-web
     hostname: master-web
@@ -105,7 +105,7 @@
 
 DOCKER_COMPOSE_TEMPLATE = """
   {master_name}:
-    image: quay.io/mariadb-foundation/bb-master:master
+    image: quay.io/mariadb-foundation/bb-master:{environment}master
     restart: unless-stopped
     container_name: {master_name}
     hostname: {master_name}
@@ -192,8 +192,10 @@ def main(args):
         )
         file.write(
             start_template.format(
-                port=master_web_port, cr_host_wg_addr=env_vars["CR_HOST_WG_ADDR"]
-            )
+              port=master_web_port,
+              cr_host_wg_addr=env_vars["CR_HOST_WG_ADDR"],
+              environment="" if args.env == "prod" else "dev_",
+          )
         )
         port = starting_port
         for master_directory in MASTER_DIRECTORIES:
@@ -206,6 +208,7 @@ def main(args):
                 port=port,
                 mc_host=mc_host,
                 volumes=generate_volumes(master_volumes[master_name]),
+                environment="" if args.env == "prod" else "dev_",
             )
             port += 1
 

From 59b2d6987a9fb738eba6fc6e678a39d2a38484f1 Mon Sep 17 00:00:00 2001
From: Razvan-Liviu Varzaru <razvan@mariadb.org>
Date: Wed, 27 Nov 2024 14:46:32 +0200
Subject: [PATCH 05/10] pre-commit fixes

---
 .github/workflows/bbm_deploy.yml  | 4 ++--
 docker-compose/generate-config.py | 8 ++++----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/bbm_deploy.yml b/.github/workflows/bbm_deploy.yml
index ad525435..0ea06cb8 100644
--- a/.github/workflows/bbm_deploy.yml
+++ b/.github/workflows/bbm_deploy.yml
@@ -92,8 +92,8 @@ jobs:
   production-deploy:
     runs-on: ubuntu-22.04
     needs: check
-    if: > 
-      github.event_name == 'workflow_dispatch' && 
+    if: >
+      github.event_name == 'workflow_dispatch' &&
       github.ref == 'refs/heads/main' &&
       github.repository == 'MariaDB/buildbot'
     steps:
diff --git a/docker-compose/generate-config.py b/docker-compose/generate-config.py
index 0d56d853..865aba38 100755
--- a/docker-compose/generate-config.py
+++ b/docker-compose/generate-config.py
@@ -192,10 +192,10 @@ def main(args):
         )
         file.write(
             start_template.format(
-              port=master_web_port,
-              cr_host_wg_addr=env_vars["CR_HOST_WG_ADDR"],
-              environment="" if args.env == "prod" else "dev_",
-          )
+                port=master_web_port,
+                cr_host_wg_addr=env_vars["CR_HOST_WG_ADDR"],
+                environment="" if args.env == "prod" else "dev_",
+            )
         )
         port = starting_port
         for master_directory in MASTER_DIRECTORIES:

From 9e553ca5787f99c2f4e5ad581f05163edfd28409 Mon Sep 17 00:00:00 2001
From: Razvan-Liviu Varzaru <razvan@mariadb.org>
Date: Thu, 28 Nov 2024 12:04:18 +0200
Subject: [PATCH 06/10] De-duplicate bbm-deploy code

Use environment variables to distinguish between PROD and DEV environments.
Use format syntax to substitute the secret variable names for DEV and PROD.
The Set up environment variables step will mark the run as deployable if:

The repository is MariaDB/buildbot.
DEV-specific variables will be assigned if the branch is dev.
The deployment will go to PROD if the event is workflow_dispatch, with PROD-specific variables assigned when the branch is main.
The Start/Stop stack will only run for the Development environment.

The Home.jade replacement has been removed, as it should already be handled.
---
 .github/workflows/bbm_deploy.yml | 81 +++++++++++++++++---------------
 validate_master_cfg.sh           |  2 +-
 2 files changed, 43 insertions(+), 40 deletions(-)

diff --git a/.github/workflows/bbm_deploy.yml b/.github/workflows/bbm_deploy.yml
index 0ea06cb8..aa1d3064 100644
--- a/.github/workflows/bbm_deploy.yml
+++ b/.github/workflows/bbm_deploy.yml
@@ -57,59 +57,62 @@ jobs:
           cd master-libvirt
           python get_ssh_cnx_num.py
 
-  development-deploy:
+  deploy:
     runs-on: ubuntu-22.04
     needs: check
-    if: >
-      github.ref == 'refs/heads/dev' &&
-      github.repository == 'MariaDB/buildbot'
     steps:
+      - name: Set up env vars
+        run: |
+          echo "DEPLOY=false" >>$GITHUB_ENV
+
+          # INFO
+          echo "GitHub Branch is ${{ github.ref }}"
+          echo "GitHub Repository is ${{ github.repository }}"
+          echo "GitHub Event Name is ${{ github.event_name }}"
+
+          # DEV environment
+          if [[ ${{ github.repository }} == 'MariaDB/buildbot' ]] && [[ ${{ github.ref }} == 'refs/heads/dev' ]]; then
+            echo "DEPLOY=true" >>$GITHUB_ENV
+            echo "BB_ENV=DEV" >>$GITHUB_ENV
+            echo "DEPLOY_PATH=/srv/dev" >>$GITHUB_ENV
+            echo "ENV_FILE=.env.dev" >>$GITHUB_ENV
+          fi
+
+          # PROD environment
+          if [[ ${{ github.repository }} == 'MariaDB/buildbot' ]] && [[ ${{ github.ref }} == 'refs/heads/main' ]] && [[ ${{ github.event_name }} == 'workflow_dispatch' ]]; then
+            echo "DEPLOY=true" >>$GITHUB_ENV
+            echo "BB_ENV=PROD" >>$GITHUB_ENV
+            echo "DEPLOY_PATH=/srv/prod" >>$GITHUB_ENV
+            echo "ENV_FILE=.env" >>$GITHUB_ENV
+          fi
+
       - uses: actions/checkout@v4
+
       - name: prepare
         run: |
           install -m 600 -D /dev/null ~/.ssh/id_ed25519
           install -m 600 -D /dev/null ~/.ssh/known_hosts
-          echo "${{ secrets.BBM_DEV_SSH_PRIVATE_KEY }}" >~/.ssh/id_ed25519
-          echo "${{ secrets.BBM_DEV_SSH_KNOWN_HOSTS }}" >~/.ssh/known_hosts
+          echo '${{ secrets[format('BBM_{0}_SSH_PRIVATE_KEY', env.BB_ENV)] }}' >~/.ssh/id_ed25519
+          echo '${{ secrets[format('BBM_{0}_SSH_KNOWN_HOSTS', env.BB_ENV)] }}' >~/.ssh/known_hosts
+
       - name: shutdown stack
+        if: ${{ env.DEPLOY == 'true' && env.BB_ENV == 'DEV' }}
         run: |
-          ssh -p ${{ secrets.BBM_DEV_SERVER_PORT }} ${{ secrets.BBM_DEV_USER }}@${{ secrets.BBM_DEV_SERVER_IP }} "if [[ -f /srv/dev/docker-compose/docker-compose.yaml ]]; then docker-compose -f /srv/dev/docker-compose/docker-compose.yaml down; fi"
+          ssh -p ${{ secrets[format('BBM_{0}_SERVER_PORT', env.BB_ENV)] }} ${{ secrets[format('BBM_{0}_USER', env.BB_ENV)] }}@${{ secrets[format('BBM_{0}_SERVER_IP', env.BB_ENV)] }} "if [[ -f ${{ env.DEPLOY_PATH }}/docker-compose/docker-compose.yaml ]]; then docker-compose -f ${{ env.DEPLOY_PATH }}/docker-compose/docker-compose.yaml down; fi"
+
       - name: deploy
+        if: ${{ env.DEPLOY == 'true' }}
         run: |
-          # temporary fix of jade templating
-          sed -i 's#https://ci.mariadb.org#https://ci.dev.mariadb.org#g' master-web/templates/home.jade
-          rsync -a --progress --delete --exclude-from=rsync.exclude -e "ssh -p ${{ secrets.BBM_DEV_SERVER_PORT }}" ./ ${{ secrets.BBM_DEV_USER }}@${{ secrets.BBM_DEV_SERVER_IP }}:/srv/dev/
-          ssh -p ${{ secrets.BBM_DEV_SERVER_PORT }} ${{ secrets.BBM_DEV_USER }}@${{ secrets.BBM_DEV_SERVER_IP }} "cd /srv/dev/docker-compose/ && ./generate-config.py --env=dev"
-          ssh -p ${{ secrets.BBM_DEV_SERVER_PORT }} ${{ secrets.BBM_DEV_USER }}@${{ secrets.BBM_DEV_SERVER_IP }} "cd /srv/dev/ && ./define_masters.py"
-          ssh -p ${{ secrets.BBM_DEV_SERVER_PORT }} ${{ secrets.BBM_DEV_USER }}@${{ secrets.BBM_DEV_SERVER_IP }} "cd /srv/dev/docker-compose/ && ln -sf .. buildbot"
+          rsync -a --progress --delete --exclude-from=rsync.exclude -e "ssh -p ${{ secrets[format('BBM_{0}_SERVER_PORT', env.BB_ENV)] }}" ./ ${{ secrets[format('BBM_{0}_USER', env.BB_ENV)] }}@${{ secrets[format('BBM_{0}_SERVER_IP', env.BB_ENV)] }}:${{ env.DEPLOY_PATH }}/
+          ssh -p ${{ secrets[format('BBM_{0}_SERVER_PORT', env.BB_ENV)] }} ${{ secrets[format('BBM_{0}_USER', env.BB_ENV)] }}@${{ secrets[format('BBM_{0}_SERVER_IP', env.BB_ENV)] }} "cd ${{ env.DEPLOY_PATH }}/docker-compose/ && ./generate-config.py --env=${BB_ENV,,}"
+          ssh -p ${{ secrets[format('BBM_{0}_SERVER_PORT', env.BB_ENV)] }} ${{ secrets[format('BBM_{0}_USER', env.BB_ENV)] }}@${{ secrets[format('BBM_{0}_SERVER_IP', env.BB_ENV)] }} "cd ${{ env.DEPLOY_PATH }}/ && ./define_masters.py"
+          ssh -p ${{ secrets[format('BBM_{0}_SERVER_PORT', env.BB_ENV)] }} ${{ secrets[format('BBM_{0}_USER', env.BB_ENV)] }}@${{ secrets[format('BBM_{0}_SERVER_IP', env.BB_ENV)] }} "cd ${{ env.DEPLOY_PATH }}/docker-compose/ && ln -sf .. buildbot"
+
       - name: start stack
+        if: ${{ env.DEPLOY == 'true' && env.BB_ENV == 'DEV' }}
         run: |
-          ssh -p ${{ secrets.BBM_DEV_SERVER_PORT }} ${{ secrets.BBM_DEV_USER }}@${{ secrets.BBM_DEV_SERVER_IP }} "cd /srv/dev/docker-compose && docker-compose pull && docker-compose --env-file .env.dev up -d"
-      - name: clean
-        run: |
-          rm ~/.ssh/id_ed25519
+          ssh -p ${{ secrets[format('BBM_{0}_SERVER_PORT', env.BB_ENV)] }} ${{ secrets[format('BBM_{0}_USER', env.BB_ENV)] }}@${{ secrets[format('BBM_{0}_SERVER_IP', env.BB_ENV)] }} "cd ${{ env.DEPLOY_PATH }}/docker-compose && docker-compose pull && docker-compose --env-file ${{ env.ENV_FILE }} up -d"
 
-  production-deploy:
-    runs-on: ubuntu-22.04
-    needs: check
-    if: >
-      github.event_name == 'workflow_dispatch' &&
-      github.ref == 'refs/heads/main' &&
-      github.repository == 'MariaDB/buildbot'
-    steps:
-      - uses: actions/checkout@v4
-      - name: prepare
-        run: |
-          install -m 600 -D /dev/null ~/.ssh/id_ed25519
-          install -m 600 -D /dev/null ~/.ssh/known_hosts
-          echo "${{ secrets.BBM_PROD_SSH_PRIVATE_KEY }}" >~/.ssh/id_ed25519
-          echo "${{ secrets.BBM_PROD_SSH_KNOWN_HOSTS }}" >~/.ssh/known_hosts
-      - name: deploy
-        run: |
-          rsync -a --progress --delete --exclude-from=rsync.exclude -e "ssh -p ${{ secrets.BBM_PROD_SERVER_PORT }}" ./ ${{ secrets.BBM_PROD_USER }}@${{ secrets.BBM_PROD_SERVER_IP }}:/srv/prod/
-          ssh -p ${{ secrets.BBM_PROD_SERVER_PORT }} ${{ secrets.BBM_PROD_USER }}@${{ secrets.BBM_PROD_SERVER_IP }} "cd /srv/prod/docker-compose/ && ./generate-config.py --env=prod"
-          ssh -p ${{ secrets.BBM_PROD_SERVER_PORT }} ${{ secrets.BBM_PROD_USER }}@${{ secrets.BBM_PROD_SERVER_IP }} "cd /srv/prod/ && ./define_masters.py"
-          ssh -p ${{ secrets.BBM_PROD_SERVER_PORT }} ${{ secrets.BBM_PROD_USER }}@${{ secrets.BBM_PROD_SERVER_IP }} "cd /srv/prod/docker-compose/ && ln -sf .. buildbot"
       - name: clean
         run: |
-          rm ~/.ssh/id_ed25519
+          rm ~/.ssh/id_ed25519
\ No newline at end of file
diff --git a/validate_master_cfg.sh b/validate_master_cfg.sh
index 699ed77c..795f844e 100755
--- a/validate_master_cfg.sh
+++ b/validate_master_cfg.sh
@@ -12,7 +12,7 @@ err() {
 
 usage() {
   echo "Usage: $0 -e <DEV|PROD>"
-  exit 1
+  exit 0
 }
 
 ENVIRONMENT=""

From 1038d4e0ebd37eed5216c95da0a2b5e15d2d2a04 Mon Sep 17 00:00:00 2001
From: Razvan-Liviu Varzaru <razvan@mariadb.org>
Date: Thu, 28 Nov 2024 16:49:02 +0200
Subject: [PATCH 07/10] Refactor code to max 80 chars per line

---
 .github/workflows/bbm_deploy.yml | 56 ++++++++++++++++++++++++++------
 1 file changed, 46 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/bbm_deploy.yml b/.github/workflows/bbm_deploy.yml
index aa1d3064..54f46203 100644
--- a/.github/workflows/bbm_deploy.yml
+++ b/.github/workflows/bbm_deploy.yml
@@ -71,7 +71,8 @@ jobs:
           echo "GitHub Event Name is ${{ github.event_name }}"
 
           # DEV environment
-          if [[ ${{ github.repository }} == 'MariaDB/buildbot' ]] && [[ ${{ github.ref }} == 'refs/heads/dev' ]]; then
+          if [[ ${{ github.repository }} == 'MariaDB/buildbot' ]] && \
+             [[ ${{ github.ref }} == 'refs/heads/dev' ]]; then
             echo "DEPLOY=true" >>$GITHUB_ENV
             echo "BB_ENV=DEV" >>$GITHUB_ENV
             echo "DEPLOY_PATH=/srv/dev" >>$GITHUB_ENV
@@ -79,7 +80,9 @@ jobs:
           fi
 
           # PROD environment
-          if [[ ${{ github.repository }} == 'MariaDB/buildbot' ]] && [[ ${{ github.ref }} == 'refs/heads/main' ]] && [[ ${{ github.event_name }} == 'workflow_dispatch' ]]; then
+          if [[ ${{ github.repository }} == 'MariaDB/buildbot' ]] && \
+             [[ ${{ github.ref }} == 'refs/heads/main' ]] && \
+             [[ ${{ github.event_name }} == 'workflow_dispatch' ]]; then
             echo "DEPLOY=true" >>$GITHUB_ENV
             echo "BB_ENV=PROD" >>$GITHUB_ENV
             echo "DEPLOY_PATH=/srv/prod" >>$GITHUB_ENV
@@ -89,29 +92,62 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: prepare
+        env:
+          PKEY: ${{ secrets[format('BBM_{0}_SSH_PRIVATE_KEY', env.BB_ENV)] }}
+          THOST: ${{ secrets[format('BBM_{0}_SSH_KNOWN_HOSTS', env.BB_ENV)] }}
         run: |
           install -m 600 -D /dev/null ~/.ssh/id_ed25519
           install -m 600 -D /dev/null ~/.ssh/known_hosts
-          echo '${{ secrets[format('BBM_{0}_SSH_PRIVATE_KEY', env.BB_ENV)] }}' >~/.ssh/id_ed25519
-          echo '${{ secrets[format('BBM_{0}_SSH_KNOWN_HOSTS', env.BB_ENV)] }}' >~/.ssh/known_hosts
+          echo "$PKEY" >~/.ssh/id_ed25519
+          echo "$THOST" >~/.ssh/known_hosts
 
       - name: shutdown stack
+        env:
+          TUSER: ${{ secrets[format('BBM_{0}_USER', env.BB_ENV)] }}
+          TPORT: ${{ secrets[format('BBM_{0}_SERVER_PORT', env.BB_ENV)] }}
+          TIP: ${{ secrets[format('BBM_{0}_SERVER_IP', env.BB_ENV)] }}
+          TPATH: ${{ env.DEPLOY_PATH }}
         if: ${{ env.DEPLOY == 'true' && env.BB_ENV == 'DEV' }}
         run: |
-          ssh -p ${{ secrets[format('BBM_{0}_SERVER_PORT', env.BB_ENV)] }} ${{ secrets[format('BBM_{0}_USER', env.BB_ENV)] }}@${{ secrets[format('BBM_{0}_SERVER_IP', env.BB_ENV)] }} "if [[ -f ${{ env.DEPLOY_PATH }}/docker-compose/docker-compose.yaml ]]; then docker-compose -f ${{ env.DEPLOY_PATH }}/docker-compose/docker-compose.yaml down; fi"
+          ssh -p $TPORT $TUSER@$TIP "cd $TPATH/docker-compose && 
+            docker-compose down"
 
       - name: deploy
+        env:
+          TUSER: ${{ secrets[format('BBM_{0}_USER', env.BB_ENV)] }}
+          TPORT: ${{ secrets[format('BBM_{0}_SERVER_PORT', env.BB_ENV)] }}
+          TIP: ${{ secrets[format('BBM_{0}_SERVER_IP', env.BB_ENV)] }}
+          TPATH: ${{ env.DEPLOY_PATH }}
         if: ${{ env.DEPLOY == 'true' }}
         run: |
-          rsync -a --progress --delete --exclude-from=rsync.exclude -e "ssh -p ${{ secrets[format('BBM_{0}_SERVER_PORT', env.BB_ENV)] }}" ./ ${{ secrets[format('BBM_{0}_USER', env.BB_ENV)] }}@${{ secrets[format('BBM_{0}_SERVER_IP', env.BB_ENV)] }}:${{ env.DEPLOY_PATH }}/
-          ssh -p ${{ secrets[format('BBM_{0}_SERVER_PORT', env.BB_ENV)] }} ${{ secrets[format('BBM_{0}_USER', env.BB_ENV)] }}@${{ secrets[format('BBM_{0}_SERVER_IP', env.BB_ENV)] }} "cd ${{ env.DEPLOY_PATH }}/docker-compose/ && ./generate-config.py --env=${BB_ENV,,}"
-          ssh -p ${{ secrets[format('BBM_{0}_SERVER_PORT', env.BB_ENV)] }} ${{ secrets[format('BBM_{0}_USER', env.BB_ENV)] }}@${{ secrets[format('BBM_{0}_SERVER_IP', env.BB_ENV)] }} "cd ${{ env.DEPLOY_PATH }}/ && ./define_masters.py"
-          ssh -p ${{ secrets[format('BBM_{0}_SERVER_PORT', env.BB_ENV)] }} ${{ secrets[format('BBM_{0}_USER', env.BB_ENV)] }}@${{ secrets[format('BBM_{0}_SERVER_IP', env.BB_ENV)] }} "cd ${{ env.DEPLOY_PATH }}/docker-compose/ && ln -sf .. buildbot"
+          # Copy files to remote server
+          rsync -a \ 
+            --progress \ 
+            --delete \ 
+            --exclude-from=rsync.exclude \ 
+            -e "ssh -p $TPORT" ./ $TUSER@$TIP:$TPATH/
+
+          # Generate docker-compose
+          ssh -p $TPORT $USER@$TIP "cd $TPATH/docker-compose/ 
+            && ./generate-config.py --env=${BB_ENV,,}"
+
+          # Define auto-generated masters
+          ssh -p $TPORT $USER@$TIP \ "cd $TPATH/ && ./define_masters.py"
+
+          ssh -p $TPORT $USER@$TIP \ "cd $TPATH/docker-compose/ && 
+            ln -sf .. buildbot"
 
       - name: start stack
+        env:
+          TUSER: ${{ secrets[format('BBM_{0}_USER', env.BB_ENV)] }}
+          TPORT: ${{ secrets[format('BBM_{0}_SERVER_PORT', env.BB_ENV)] }}
+          TIP: ${{ secrets[format('BBM_{0}_SERVER_IP', env.BB_ENV)] }}
+          TPATH: ${{ env.DEPLOY_PATH }}
         if: ${{ env.DEPLOY == 'true' && env.BB_ENV == 'DEV' }}
         run: |
-          ssh -p ${{ secrets[format('BBM_{0}_SERVER_PORT', env.BB_ENV)] }} ${{ secrets[format('BBM_{0}_USER', env.BB_ENV)] }}@${{ secrets[format('BBM_{0}_SERVER_IP', env.BB_ENV)] }} "cd ${{ env.DEPLOY_PATH }}/docker-compose && docker-compose pull && docker-compose --env-file ${{ env.ENV_FILE }} up -d"
+          ssh -p $TPORT $TUSER@$TIP "cd $TPATH/docker-compose &&
+             docker-compose pull && 
+             docker-compose --env-file ${{ env.ENV_FILE }} up -d"
 
       - name: clean
         run: |

From 72f6297fd423c9cfd9b8178f0817830bd92488c6 Mon Sep 17 00:00:00 2001
From: Razvan-Liviu Varzaru <razvan@mariadb.org>
Date: Thu, 28 Nov 2024 16:58:37 +0200
Subject: [PATCH 08/10] pre-commit fixes

---
 .github/workflows/bbm_deploy.yml | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/bbm_deploy.yml b/.github/workflows/bbm_deploy.yml
index 54f46203..db89f110 100644
--- a/.github/workflows/bbm_deploy.yml
+++ b/.github/workflows/bbm_deploy.yml
@@ -109,7 +109,7 @@ jobs:
           TPATH: ${{ env.DEPLOY_PATH }}
         if: ${{ env.DEPLOY == 'true' && env.BB_ENV == 'DEV' }}
         run: |
-          ssh -p $TPORT $TUSER@$TIP "cd $TPATH/docker-compose && 
+          ssh -p $TPORT $TUSER@$TIP "cd $TPATH/docker-compose &&
             docker-compose down"
 
       - name: deploy
@@ -121,20 +121,20 @@ jobs:
         if: ${{ env.DEPLOY == 'true' }}
         run: |
           # Copy files to remote server
-          rsync -a \ 
-            --progress \ 
-            --delete \ 
-            --exclude-from=rsync.exclude \ 
+          rsync -a \
+            --progress \
+            --delete \
+            --exclude-from=rsync.exclude \
             -e "ssh -p $TPORT" ./ $TUSER@$TIP:$TPATH/
 
           # Generate docker-compose
-          ssh -p $TPORT $USER@$TIP "cd $TPATH/docker-compose/ 
+          ssh -p $TPORT $USER@$TIP "cd $TPATH/docker-compose/
             && ./generate-config.py --env=${BB_ENV,,}"
 
           # Define auto-generated masters
           ssh -p $TPORT $USER@$TIP \ "cd $TPATH/ && ./define_masters.py"
 
-          ssh -p $TPORT $USER@$TIP \ "cd $TPATH/docker-compose/ && 
+          ssh -p $TPORT $USER@$TIP \ "cd $TPATH/docker-compose/ &&
             ln -sf .. buildbot"
 
       - name: start stack
@@ -146,9 +146,9 @@ jobs:
         if: ${{ env.DEPLOY == 'true' && env.BB_ENV == 'DEV' }}
         run: |
           ssh -p $TPORT $TUSER@$TIP "cd $TPATH/docker-compose &&
-             docker-compose pull && 
+             docker-compose pull &&
              docker-compose --env-file ${{ env.ENV_FILE }} up -d"
 
       - name: clean
         run: |
-          rm ~/.ssh/id_ed25519
\ No newline at end of file
+          rm ~/.ssh/id_ed25519

From cd85e05aa63d2161cdfc237262389e38eaf9eefe Mon Sep 17 00:00:00 2001
From: Razvan-Liviu Varzaru <razvan@mariadb.org>
Date: Thu, 28 Nov 2024 17:01:16 +0200
Subject: [PATCH 09/10] fix ssh commands

---
 .github/workflows/bbm_deploy.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/bbm_deploy.yml b/.github/workflows/bbm_deploy.yml
index db89f110..f4f32ca8 100644
--- a/.github/workflows/bbm_deploy.yml
+++ b/.github/workflows/bbm_deploy.yml
@@ -132,9 +132,9 @@ jobs:
             && ./generate-config.py --env=${BB_ENV,,}"
 
           # Define auto-generated masters
-          ssh -p $TPORT $USER@$TIP \ "cd $TPATH/ && ./define_masters.py"
+          ssh -p $TPORT $USER@$TIP "cd $TPATH/ && ./define_masters.py"
 
-          ssh -p $TPORT $USER@$TIP \ "cd $TPATH/docker-compose/ &&
+          ssh -p $TPORT $USER@$TIP "cd $TPATH/docker-compose/ &&
             ln -sf .. buildbot"
 
       - name: start stack

From 4a5e565a06cb47a45822be3c6424050fe9a9892d Mon Sep 17 00:00:00 2001
From: Razvan-Liviu Varzaru <razvan@mariadb.org>
Date: Thu, 28 Nov 2024 17:02:21 +0200
Subject: [PATCH 10/10] format &&

---
 .github/workflows/bbm_deploy.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/bbm_deploy.yml b/.github/workflows/bbm_deploy.yml
index f4f32ca8..b82977ef 100644
--- a/.github/workflows/bbm_deploy.yml
+++ b/.github/workflows/bbm_deploy.yml
@@ -128,8 +128,8 @@ jobs:
             -e "ssh -p $TPORT" ./ $TUSER@$TIP:$TPATH/
 
           # Generate docker-compose
-          ssh -p $TPORT $USER@$TIP "cd $TPATH/docker-compose/
-            && ./generate-config.py --env=${BB_ENV,,}"
+          ssh -p $TPORT $USER@$TIP "cd $TPATH/docker-compose/ &&
+           ./generate-config.py --env=${BB_ENV,,}"
 
           # Define auto-generated masters
           ssh -p $TPORT $USER@$TIP "cd $TPATH/ && ./define_masters.py"