MDBF-832 & MDBF-828 - Prepare BBM deployment for Production

.github/workflows/bbm_build_container.yml

@@ -81,7 +81,7 @@ jobs:
           for image in master master-web; do
             skopeo copy --all --src-tls-verify=0 \
             docker://localhost:5000/${{ env.REPO }}:${image} \
-            docker://ghcr.io/mariadb/buildbot:${image}
+            docker://ghcr.io/mariadb/buildbot:dev_${image}
           done
       - name: Login to quay.io
         if: ${{ env.DEPLOY_IMAGES == 'true' }}
@@ -99,5 +99,5 @@ jobs:
           for image in master master-web; do
             skopeo copy --all --src-tls-verify=0 \
             docker://localhost:5000/${{ env.REPO }}:${image} \
-            docker://quay.io/mariadb-foundation/${{ env.REPO }}:${image}
+            docker://quay.io/mariadb-foundation/${{ env.REPO }}:dev_${image}
           done

.github/workflows/bbm_deploy.yml

@@ -41,8 +41,17 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v4
-      - name: Check master.cfg files
-        run: ./validate_master_cfg.sh
+
+      - name: Check master.cfg files for DEV container
+        run: ./validate_master_cfg.sh -e DEV
+
+      # When master container is under development we must ensure
+      # that configuration changes are valid on both buildbot master
+      # versions, PROD/DEV. Let's say one will bring a new feature
+      # to Production independently of a master upgrade.
+      - name: Check master.cfg files for PROD container
+        run: ./validate_master_cfg.sh -e PROD
+
       - name: Check get_ssh_cnx_num.py
         run: |
           cd master-libvirt
@@ -51,31 +60,95 @@ jobs:
   deploy:
     runs-on: ubuntu-22.04
     needs: check
-    if: >
-      github.ref == 'refs/heads/dev' &&
-      github.repository == 'MariaDB/buildbot'
     steps:
+      - name: Set up env vars
+        run: |
+          echo "DEPLOY=false" >>$GITHUB_ENV
+
+          # INFO
+          echo "GitHub Branch is ${{ github.ref }}"
+          echo "GitHub Repository is ${{ github.repository }}"
+          echo "GitHub Event Name is ${{ github.event_name }}"
+
+          # DEV environment
+          if [[ ${{ github.repository }} == 'MariaDB/buildbot' ]] && \
+             [[ ${{ github.ref }} == 'refs/heads/dev' ]]; then
+            echo "DEPLOY=true" >>$GITHUB_ENV
+            echo "BB_ENV=DEV" >>$GITHUB_ENV
+            echo "DEPLOY_PATH=/srv/dev" >>$GITHUB_ENV
+            echo "ENV_FILE=.env.dev" >>$GITHUB_ENV
+          fi
+
+          # PROD environment
+          if [[ ${{ github.repository }} == 'MariaDB/buildbot' ]] && \
+             [[ ${{ github.ref }} == 'refs/heads/main' ]] && \
+             [[ ${{ github.event_name }} == 'workflow_dispatch' ]]; then
+            echo "DEPLOY=true" >>$GITHUB_ENV
+            echo "BB_ENV=PROD" >>$GITHUB_ENV
+            echo "DEPLOY_PATH=/srv/prod" >>$GITHUB_ENV
+            echo "ENV_FILE=.env" >>$GITHUB_ENV
+          fi
+
       - uses: actions/checkout@v4
+
       - name: prepare
+        env:
+          PKEY: ${{ secrets[format('BBM_{0}_SSH_PRIVATE_KEY', env.BB_ENV)] }}
+          THOST: ${{ secrets[format('BBM_{0}_SSH_KNOWN_HOSTS', env.BB_ENV)] }}
         run: |
           install -m 600 -D /dev/null ~/.ssh/id_ed25519
           install -m 600 -D /dev/null ~/.ssh/known_hosts
-          echo "${{ secrets.SSH_PRIVATE_KEY }}" >~/.ssh/id_ed25519
-          echo "${{ secrets.SSH_KNOWN_HOSTS }}" >~/.ssh/known_hosts
+          echo "$PKEY" >~/.ssh/id_ed25519
+          echo "$THOST" >~/.ssh/known_hosts
+
       - name: shutdown stack
+        env:
+          TUSER: ${{ secrets[format('BBM_{0}_USER', env.BB_ENV)] }}
+          TPORT: ${{ secrets[format('BBM_{0}_SERVER_PORT', env.BB_ENV)] }}
+          TIP: ${{ secrets[format('BBM_{0}_SERVER_IP', env.BB_ENV)] }}
+          TPATH: ${{ env.DEPLOY_PATH }}
+        if: ${{ env.DEPLOY == 'true' && env.BB_ENV == 'DEV' }}
         run: |
-          ssh -p ${{ secrets.SERVER_PORT }} ${{ secrets.USER }}@${{ secrets.SERVER_IP }} "if [[ -f /srv/dev/docker-compose/docker-compose.yaml ]]; then docker-compose -f /srv/dev/docker-compose/docker-compose.yaml down; fi"
+          ssh -p $TPORT $TUSER@$TIP "cd $TPATH/docker-compose &&
+            docker-compose down"
+
       - name: deploy
+        env:
+          TUSER: ${{ secrets[format('BBM_{0}_USER', env.BB_ENV)] }}
+          TPORT: ${{ secrets[format('BBM_{0}_SERVER_PORT', env.BB_ENV)] }}
+          TIP: ${{ secrets[format('BBM_{0}_SERVER_IP', env.BB_ENV)] }}
+          TPATH: ${{ env.DEPLOY_PATH }}
+        if: ${{ env.DEPLOY == 'true' }}
         run: |
-          # temporary fix of jade templating
-          sed -i 's#https://ci.mariadb.org#https://ci.dev.mariadb.org#g' master-web/templates/home.jade
-          rsync -a --progress --delete --exclude-from=rsync.exclude -e "ssh -p ${{ secrets.SERVER_PORT }}" ./ ${{ secrets.USER }}@${{ secrets.SERVER_IP }}:/srv/dev/
-          ssh -p ${{ secrets.SERVER_PORT }} ${{ secrets.USER }}@${{ secrets.SERVER_IP }} "cd /srv/dev/docker-compose/ && ./generate-config.py --env=dev"
-          ssh -p ${{ secrets.SERVER_PORT }} ${{ secrets.USER }}@${{ secrets.SERVER_IP }} "cd /srv/dev/ && ./define_masters.py"
-          ssh -p ${{ secrets.SERVER_PORT }} ${{ secrets.USER }}@${{ secrets.SERVER_IP }} "cd /srv/dev/docker-compose/ && ln -sf .. buildbot"
+          # Copy files to remote server
+          rsync -a \
+            --progress \
+            --delete \
+            --exclude-from=rsync.exclude \
+            -e "ssh -p $TPORT" ./ $TUSER@$TIP:$TPATH/
+
+          # Generate docker-compose
+          ssh -p $TPORT $USER@$TIP "cd $TPATH/docker-compose/ &&
+           ./generate-config.py --env=${BB_ENV,,}"
+
+          # Define auto-generated masters
+          ssh -p $TPORT $USER@$TIP "cd $TPATH/ && ./define_masters.py"
+
+          ssh -p $TPORT $USER@$TIP "cd $TPATH/docker-compose/ &&
+            ln -sf .. buildbot"
+
       - name: start stack
+        env:
+          TUSER: ${{ secrets[format('BBM_{0}_USER', env.BB_ENV)] }}
+          TPORT: ${{ secrets[format('BBM_{0}_SERVER_PORT', env.BB_ENV)] }}
+          TIP: ${{ secrets[format('BBM_{0}_SERVER_IP', env.BB_ENV)] }}
+          TPATH: ${{ env.DEPLOY_PATH }}
+        if: ${{ env.DEPLOY == 'true' && env.BB_ENV == 'DEV' }}
         run: |
-          ssh -p ${{ secrets.SERVER_PORT }} ${{ secrets.USER }}@${{ secrets.SERVER_IP }} "cd /srv/dev/docker-compose && docker-compose pull && docker-compose --env-file .env.dev up -d"
+          ssh -p $TPORT $TUSER@$TIP "cd $TPATH/docker-compose &&
+             docker-compose pull &&
+             docker-compose --env-file ${{ env.ENV_FILE }} up -d"
+
       - name: clean
         run: |
           rm ~/.ssh/id_ed25519

docker-compose/docker-compose.yaml

@@ -60,7 +60,7 @@ services:
         tag: "bb-nginx"
 
   master-web:
-    image: quay.io/mariadb-foundation/bb-master:master-web
+    image: quay.io/mariadb-foundation/bb-master:dev_master-web
     restart: unless-stopped
     container_name: master-web
     environment:
@@ -96,7 +96,7 @@ services:
         condition: service_started
 
   master-nonlatent:
-    image: quay.io/mariadb-foundation/bb-master:master
+    image: quay.io/mariadb-foundation/bb-master:dev_master
     restart: unless-stopped
     container_name: master-nonlatent
     environment:
@@ -136,7 +136,7 @@ services:
         condition: service_started
 
   master-libvirt:
-    image: quay.io/mariadb-foundation/bb-master:master
+    image: quay.io/mariadb-foundation/bb-master:dev_master
     restart: unless-stopped
     container_name: master-libvirt
     environment:
@@ -175,7 +175,7 @@ services:
         condition: service_started
 
   autogen_aarch64-master-0:
-    image: quay.io/mariadb-foundation/bb-master:master
+    image: quay.io/mariadb-foundation/bb-master:dev_master
     restart: unless-stopped
     container_name: autogen_aarch64-master-0
     environment:
@@ -214,7 +214,7 @@ services:
         condition: service_started
 
   autogen_amd64-master-0:
-    image: quay.io/mariadb-foundation/bb-master:master
+    image: quay.io/mariadb-foundation/bb-master:dev_master
     restart: unless-stopped
     container_name: autogen_amd64-master-0
     environment:
@@ -253,7 +253,7 @@ services:
         condition: service_started
 
   autogen_amd64-master-1:
-    image: quay.io/mariadb-foundation/bb-master:master
+    image: quay.io/mariadb-foundation/bb-master:dev_master
     restart: unless-stopped
     container_name: autogen_amd64-master-1
     environment:
@@ -292,7 +292,7 @@ services:
         condition: service_started
 
   autogen_ppc64le-master-0:
-    image: quay.io/mariadb-foundation/bb-master:master
+    image: quay.io/mariadb-foundation/bb-master:dev_master
     restart: unless-stopped
     container_name: autogen_ppc64le-master-0
     environment:
@@ -331,7 +331,7 @@ services:
         condition: service_started
 
   autogen_s390x-master-0:
-    image: quay.io/mariadb-foundation/bb-master:master
+    image: quay.io/mariadb-foundation/bb-master:dev_master
     restart: unless-stopped
     container_name: autogen_s390x-master-0
     environment:
@@ -370,7 +370,7 @@ services:
         condition: service_started
 
   autogen_x86-master-0:
-    image: quay.io/mariadb-foundation/bb-master:master
+    image: quay.io/mariadb-foundation/bb-master:dev_master
     restart: unless-stopped
     container_name: autogen_x86-master-0
     environment:
@@ -409,7 +409,7 @@ services:
         condition: service_started
 
   master-docker-nonstandard:
-    image: quay.io/mariadb-foundation/bb-master:master
+    image: quay.io/mariadb-foundation/bb-master:dev_master
     restart: unless-stopped
     container_name: master-docker-nonstandard
     environment:
@@ -448,7 +448,7 @@ services:
         condition: service_started
 
   master-galera:
-    image: quay.io/mariadb-foundation/bb-master:master
+    image: quay.io/mariadb-foundation/bb-master:dev_master
     restart: unless-stopped
     container_name: master-galera
     environment:
@@ -487,7 +487,7 @@ services:
         condition: service_started
 
   master-protected-branches:
-    image: quay.io/mariadb-foundation/bb-master:master
+    image: quay.io/mariadb-foundation/bb-master:dev_master
     restart: unless-stopped
     container_name: master-protected-branches
     environment:
@@ -526,7 +526,7 @@ services:
         condition: service_started
 
   master-docker-nonstandard-2:
-    image: quay.io/mariadb-foundation/bb-master:master
+    image: quay.io/mariadb-foundation/bb-master:dev_master
     restart: unless-stopped
     container_name: master-docker-nonstandard-2
     environment:
@@ -565,7 +565,7 @@ services:
         condition: service_started
 
   master-bintars:
-    image: quay.io/mariadb-foundation/bb-master:master
+    image: quay.io/mariadb-foundation/bb-master:dev_master
     restart: unless-stopped
     container_name: master-bintars
     environment:

docker-compose/generate-config.py

@@ -86,7 +86,7 @@
         tag: "bb-nginx"
 
   master-web:
-    image: quay.io/mariadb-foundation/bb-master:master-web
+    image: quay.io/mariadb-foundation/bb-master:{environment}master-web
     restart: unless-stopped
     container_name: master-web
     hostname: master-web
@@ -105,7 +105,7 @@
 
 DOCKER_COMPOSE_TEMPLATE = """
   {master_name}:
-    image: quay.io/mariadb-foundation/bb-master:master
+    image: quay.io/mariadb-foundation/bb-master:{environment}master
     restart: unless-stopped
     container_name: {master_name}
     hostname: {master_name}
@@ -192,7 +192,9 @@ def main(args):
         )
         file.write(
             start_template.format(
-                port=master_web_port, cr_host_wg_addr=env_vars["CR_HOST_WG_ADDR"]
+                port=master_web_port,
+                cr_host_wg_addr=env_vars["CR_HOST_WG_ADDR"],
+                environment="" if args.env == "prod" else "dev_",
             )
         )
         port = starting_port
@@ -206,6 +208,7 @@ def main(args):
                 port=port,
                 mc_host=mc_host,
                 volumes=generate_volumes(master_volumes[master_name]),
+                environment="" if args.env == "prod" else "dev_",
             )
             port += 1
 

validate_master_cfg.sh

@@ -10,6 +10,43 @@ err() {
   exit 1
 }
 
+usage() {
+  echo "Usage: $0 -e <DEV|PROD>"
+  exit 0
+}
+
+ENVIRONMENT=""
+
+while getopts ":e:" opt; do
+  case ${opt} in
+    e )
+      ENVIRONMENT=$OPTARG
+      ;;
+    \? )
+      usage
+      ;;
+    : )
+      usage
+      ;;
+  esac
+done
+
+if [[ -z "$ENVIRONMENT" ]]; then
+  usage
+fi
+
+case $ENVIRONMENT in
+  DEV)
+    IMAGE="quay.io/mariadb-foundation/bb-master:dev_master"
+    ;;
+  PROD)
+    IMAGE="quay.io/mariadb-foundation/bb-master:master"
+    ;;
+  *)
+    err "Unknown environment: $ENVIRONMENT. Use DEV or PROD."
+    ;;
+esac
+
 mkdir -p master-credential-provider
 [[ -f master-private.cfg ]] ||
   ln -s master-private.cfg-sample master-private.cfg
@@ -33,7 +70,7 @@ python3 define_masters.py
 echo "Checking master.cfg"
 $RUNC run -i -v "$(pwd):/srv/buildbot/master" \
   -w /srv/buildbot/master \
-  quay.io/mariadb-foundation/bb-master:master \
+  $IMAGE \
   buildbot checkconfig master.cfg
 echo -e "done\n"
 # not checking libvirt config file (//TEMP we need to find a solution
@@ -49,7 +86,7 @@ for dir in autogen/* \
   echo "Checking $dir/master.cfg"
   $RUNC run -i -v "$(pwd):/srv/buildbot/master" \
     -w /srv/buildbot/master \
-    quay.io/mariadb-foundation/bb-master:master \
+    $IMAGE \
     bash -c "cd $dir && buildbot checkconfig master.cfg"
   echo -e "done\n"
-done
+done