MDBF-864 - Make master container handle SIGTERM

Need to replace the start.sh process with buildbot one when starting
the master containers so docker-compose down SIGTERM will be handled by twisted
for a graceful shutdown.

Also adding stop_grace_period to the master containers
to allow for more than 10 seconds timeout before sending a SIGKILL.

docker-compose/docker-compose.yaml

@@ -83,6 +83,7 @@ services:
       - PORT=8010
       - TITLE
       - TITLE_URL
+    stop_grace_period: 5m
     hostname: master-web
     volumes:
       - ./logs:/var/log/buildbot
@@ -119,6 +120,7 @@ services:
       - PORT=9996
       - TITLE
       - TITLE_URL
+    stop_grace_period: 5m
     hostname: master-nonlatent
     volumes:
       - ./logs:/var/log/buildbot
@@ -158,6 +160,7 @@ services:
       - PORT=9997
       - TITLE
       - TITLE_URL
+    stop_grace_period: 5m
     hostname: master-libvirt
     volumes:
       - ./logs:/var/log/buildbot
@@ -196,6 +199,7 @@ services:
       - PORT=9998
       - TITLE
       - TITLE_URL
+    stop_grace_period: 5m
     hostname: autogen_aarch64-master-0
     volumes:
       - ./logs:/var/log/buildbot
@@ -234,6 +238,7 @@ services:
       - PORT=9999
       - TITLE
       - TITLE_URL
+    stop_grace_period: 5m
     hostname: autogen_amd64-master-0
     volumes:
       - ./logs:/var/log/buildbot
@@ -272,6 +277,7 @@ services:
       - PORT=10000
       - TITLE
       - TITLE_URL
+    stop_grace_period: 5m
     hostname: autogen_amd64-master-1
     volumes:
       - ./logs:/var/log/buildbot
@@ -310,6 +316,7 @@ services:
       - PORT=10001
       - TITLE
       - TITLE_URL
+    stop_grace_period: 5m
     hostname: autogen_ppc64le-master-0
     volumes:
       - ./logs:/var/log/buildbot
@@ -348,6 +355,7 @@ services:
       - PORT=10002
       - TITLE
       - TITLE_URL
+    stop_grace_period: 5m
     hostname: autogen_s390x-master-0
     volumes:
       - ./logs:/var/log/buildbot
@@ -386,6 +394,7 @@ services:
       - PORT=10003
       - TITLE
       - TITLE_URL
+    stop_grace_period: 5m
     hostname: autogen_x86-master-0
     volumes:
       - ./logs:/var/log/buildbot
@@ -424,6 +433,7 @@ services:
       - PORT=10004
       - TITLE
       - TITLE_URL
+    stop_grace_period: 5m
     hostname: master-docker-nonstandard
     volumes:
       - ./logs:/var/log/buildbot
@@ -463,6 +473,7 @@ services:
       - PORT=10005
       - TITLE
       - TITLE_URL
+    stop_grace_period: 5m
     hostname: master-galera
     volumes:
       - ./logs:/var/log/buildbot
@@ -501,6 +512,7 @@ services:
       - PORT=10006
       - TITLE
       - TITLE_URL
+    stop_grace_period: 5m
     hostname: master-protected-branches
     volumes:
       - ./logs:/var/log/buildbot
@@ -539,6 +551,7 @@ services:
       - PORT=10007
       - TITLE
       - TITLE_URL
+    stop_grace_period: 5m
     hostname: master-docker-nonstandard-2
     volumes:
       - ./logs:/var/log/buildbot
@@ -578,6 +591,7 @@ services:
       - PORT=10008
       - TITLE
       - TITLE_URL
+    stop_grace_period: 5m
     hostname: master-bintars
     volumes:
       - ./logs:/var/log/buildbot

docker-compose/generate-config.py

@@ -7,6 +7,8 @@
 config = {"private": {}}
 exec(open("../master-private.cfg").read(), config, {})
 
+BUILDBOT_STOP_GRACE_PERIOD = "5m"
+
 MASTER_DIRECTORIES = [
     "master-nonlatent",
     "master-libvirt",
@@ -91,6 +93,7 @@
     image: quay.io/mariadb-foundation/bb-master:{environment}master-web
     restart: unless-stopped
     container_name: master-web
+    stop_grace_period: {buildbot_stop_grace_period}
     hostname: master-web
     volumes:
       - ./logs:/var/log/buildbot
@@ -110,6 +113,7 @@
     image: quay.io/mariadb-foundation/bb-master:{environment}master
     restart: unless-stopped
     container_name: {master_name}
+    stop_grace_period: {buildbot_stop_grace_period}
     hostname: {master_name}
     {volumes}
     entrypoint:
@@ -202,6 +206,7 @@ def main(args):
                 port=master_web_port,
                 environment="" if args.env == "prod" else "dev_",
                 config_path=args.env,
+                buildbot_stop_grace_period=BUILDBOT_STOP_GRACE_PERIOD,
             )
         )
         port = starting_port
@@ -216,6 +221,7 @@ def main(args):
                 mc_host=mc_host,
                 volumes=generate_volumes(master_volumes[master_name]),
                 environment="" if args.env == "prod" else "dev_",
+                buildbot_stop_grace_period=BUILDBOT_STOP_GRACE_PERIOD,
             )
             port += 1
 

docker-compose/start-bbm-web.sh

@@ -30,4 +30,4 @@ while ! nc -z 127.0.0.1 8080; do
 done
 echo "Crossbar started"
 buildbot upgrade-master /srv/buildbot/master/master-web
-buildbot start --nodaemon
+exec buildbot start --nodaemon

docker-compose/start.sh

@@ -41,4 +41,4 @@ fi
 # loop for debug
 # while true; do date && sleep 30; done
 
-buildbot start --nodaemon
+exec buildbot start --nodaemon