From d69846d754e2a38ed68e5f0bbcb80f0e7e61b9a5 Mon Sep 17 00:00:00 2001 From: Faustin Lammler Date: Sun, 8 Oct 2023 10:11:09 +0200 Subject: [PATCH] Local storage for artifacts needs to be env var too prod: /mnt/autofs/master_packages dev: /mnt/autofs/master_dev_packages --- .github/workflows/bbm_check_conf.yml | 18 ++++---- .github/workflows/deploy.yml | 19 ++------- define_masters.py | 62 +++++++++++++++++----------- docker-compose/.env | 3 +- docker-compose/.env.dev | 3 +- docker-compose/generate-config.py | 10 ++--- master-protected-branches/master.cfg | 11 ++--- 7 files changed, 63 insertions(+), 63 deletions(-) diff --git a/.github/workflows/bbm_check_conf.yml b/.github/workflows/bbm_check_conf.yml index 3296394a..5f609c40 100644 --- a/.github/workflows/bbm_check_conf.yml +++ b/.github/workflows/bbm_check_conf.yml @@ -1,36 +1,32 @@ --- -name: bbm conf check +name: bbm-check-conf on: push: paths: - ".github/workflows/bbm_check_conf.yml" + - ".github/workflows/deploy.yml" - "autogen/**" - "builtbot.tac" - "common_factories.py" - "constants.py" + - "docker-compose/**" - "locks.py" - - "master-docker-nonstandard/master.cfg" - - "master-galera/master.cfg" - - "master-nonlatent/master.cfg" - - "master-protected-branches/master.cfg" - - "master-web/master.cfg" + - "master-**" - "master.cfg" - "utils.py" - "validate_master_cfg.sh" pull_request: paths: - ".github/workflows/bbm_check_conf.yml" + - ".github/workflows/deploy.yml" - "autogen/**" - "builtbot.tac" - "common_factories.py" - "constants.py" + - "docker-compose/**" - "locks.py" - - "master-docker-nonstandard/master.cfg" - - "master-galera/master.cfg" - - "master-nonlatent/master.cfg" - - "master-protected-branches/master.cfg" - - "master-web/master.cfg" + - "master-**" - "master.cfg" - "utils.py" - "validate_master_cfg.sh" diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index f5cfe950..52a4568b 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -2,22 +2,9 @@ name: deploy on: - push: - paths: - - ".github/workflows/deploy.yml" - - "autogen/**" - - "docker-compose/**" - - "master-**" - - "master.cfg" - - "rsync.exclude" - pull_request: - paths: - - ".github/workflows/deploy.yml" - - "autogen/**" - - "docker-compose/**" - - "master-**" - - "master.cfg" - - "rsync.exclude" + workflow_run: + workflows: ["bbm-check-conf", "pre-commit"] + types: [completed] jobs: build: diff --git a/define_masters.py b/define_masters.py index bda121b5..67727f97 100755 --- a/define_masters.py +++ b/define_masters.py @@ -1,57 +1,71 @@ #!/usr/bin/env python3 -import yaml import os import shutil +import yaml -base_path = "autogen/" +BASE_PATH = "autogen/" config = {"private": {}} exec(open("master-private.cfg").read(), config, {}) -with open('os_info.yaml', 'r') as f: - os_info = yaml.safe_load(f) +with open("os_info.yaml", mode="r", encoding="utf-8") as file: + os_info = yaml.safe_load(file) platforms = {} for os_name in os_info: - for arch in os_info[os_name]['arch']: - builder_name = arch + '-' + os_name + for arch in os_info[os_name]["arch"]: + builder_name = arch + "-" + os_name if arch not in platforms: platforms[arch] = [] platforms[arch].append(builder_name) # Clear old configurations -if os.path.exists(base_path): - shutil.rmtree(base_path) +if os.path.exists(BASE_PATH): + shutil.rmtree(BASE_PATH) -idx = 0 +IDX = 0 for arch in platforms: # Create the directory for the architecture that is handled by each master # If for a given architecture there are more than "max_builds" builds, # create multiple masters # "max_builds" is defined is master-private.py - num_masters = int(len(platforms[arch]) / config['private']['master-variables']['max_builds']) + 1 + num_masters = ( + int(len(platforms[arch]) / config["private"]["master-variables"]["max_builds"]) + + 1 + ) for master_id in range(num_masters): - dir_path = base_path + arch + "-master-" + str(master_id) + dir_path = BASE_PATH + arch + "-master-" + str(master_id) os.makedirs(dir_path) master_config = {} - master_config['builders'] = platforms[arch] - master_config['workers'] = config['private']['master-variables']['workers'][arch] + master_config["builders"] = platforms[arch] + master_config["workers"] = config["private"]["master-variables"]["workers"][ + arch + ] - starting_port = int(os.getenv('PORT', default=config['private']['master-variables']['starting_port'])) - master_config['port'] = starting_port + idx - master_config['log_name'] = "master-docker-" + arch + "-" + str(master_id) + '.log' + starting_port = int( + os.getenv( + "PORT", default=config["private"]["master-variables"]["starting_port"] + ) + ) + master_config["port"] = starting_port + IDX + master_config["log_name"] = ( + "master-docker-" + arch + "-" + str(master_id) + ".log" + ) - with open(dir_path + '/master-config.yaml', 'w') as f: - yaml.dump(master_config, f) + with open(dir_path + "/master-config.yaml", mode="w", encoding="utf-8") as file: + yaml.dump(master_config, file) - shutil.copyfile('master.cfg', dir_path + '/master.cfg') - shutil.copyfile('master-private.cfg', dir_path + '/master-private.cfg') + shutil.copyfile("master.cfg", dir_path + "/master.cfg") + shutil.copyfile("master-private.cfg", dir_path + "/master-private.cfg") - buildbot_tac = open("buildbot.tac", "r").read() % master_config['log_name'] - with open(dir_path + '/buildbot.tac', 'w') as f: + buildbot_tac = ( + open("buildbot.tac", mode="r", encoding="utf-8").read() + % master_config["log_name"] + ) + with open(dir_path + "/buildbot.tac", mode="w", encoding="utf-8") as f: f.write(buildbot_tac) - idx += 1 - print(arch, len(master_config['builders'])) + IDX += 1 + print(arch, len(master_config["builders"])) diff --git a/docker-compose/.env b/docker-compose/.env index 2c6e2df9..664253ed 100644 --- a/docker-compose/.env +++ b/docker-compose/.env @@ -1,4 +1,5 @@ -TITLE="MariaDB CI (DEV)" +TITLE="MariaDB CI" TITLE_URL=https://github.com/MariaDB/server BUILDMASTER_URL=https://buildbot.mariadb.org/ MQ_ROUTER_URL=ws://localhost:8085/ws +MASTER_PACKAGES_DIR="/mnt/autofs/master_dev_packages" diff --git a/docker-compose/.env.dev b/docker-compose/.env.dev index 85f5b261..39036b71 100644 --- a/docker-compose/.env.dev +++ b/docker-compose/.env.dev @@ -1,5 +1,6 @@ -TITLE="MariaDB CI" +TITLE="MariaDB CI (DEV)" TITLE_URL=https://github.com/MariaDB/server BUILDMASTER_URL=https://buildbot.dev.mariadb.org/ BUILDMASTER_WG_IP=100.64.101.1 MQ_ROUTER_URL=ws://crossbar:8080/ws +MASTER_PACKAGES_DIR="/mnt/autofs/master_dev_packages" diff --git a/docker-compose/generate-config.py b/docker-compose/generate-config.py index 4f06475b..0417e991 100755 --- a/docker-compose/generate-config.py +++ b/docker-compose/generate-config.py @@ -174,12 +174,12 @@ def main(args): starting_port = config["private"]["master-variables"]["starting_port"] master_web_port = 8010 # Generate startup scripts and Docker Compose pieces for each master directory - with open("docker-compose.yaml", "w") as f: - f.write( + with open("docker-compose.yaml", mode="w", encoding="utf-8") as file: + file.write( "# This is an autogenerated file. Do not edit it manually.\n\ # Use `python generate-config.py` instead." ) - f.write(start_template.format(port=master_web_port)) + file.write(start_template.format(port=master_web_port)) port = starting_port for master_directory in MASTER_DIRECTORIES: master_name = master_directory.replace("/", "_") @@ -193,8 +193,8 @@ def main(args): ) port += 1 - f.write(docker_compose_piece) - f.write(END_TEMPLATE) + file.write(docker_compose_piece) + file.write(END_TEMPLATE) if __name__ == "__main__": diff --git a/master-protected-branches/master.cfg b/master-protected-branches/master.cfg index 687526e1..3cb90a4c 100644 --- a/master-protected-branches/master.cfg +++ b/master-protected-branches/master.cfg @@ -96,7 +96,8 @@ c['schedulers'] = getSchedulers() c['workers'] = [] # Docker workers -fqdn = os.getenv('BUILDMASTER_WG_IP', default='100.64.100.1') +FQDN = os.getenv('BUILDMASTER_WG_IP', default='100.64.100.1') +MASTER_PACKAGES = os.getenv('MASTER_PACKAGES_DIR', default='/mnt/autofs/master_packages') ## hz-bbw2-docker c['workers'].append(worker.DockerLatentWorker("hz-bbw1-docker-tarball-debian-10", None, @@ -105,9 +106,9 @@ c['workers'].append(worker.DockerLatentWorker("hz-bbw1-docker-tarball-debian-10" followStartupLogs=False, autopull=True, alwaysPull=True, - masterFQDN=fqdn, + masterFQDN=FQDN, hostconfig={ 'shm_size':'1G' }, - volumes=['/mnt/autofs/master_packages/:/packages'], + volumes=['MASTER_PACKAGES/:/packages'], max_builds=1, build_wait_timeout=0, properties={ 'jobs':4, 'save_packages':True })) @@ -118,9 +119,9 @@ c['workers'].append(worker.DockerLatentWorker("hz-bbw4-docker-tarball-debian-10" followStartupLogs=False, autopull=True, alwaysPull=True, - masterFQDN=fqdn, + masterFQDN=FQDN, hostconfig={ 'shm_size':'1G' }, - volumes=['/mnt/autofs/master_packages/:/packages'], + volumes=['MASTER_PACKAGES/:/packages'], max_builds=1, build_wait_timeout=0, properties={ 'jobs':4, 'save_packages':True }))