From 64e29bf1cdcf9800706237ccd7ea505e1e775a6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Fri, 29 Nov 2024 11:41:42 +0200 Subject: [PATCH] Clean-up master.cfg, initial step --- master-bintars/master.cfg | 2 +- master-docker-nonstandard-2/master.cfg | 2 +- master-docker-nonstandard/master.cfg | 2 +- master-galera/master.cfg | 2 +- master-nonlatent/master.cfg | 2 +- master-protected-branches/master.cfg | 2 +- master-web/master.cfg | 2 +- master.cfg | 62 ++++++++++++++++---------- validate_master_cfg.sh | 2 +- 9 files changed, 46 insertions(+), 32 deletions(-) diff --git a/master-bintars/master.cfg b/master-bintars/master.cfg index 3d8237ee..dec48136 100644 --- a/master-bintars/master.cfg +++ b/master-bintars/master.cfg @@ -33,7 +33,7 @@ c = BuildmasterConfig = {} # Load the slave, database passwords and 3rd-party tokens from an external private file, so # that the rest of the configuration can be public. config = {"private": {}} -exec(open("../master-private.cfg").read(), config, {}) +exec(open("master-private.cfg").read(), config, {}) ####### BUILDBOT SERVICES diff --git a/master-docker-nonstandard-2/master.cfg b/master-docker-nonstandard-2/master.cfg index e4a9b760..dee4b62c 100644 --- a/master-docker-nonstandard-2/master.cfg +++ b/master-docker-nonstandard-2/master.cfg @@ -32,7 +32,7 @@ c = BuildmasterConfig = {} # Load the slave, database passwords and 3rd-party tokens from an external private file, so # that the rest of the configuration can be public. config = {"private": {}} -exec(open("../master-private.cfg").read(), config, {}) +exec(open("master-private.cfg").read(), config, {}) ####### BUILDBOT SERVICES diff --git a/master-docker-nonstandard/master.cfg b/master-docker-nonstandard/master.cfg index 8e64ca11..6155934d 100644 --- a/master-docker-nonstandard/master.cfg +++ b/master-docker-nonstandard/master.cfg @@ -32,7 +32,7 @@ c = BuildmasterConfig = {} # Load the slave, database passwords and 3rd-party tokens from an external private file, so # that the rest of the configuration can be public. config = {"private": {}} -exec(open("../master-private.cfg").read(), config, {}) +exec(open("master-private.cfg").read(), config, {}) ####### BUILDBOT SERVICES diff --git a/master-galera/master.cfg b/master-galera/master.cfg index 46542537..aa3642f3 100644 --- a/master-galera/master.cfg +++ b/master-galera/master.cfg @@ -26,7 +26,7 @@ c = BuildmasterConfig = {} # Load the slave, database passwords and 3rd-party tokens from an external private file, so # that the rest of the configuration can be public. config = {"private": {}} -exec(open("../master-private.cfg").read(), config, {}) +exec(open("master-private.cfg").read(), config, {}) FQDN = os.getenv("BUILDMASTER_WG_IP", default="100.64.100.1") diff --git a/master-nonlatent/master.cfg b/master-nonlatent/master.cfg index 82d5b650..131b1ae1 100644 --- a/master-nonlatent/master.cfg +++ b/master-nonlatent/master.cfg @@ -32,7 +32,7 @@ c = BuildmasterConfig = {} # Load the slave, database passwords and 3rd-party tokens from an external private file, so # that the rest of the configuration can be public. config = {"private": {}} -exec(open("../master-private.cfg").read(), config, {}) +exec(open("master-private.cfg").read(), config, {}) ####### PROJECT IDENTITY diff --git a/master-protected-branches/master.cfg b/master-protected-branches/master.cfg index 836e9c48..eadefb98 100644 --- a/master-protected-branches/master.cfg +++ b/master-protected-branches/master.cfg @@ -31,7 +31,7 @@ c = BuildmasterConfig = {} # Load the slave, database passwords and 3rd-party tokens from an external private file, so # that the rest of the configuration can be public. config = {"private": {}} -exec(open("../master-private.cfg").read(), config, {}) +exec(open("master-private.cfg").read(), config, {}) ####### BUILDBOT SERVICES diff --git a/master-web/master.cfg b/master-web/master.cfg index c2e6685f..5346212b 100644 --- a/master-web/master.cfg +++ b/master-web/master.cfg @@ -33,7 +33,7 @@ c = BuildmasterConfig = {} # Load the slave, database passwords and 3rd-party tokens from an external # private file, so that the rest of the configuration can be public. config = {"private": {}} -exec(open("../master-private.cfg").read(), config, {}) +exec(open("master-private.cfg").read(), config, {}) ####### PROJECT IDENTITY diff --git a/master.cfg b/master.cfg index 6a3f00b4..9c4c029d 100644 --- a/master.cfg +++ b/master.cfg @@ -1,28 +1,40 @@ # -*- python -*- # ex: set filetype=python: -from buildbot.plugins import * -from buildbot.process.properties import Property, Properties -from buildbot.steps.shell import ShellCommand, Compile, Test, SetPropertyFromCommand -from buildbot.steps.mtrlogobserver import MTR, MtrLogObserver -from buildbot.steps.source.github import GitHub -from buildbot.process.remotecommand import RemoteCommand -from datetime import timedelta -from twisted.internet import defer - -import docker import os -import sys -sys.setrecursionlimit(10000) +import yaml -sys.path.insert(0, "/srv/buildbot/master") - -from common_factories import * -from constants import * -from locks import * +from buildbot.plugins import reporters, secrets, steps, util +from buildbot.process.properties import Property +from common_factories import ( + getQuickBuildFactory, + getRpmAutobakeFactory, + getSourceTarball, +) +from constants import ( + BUILDERS_GALERA_MTR, + GITHUB_STATUS_BUILDERS, + SAVED_PACKAGE_BRANCHES, + os_info, +) +from locks import getLocks from schedulers_definition import SCHEDULERS -from utils import * +from utils import ( + canStartBuild, + createDebRepo, + createWorker, + dockerfile, + hasDockerLibrary, + hasFiles, + hasInstall, + hasUpgrade, + ls2string, + nextBuild, + printEnv, + savePackage, + uploadDebArtifacts, +) with open("master-config.yaml", "r") as f: master_config = yaml.safe_load(f) @@ -31,12 +43,14 @@ with open("master-config.yaml", "r") as f: # a shorter alias to save typing. c = BuildmasterConfig = {} -# Load the slave, database passwords and 3rd-party tokens from an external private file, so -# that the rest of the configuration can be public. +# Load the slave, database passwords and 3rd-party tokens from an external +# private file, so that the rest of the configuration can be public. config = {"private": {}} exec(open("master-private.cfg").read(), config, {}) -####### BUILDBOT SERVICES +####### +# BUILDBOT SERVICES +####### # 'services' is a list of BuildbotService items like reporter targets. The # status of each build will be pushed to these targets. buildbot/reporters/*.py @@ -235,7 +249,7 @@ f_deb_autobake.addStep( "master_branch": Property("master_branch"), "parentbuildername": Property("buildername"), }, - doStepIf=lambda step: savePackage(step), + doStepIf=lambda step: savePackage(step, SAVED_PACKAGE_BRANCHES), ) ) f_deb_autobake.addStep( @@ -250,7 +264,7 @@ f_deb_autobake.addStep( "master_branch": Property("master_branch"), "parentbuildername": Property("buildername"), }, - doStepIf=lambda step: hasInstall(step) and savePackage(step) and hasFiles(step), + doStepIf=lambda step: hasInstall(step) and savePackage(step, SAVED_PACKAGE_BRANCHES) and hasFiles(step), ) ) f_deb_autobake.addStep( @@ -265,7 +279,7 @@ f_deb_autobake.addStep( "master_branch": Property("master_branch"), "parentbuildername": Property("buildername"), }, - doStepIf=lambda step: hasUpgrade(step) and savePackage(step) and hasFiles(step), + doStepIf=lambda step: hasUpgrade(step) and savePackage(step, SAVED_PACKAGE_BRANCHES) and hasFiles(step), ) ) f_deb_autobake.addStep( diff --git a/validate_master_cfg.sh b/validate_master_cfg.sh index 0349698e..edec7c12 100755 --- a/validate_master_cfg.sh +++ b/validate_master_cfg.sh @@ -50,6 +50,6 @@ for dir in autogen/* \ $RUNC run -i -v "$(pwd):/srv/buildbot/master" \ -w /srv/buildbot/master \ quay.io/mariadb-foundation/bb-master:master \ - bash -c "cd $dir && buildbot checkconfig master.cfg" + bash -c "buildbot checkconfig $dir/master.cfg" echo -e "done\n" done