From 19e99e9d5ae5b071fc3ff3bb50f9ead2148f7c89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Wed, 4 Dec 2024 16:24:29 +0200 Subject: [PATCH] master-nonlatent move to common class --- master-nonlatent/master.cfg | 232 +++++++++++++++--------------------- 1 file changed, 98 insertions(+), 134 deletions(-) diff --git a/master-nonlatent/master.cfg b/master-nonlatent/master.cfg index 1fe1f091..60437479 100644 --- a/master-nonlatent/master.cfg +++ b/master-nonlatent/master.cfg @@ -1,66 +1,53 @@ # -*- python -*- # ex: set filetype=python: -from buildbot.plugins import * -from buildbot.process.properties import Property, Properties -from buildbot.steps.shell import ShellCommand, Compile, Test, SetPropertyFromCommand -from buildbot.steps.mtrlogobserver import MTR, MtrLogObserver -from buildbot.steps.source.github import GitHub -from buildbot.process.remotecommand import RemoteCommand -from datetime import timedelta -from twisted.internet import defer - -import docker import os -import sys - -sys.setrecursionlimit(10000) +from datetime import timedelta -sys.path.insert(0, "/srv/buildbot/master") -from common_factories import * -from locks import * -from schedulers_definition import SCHEDULERS -from utils import * +from buildbot.plugins import steps, util, worker +from buildbot.process.properties import Property +from common_factories import addWinTests, getQuickBuildFactory +from constants import MTR_ENV, SAVED_PACKAGE_BRANCHES +from master_common import base_master_config +from utils import ( + canStartBuild, + createVar, + getHTMLLogString, + getSourceTarball, + hasFailed, + ls2list, + moveMTRLogs, + mtrJobsMultiplier, + nextBuild, + prioritizeBuilders, + savePackage, +) ####### VARIABLES vsWarningPattern = "^.*: warning C.*$" -# This is the dictionary that the buildmaster pays attention to. We also use -# a shorter alias to save typing. -c = BuildmasterConfig = {} - # Load the slave, database passwords and 3rd-party tokens from an external private file, so # that the rest of the configuration can be public. config = {"private": {}} exec(open("master-private.cfg").read(), config, {}) -####### PROJECT IDENTITY - -# the 'title' string will appear at the top of this buildbot installation's -# home pages (linked to the 'titleURL'). -c["title"] = os.getenv("TITLE", default="MariaDB CI") -c["titleURL"] = os.getenv("TITLE_URL", default="https://github.com/MariaDB/server") - -# the 'buildbotURL' string should point to the location where the buildbot's -# internal web server is visible. This typically uses the port number set in -# the 'www' entry below, but with an externally-visible host name which the -# buildbot cannot figure out without some help. -c["buildbotURL"] = os.getenv("BUILDMASTER_URL", default="https://buildbot.mariadb.org/") - -# 'protocols' contains information about protocols which master will use for -# communicating with workers. You must define at least 'port' option that workers -# could connect to your master with this protocol. -# 'port' must match the value configured into the workers (with their -# --master option) -port = int(os.getenv("PORT", default="9989")) -c["protocols"] = {"pb": {"port": port}} - -####### DB URL +# This is the dictionary that the buildmaster pays attention to. We also use +# a shorter alias to save typing. +c = BuildmasterConfig = base_master_config( + title=os.getenv("TITLE", default="MariaDB CI"), + title_url=os.getenv("TITLE_URL", default="https://github.com/MariaDB/server"), + buildbot_url=os.getenv("BUILDMASTER_URL", default="https://buildbot.mariadb.org/"), + github_access_token=config["private"]["gh_mdbci"]["access_token"], + # TODO(cvicentiu) undo this hardcoding + secrets_provider_file=os.getenv( + "MASTER_CREDENTIALS_DIR", + default="/srv/buildbot/master/master-credential-provider", + ), + master_port=int(os.getenv("PORT", default="9989")), + db_url=config["private"]["db_url"], + mq_router_url=os.getenv("MQ_ROUTER_URL", default="ws://localhost:8085/ws"), +) -c["db"] = { - # This specifies what database buildbot uses to store its state. - "db_url": config["private"]["db_url"] -} mtrDbPool = util.EqConnectionPool( "MySQLdb", @@ -70,29 +57,10 @@ mtrDbPool = util.EqConnectionPool( config["private"]["db_mtr_db"], ) -####### Disable net usage reports from being sent to buildbot.net -c["buildbotNetUsageData"] = None -####### Services -c["services"] = [] -context = util.Interpolate("buildbot/%(prop:buildername)s") -gs = reporters.GitHubStatusPush( - token=config["private"]["gh_mdbci"]["access_token"], - context=context, - startDescription="Build started.", - endDescription="Build done.", - verbose=True, - builders=GITHUB_STATUS_BUILDERS, -) -c["services"].append(gs) -c['secretsProviders'] = [secrets.SecretInAFile(dirname=os.getenv("MASTER_CREDENTIALS_DIR", default="/srv/buildbot/master/master-credential-provider"))] ####### Builder priority c["prioritizeBuilders"] = prioritizeBuilders -####### SCHEDULERS - -# Configure the Schedulers, which decide how to react to incoming changes. -c["schedulers"] = SCHEDULERS ####### WORKERS @@ -159,9 +127,6 @@ f_windows_env = { } f_windows_env.update(MTR_ENV) -# f_quick_build = getQuickBuildFactory(mtrDbPool) -f_rpm_autobake = getRpmAutobakeFactory(mtrDbPool) - ## f_windows f_windows = util.BuildFactory() f_windows.addStep( @@ -268,23 +233,22 @@ f_windows.addStep( ) windows_tests = { - "nm": { - "create_scripts": True - }, + "nm": {"create_scripts": True}, "connect": { "suites": ["connect"], }, } for typ in windows_tests: - addWinTests(f_windows, - mtr_test_type=typ, - mtr_env=f_windows_env, - mtr_additional_args=util.Property("mtr_additional_args", default=""), - mtr_step_db_pool=mtrDbPool, - mtr_suites=windows_tests[typ].get('suites',["default"]), - create_scripts=windows_tests[typ].get('create_scripts',False) - ) + addWinTests( + f_windows, + mtr_test_type=typ, + mtr_env=f_windows_env, + mtr_additional_args=util.Property("mtr_additional_args", default=""), + mtr_step_db_pool=mtrDbPool, + mtr_suites=windows_tests[typ].get("suites", ["default"]), + create_scripts=windows_tests[typ].get("create_scripts", False), + ) f_windows.addStep( @@ -300,13 +264,13 @@ f_windows.addStep( ), url=util.Interpolate( f'{os.getenv("ARTIFACTS_URL", default="https://ci.mariadb.org")}' - '/' + "/" "%(prop:tarbuildnum)s" - '/' - 'logs' - '/' - '%(prop:buildername)s' - '/' + "/" + "logs" + "/" + "%(prop:buildername)s" + "/" ), ) ) @@ -331,8 +295,16 @@ f_windows.addStep( ## f_windows_msi f_windows_msi_env = { - "TMP": util.Interpolate("{0}\\%(prop:buildername)s\\build\\tmpdir".format('D:\\DEV\\Buildbot' if os.getenv('ENVIRON') == 'DEV' else 'D:\\Buildbot')), - "TEMP": util.Interpolate("{0}\\%(prop:buildername)s\\build\\tmpdir".format('D:\\DEV\\Buildbot' if os.getenv('ENVIRON') == 'DEV' else 'D:\\Buildbot')), + "TMP": util.Interpolate( + "{0}\\%(prop:buildername)s\\build\\tmpdir".format( + "D:\\DEV\\Buildbot" if os.getenv("ENVIRON") == "DEV" else "D:\\Buildbot" + ) + ), + "TEMP": util.Interpolate( + "{0}\\%(prop:buildername)s\\build\\tmpdir".format( + "D:\\DEV\\Buildbot" if os.getenv("ENVIRON") == "DEV" else "D:\\Buildbot" + ) + ), } f_windows_msi_env.update(MTR_ENV) @@ -457,23 +429,22 @@ f_windows_msi.addStep( ) ) windows_msi_tests = { - "nm": { - "create_scripts": True - }, + "nm": {"create_scripts": True}, "connect": { "suites": ["connect"], }, } for typ in windows_msi_tests: - addWinTests(f_windows_msi, - mtr_test_type=typ, - mtr_env=f_windows_msi_env, - mtr_additional_args=util.Property("mtr_additional_args", default=""), - mtr_step_db_pool=mtrDbPool, - mtr_suites=windows_msi_tests[typ].get('suites',["default"]), - create_scripts=windows_msi_tests[typ].get('create_scripts',False) - ) + addWinTests( + f_windows_msi, + mtr_test_type=typ, + mtr_env=f_windows_msi_env, + mtr_additional_args=util.Property("mtr_additional_args", default=""), + mtr_step_db_pool=mtrDbPool, + mtr_suites=windows_msi_tests[typ].get("suites", ["default"]), + create_scripts=windows_msi_tests[typ].get("create_scripts", False), + ) f_windows_msi.addStep( steps.DirectoryUpload( @@ -488,13 +459,13 @@ f_windows_msi.addStep( ), url=util.Interpolate( f'{os.getenv("ARTIFACTS_URL", default="https://ci.mariadb.org")}' - '/' + "/" "%(prop:tarbuildnum)s" - '/' - 'logs' - '/' - '%(prop:buildername)s' - '/' + "/" + "logs" + "/" + "%(prop:buildername)s" + "/" ), ) ) @@ -528,12 +499,11 @@ f_windows_msi.addStep( mode=0o755, url=util.Interpolate( f'{os.getenv("ARTIFACTS_URL", default="https://ci.mariadb.org")}' - '/' - '%(prop:tarbuildnum)s' - '/' - '%(prop:buildername)s' - '/' - + "/" + "%(prop:tarbuildnum)s" + "/" + "%(prop:buildername)s" + "/" ), doStepIf=lambda step: savePackage(step, SAVED_PACKAGE_BRANCHES), ) @@ -551,11 +521,11 @@ f_windows_msi.addStep( mode=0o755, url=util.Interpolate( f'{os.getenv("ARTIFACTS_URL", default="https://ci.mariadb.org")}' - '/' - '%(prop:tarbuildnum)s' - '/' - '%(prop:buildername)s' - '/' + "/" + "%(prop:tarbuildnum)s" + "/" + "%(prop:buildername)s" + "/" ), doStepIf=lambda step: savePackage(step, SAVED_PACKAGE_BRANCHES), ) @@ -739,7 +709,11 @@ f_dockerlibrary.addStep( f_dockerlibrary.addStep( steps.SetPropertyFromCommand( name="Extract the last tag created", - command=['bash','-c','if [ -f last_tag ]; then cat last_tag; else echo ""; fi'], + command=[ + "bash", + "-c", + 'if [ -f last_tag ]; then cat last_tag; else echo ""; fi', + ], property="lasttag", ) ) @@ -747,14 +721,14 @@ f_dockerlibrary.addStep( steps.SetPropertyFromCommand( name="Determine sha for the last tag", command=[ - "bash", + "bash", "-xc", util.Interpolate( 'curl -s "https://quay.io/api/v1/repository/mariadb-foundation/mariadb-devel/tag/?filter_tag_name=eq:%(prop:lasttag)s&onlyActiveTags=true" | jq -r .tags[0].manifest_digest' ), ], property="lastsha", - doStepIf=lambda step: (str(step.getProperty("lasttag")) != "") + doStepIf=lambda step: (str(step.getProperty("lasttag")) != ""), ) ) f_dockerlibrary.addStep( @@ -765,7 +739,9 @@ f_dockerlibrary.addStep( "-xc", "gh auth login --with-token < ~/gh_auth", ], - doStepIf=lambda step: (str(step.getProperty("lastsha")) != "null" and step.hasProperty("lastsha")), + doStepIf=lambda step: ( + str(step.getProperty("lastsha")) != "null" and step.hasProperty("lastsha") + ), ) ) @@ -1132,8 +1108,8 @@ c["builders"].append( "cxx_compiler": "clang++", "mtr_env": { "WSREP_PROVIDER": "/usr/local/lib/libgalera_smm.so", - } }, + }, nextBuild=nextBuild, factory=f_freebsd, ) @@ -1141,15 +1117,3 @@ c["builders"].append( # Add a Janitor configurator that removes old logs c["configurators"] = [util.JanitorConfigurator(logHorizon=timedelta(weeks=6), hour=23)] - -c["logEncoding"] = "utf-8" - -c["multiMaster"] = True - -c["mq"] = { # Need to enable multimaster aware mq. Wamp is the only option for now. - "type": "wamp", - "router_url": os.getenv("MQ_ROUTER_URL", default="ws://localhost:8085/ws"), - "realm": "realm1", - # valid are: none, critical, error, warn, info, debug, trace - "wamp_debug_level": "info", -}