From ef6df0c6042d4f3b022641ffcc097bfa4056e1f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Tue, 3 Dec 2024 21:40:13 +0200 Subject: [PATCH] Move master-docker-nonstandard to use common master setup --- master-docker-nonstandard/master.cfg | 149 ++++++++++----------------- master.cfg | 2 +- 2 files changed, 53 insertions(+), 98 deletions(-) diff --git a/master-docker-nonstandard/master.cfg b/master-docker-nonstandard/master.cfg index 531612c0..e0bb6418 100644 --- a/master-docker-nonstandard/master.cfg +++ b/master-docker-nonstandard/master.cfg @@ -1,86 +1,63 @@ # -*- python -*- # ex: set filetype=python: -from buildbot.plugins import * -from buildbot.process.properties import Property, Properties -from buildbot.steps.shell import ShellCommand, Compile, Test, SetPropertyFromCommand -from buildbot.steps.mtrlogobserver import MTR, MtrLogObserver -from buildbot.steps.source.github import GitHub -from buildbot.process.remotecommand import RemoteCommand -from datetime import timedelta -from twisted.internet import defer - -import docker import os -import sys from collections import defaultdict -sys.path.insert(0, "/srv/buildbot/master") -sys.setrecursionlimit(10000) - -from common_factories import * -from constants import * -from locks import * -from schedulers_definition import SCHEDULERS -from utils import * +import docker +from buildbot.plugins import steps, util, worker +from buildbot.process.properties import Property +from common_factories import ( + addTests, + getHTMLLogString, + getQuickBuildFactory, + getRpmAutobakeFactory, + getSourceTarball, +) +from constants import MTR_ENV +from locks import getLocks +from master_common import base_master_config +from utils import ( + canStartBuild, + createVar, + createWorker, + dockerfile, + filterBranch, + hasAutobake, + hasBigtest, + hasEco, + hasFailed, + moveMTRLogs, + mtrJobsMultiplier, + nextBuild, + printEnv, + saveLogs, + savePackage, +) FQDN = os.getenv("BUILDMASTER_WG_IP", default="100.64.100.1") -# This is the dictionary that the buildmaster pays attention to. We also use -# a shorter alias to save typing. -c = BuildmasterConfig = {} - -# Load the slave, database passwords and 3rd-party tokens from an external private file, so -# that the rest of the configuration can be public. +# Load the slave, database passwords and 3rd-party tokens from an external +# private file, so that the rest of the configuration can be public. config = {"private": {}} exec(open("master-private.cfg").read(), config, {}) -####### BUILDBOT SERVICES - -# 'services' is a list of BuildbotService items like reporter targets. The -# status of each build will be pushed to these targets. buildbot/reporters/*.py -# has a variety to choose from, like IRC bots. - - -c["services"] = [] -context = util.Interpolate("buildbot/%(prop:buildername)s") -gs = reporters.GitHubStatusPush( - token=config["private"]["gh_mdbci"]["access_token"], - context=context, - startDescription="Build started.", - endDescription="Build done.", - verbose=True, - builders=GITHUB_STATUS_BUILDERS, -) -c["services"].append(gs) -c['secretsProviders'] = [secrets.SecretInAFile(dirname=os.getenv("MASTER_CREDENTIALS_DIR", default="/srv/buildbot/master/master-credential-provider"))] -####### PROJECT IDENTITY - -# the 'title' string will appear at the top of this buildbot installation's -# home pages (linked to the 'titleURL'). -c["title"] = os.getenv("TITLE", default="MariaDB CI") -c["titleURL"] = os.getenv("TITLE_URL", default="https://github.com/MariaDB/server") - -# the 'buildbotURL' string should point to the location where the buildbot's -# internal web server is visible. This typically uses the port number set in -# the 'www' entry below, but with an externally-visible host name which the -# buildbot cannot figure out without some help. -c["buildbotURL"] = os.getenv("BUILDMASTER_URL", default="https://buildbot.mariadb.org/") - -# 'protocols' contains information about protocols which master will use for -# communicating with workers. You must define at least 'port' option that workers -# could connect to your master with this protocol. -# 'port' must match the value configured into the workers (with their -# --master option) -port = int(os.getenv("PORT", default="9992")) -c["protocols"] = {"pb": {"port": port}} - -####### DB URL - -c["db"] = { - # This specifies what database buildbot uses to store its state. - "db_url": config["private"]["db_url"] -} +# This is the dictionary that the buildmaster pays attention to. We also use +# a shorter alias to save typing. +c = BuildmasterConfig = base_master_config( + title=os.getenv("TITLE", default="MariaDB CI"), + title_url=os.getenv("TITLE_URL", + default="https://github.com/MariaDB/server"), + buildbot_url=os.getenv("BUILDMASTER_URL", + default="https://buildbot.mariadb.org/"), + github_access_token=config["private"]["gh_mdbci"]["access_token"], + # TODO(cvicentiu) undo this hardcoding + secrets_provider_file=os.getenv("MASTER_CREDENTIALS_DIR", + default="/srv/buildbot/master/master-credential-provider"), + master_port=int(os.getenv("PORT", default="9992")), + db_url=config["private"]["db_url"], + mq_router_url=os.getenv("MQ_ROUTER_URL", default="ws://localhost:8085/ws")) + mtrDbPool = util.EqConnectionPool( "MySQLdb", @@ -90,14 +67,6 @@ mtrDbPool = util.EqConnectionPool( config["private"]["db_mtr_db"], ) -####### Disable net usage reports from being sent to buildbot.net -c["buildbotNetUsageData"] = None - -####### SCHEDULERS - -# Configure the Schedulers, which decide how to react to incoming changes. -c["schedulers"] = SCHEDULERS - ####### WORKERS # The 'workers' list defines the set of recognized workers. Each element is @@ -132,8 +101,6 @@ def addWorker( # Docker workers -fqdn = os.getenv("BUILDMASTER_WG_IP", default="100.64.100.1") - ## hz-bbw2-docker c["workers"].append( worker.DockerLatentWorker( @@ -142,7 +109,7 @@ c["workers"].append( docker_host=config["private"]["docker_workers"]["hz-bbw2-docker"], dockerfile=open("dockerfiles/eco-php-ubuntu-2004.dockerfile").read(), followStartupLogs=False, - masterFQDN=fqdn, + masterFQDN=FQDN, hostconfig={ "shm_size": "6G", "ulimits": [ @@ -165,7 +132,7 @@ c["workers"].append( "dockerfiles/eco-pymysql-python-3-9-slim-buster.dockerfile" ).read(), followStartupLogs=False, - masterFQDN=fqdn, + masterFQDN=FQDN, hostconfig={ "shm_size": "6G", "ulimits": [ @@ -186,7 +153,7 @@ c["workers"].append( docker_host=config["private"]["docker_workers"]["hz-bbw2-docker"], dockerfile=open("dockerfiles/eco-mysqljs-nodejs15-buster.dockerfile").read(), followStartupLogs=False, - masterFQDN=fqdn, + masterFQDN=FQDN, hostconfig={ "shm_size": "6G", "ulimits": [ @@ -213,7 +180,7 @@ c["workers"].append( followStartupLogs=False, autopull=True, alwaysPull=True, - masterFQDN=fqdn, + masterFQDN=FQDN, hostconfig={ "shm_size": "20G", "ulimits": [ @@ -1758,15 +1725,3 @@ c["builders"].append( factory=f_prep_local, ) ) - -c["logEncoding"] = "utf-8" - -c["multiMaster"] = True - -c["mq"] = { # Need to enable multimaster aware mq. Wamp is the only option for now. - "type": "wamp", - "router_url": os.getenv("MQ_ROUTER_URL", default="ws://localhost:8085/ws"), - "realm": "realm1", - # valid are: none, critical, error, warn, info, debug, trace - "wamp_debug_level": "info", -} diff --git a/master.cfg b/master.cfg index 2c9cf9e3..2dce64cb 100644 --- a/master.cfg +++ b/master.cfg @@ -56,7 +56,7 @@ c = BuildmasterConfig = base_master_config( github_access_token=config["private"]["gh_mdbci"]["access_token"], # TODO(cvicentiu) undo this hardcoding secrets_provider_file="/srv/buildbot/master/master-credential-provider", - master_port=os.getenv("PORT", default=master_config["port"]), + master_port=int(os.getenv("PORT", default=master_config["port"])), db_url=config["private"]["db_url"], mq_router_url=os.getenv("MQ_ROUTER_URL", default="ws://localhost:8085/ws"))