Skip to content

Commit

Permalink
master-nonlatent move to common class
Browse files Browse the repository at this point in the history
  • Loading branch information
@cvicentiu
cvicentiu committed Dec 4, 2024
1 parent e4c87b8 commit 19e99e9
Showing 1 changed file with 98 additions and 134 deletions.
232 changes: 98 additions & 134 deletions master-nonlatent/master.cfg
View file
Original file line number Diff line number Diff line change
@@ -1,66 +1,53 @@
# -*- python -*-
# ex: set filetype=python:

from buildbot.plugins import *
from buildbot.process.properties import Property, Properties
from buildbot.steps.shell import ShellCommand, Compile, Test, SetPropertyFromCommand
from buildbot.steps.mtrlogobserver import MTR, MtrLogObserver
from buildbot.steps.source.github import GitHub
from buildbot.process.remotecommand import RemoteCommand
from datetime import timedelta
from twisted.internet import defer

import docker
import os
import sys

sys.setrecursionlimit(10000)
from datetime import timedelta

sys.path.insert(0, "/srv/buildbot/master")
from common_factories import *
from locks import *
from schedulers_definition import SCHEDULERS
from utils import *
from buildbot.plugins import steps, util, worker
from buildbot.process.properties import Property
from common_factories import addWinTests, getQuickBuildFactory
from constants import MTR_ENV, SAVED_PACKAGE_BRANCHES
from master_common import base_master_config
from utils import (
canStartBuild,
createVar,
getHTMLLogString,
getSourceTarball,
hasFailed,
ls2list,
moveMTRLogs,
mtrJobsMultiplier,
nextBuild,
prioritizeBuilders,
savePackage,
)

####### VARIABLES
vsWarningPattern = "^.*: warning C.*$"

# This is the dictionary that the buildmaster pays attention to. We also use
# a shorter alias to save typing.
c = BuildmasterConfig = {}

# Load the slave, database passwords and 3rd-party tokens from an external private file, so
# that the rest of the configuration can be public.
config = {"private": {}}
exec(open("master-private.cfg").read(), config, {})

####### PROJECT IDENTITY

# the 'title' string will appear at the top of this buildbot installation's
# home pages (linked to the 'titleURL').
c["title"] = os.getenv("TITLE", default="MariaDB CI")
c["titleURL"] = os.getenv("TITLE_URL", default="https://github.com/MariaDB/server")

# the 'buildbotURL' string should point to the location where the buildbot's
# internal web server is visible. This typically uses the port number set in
# the 'www' entry below, but with an externally-visible host name which the
# buildbot cannot figure out without some help.
c["buildbotURL"] = os.getenv("BUILDMASTER_URL", default="https://buildbot.mariadb.org/")

# 'protocols' contains information about protocols which master will use for
# communicating with workers. You must define at least 'port' option that workers
# could connect to your master with this protocol.
# 'port' must match the value configured into the workers (with their
# --master option)
port = int(os.getenv("PORT", default="9989"))
c["protocols"] = {"pb": {"port": port}}

####### DB URL
# This is the dictionary that the buildmaster pays attention to. We also use
# a shorter alias to save typing.
c = BuildmasterConfig = base_master_config(
title=os.getenv("TITLE", default="MariaDB CI"),
title_url=os.getenv("TITLE_URL", default="https://github.com/MariaDB/server"),
buildbot_url=os.getenv("BUILDMASTER_URL", default="https://buildbot.mariadb.org/"),
github_access_token=config["private"]["gh_mdbci"]["access_token"],
# TODO(cvicentiu) undo this hardcoding
secrets_provider_file=os.getenv(
"MASTER_CREDENTIALS_DIR",
default="/srv/buildbot/master/master-credential-provider",
),
master_port=int(os.getenv("PORT", default="9989")),
db_url=config["private"]["db_url"],
mq_router_url=os.getenv("MQ_ROUTER_URL", default="ws://localhost:8085/ws"),
)

c["db"] = {
# This specifies what database buildbot uses to store its state.
"db_url": config["private"]["db_url"]
}

mtrDbPool = util.EqConnectionPool(
"MySQLdb",
Expand All @@ -70,29 +57,10 @@ mtrDbPool = util.EqConnectionPool(
config["private"]["db_mtr_db"],
)

####### Disable net usage reports from being sent to buildbot.net
c["buildbotNetUsageData"] = None

####### Services
c["services"] = []
context = util.Interpolate("buildbot/%(prop:buildername)s")
gs = reporters.GitHubStatusPush(
token=config["private"]["gh_mdbci"]["access_token"],
context=context,
startDescription="Build started.",
endDescription="Build done.",
verbose=True,
builders=GITHUB_STATUS_BUILDERS,
)
c["services"].append(gs)
c['secretsProviders'] = [secrets.SecretInAFile(dirname=os.getenv("MASTER_CREDENTIALS_DIR", default="/srv/buildbot/master/master-credential-provider"))]
####### Builder priority
c["prioritizeBuilders"] = prioritizeBuilders

####### SCHEDULERS

# Configure the Schedulers, which decide how to react to incoming changes.
c["schedulers"] = SCHEDULERS

####### WORKERS

Expand Down Expand Up @@ -159,9 +127,6 @@ f_windows_env = {
}
f_windows_env.update(MTR_ENV)

# f_quick_build = getQuickBuildFactory(mtrDbPool)
f_rpm_autobake = getRpmAutobakeFactory(mtrDbPool)

## f_windows
f_windows = util.BuildFactory()
f_windows.addStep(
Expand Down Expand Up @@ -268,23 +233,22 @@ f_windows.addStep(
)

windows_tests = {
"nm": {
"create_scripts": True
},
"nm": {"create_scripts": True},
"connect": {
"suites": ["connect"],
},
}

for typ in windows_tests:
addWinTests(f_windows,
mtr_test_type=typ,
mtr_env=f_windows_env,
mtr_additional_args=util.Property("mtr_additional_args", default=""),
mtr_step_db_pool=mtrDbPool,
mtr_suites=windows_tests[typ].get('suites',["default"]),
create_scripts=windows_tests[typ].get('create_scripts',False)
)
addWinTests(
f_windows,
mtr_test_type=typ,
mtr_env=f_windows_env,
mtr_additional_args=util.Property("mtr_additional_args", default=""),
mtr_step_db_pool=mtrDbPool,
mtr_suites=windows_tests[typ].get("suites", ["default"]),
create_scripts=windows_tests[typ].get("create_scripts", False),
)


f_windows.addStep(
Expand All @@ -300,13 +264,13 @@ f_windows.addStep(
),
url=util.Interpolate(
f'{os.getenv("ARTIFACTS_URL", default="https://ci.mariadb.org")}'
'/'
"/"
"%(prop:tarbuildnum)s"
'/'
'logs'
'/'
'%(prop:buildername)s'
'/'
"/"
"logs"
"/"
"%(prop:buildername)s"
"/"
),
)
)
Expand All @@ -331,8 +295,16 @@ f_windows.addStep(
## f_windows_msi

f_windows_msi_env = {
"TMP": util.Interpolate("{0}\\%(prop:buildername)s\\build\\tmpdir".format('D:\\DEV\\Buildbot' if os.getenv('ENVIRON') == 'DEV' else 'D:\\Buildbot')),
"TEMP": util.Interpolate("{0}\\%(prop:buildername)s\\build\\tmpdir".format('D:\\DEV\\Buildbot' if os.getenv('ENVIRON') == 'DEV' else 'D:\\Buildbot')),
"TMP": util.Interpolate(
"{0}\\%(prop:buildername)s\\build\\tmpdir".format(
"D:\\DEV\\Buildbot" if os.getenv("ENVIRON") == "DEV" else "D:\\Buildbot"
)
),
"TEMP": util.Interpolate(
"{0}\\%(prop:buildername)s\\build\\tmpdir".format(
"D:\\DEV\\Buildbot" if os.getenv("ENVIRON") == "DEV" else "D:\\Buildbot"
)
),
}
f_windows_msi_env.update(MTR_ENV)

Expand Down Expand Up @@ -457,23 +429,22 @@ f_windows_msi.addStep(
)
)
windows_msi_tests = {
"nm": {
"create_scripts": True
},
"nm": {"create_scripts": True},
"connect": {
"suites": ["connect"],
},
}

for typ in windows_msi_tests:
addWinTests(f_windows_msi,
mtr_test_type=typ,
mtr_env=f_windows_msi_env,
mtr_additional_args=util.Property("mtr_additional_args", default=""),
mtr_step_db_pool=mtrDbPool,
mtr_suites=windows_msi_tests[typ].get('suites',["default"]),
create_scripts=windows_msi_tests[typ].get('create_scripts',False)
)
addWinTests(
f_windows_msi,
mtr_test_type=typ,
mtr_env=f_windows_msi_env,
mtr_additional_args=util.Property("mtr_additional_args", default=""),
mtr_step_db_pool=mtrDbPool,
mtr_suites=windows_msi_tests[typ].get("suites", ["default"]),
create_scripts=windows_msi_tests[typ].get("create_scripts", False),
)

f_windows_msi.addStep(
steps.DirectoryUpload(
Expand All @@ -488,13 +459,13 @@ f_windows_msi.addStep(
),
url=util.Interpolate(
f'{os.getenv("ARTIFACTS_URL", default="https://ci.mariadb.org")}'
'/'
"/"
"%(prop:tarbuildnum)s"
'/'
'logs'
'/'
'%(prop:buildername)s'
'/'
"/"
"logs"
"/"
"%(prop:buildername)s"
"/"
),
)
)
Expand Down Expand Up @@ -528,12 +499,11 @@ f_windows_msi.addStep(
mode=0o755,
url=util.Interpolate(
f'{os.getenv("ARTIFACTS_URL", default="https://ci.mariadb.org")}'
'/'
'%(prop:tarbuildnum)s'
'/'
'%(prop:buildername)s'
'/'

"/"
"%(prop:tarbuildnum)s"
"/"
"%(prop:buildername)s"
"/"
),
doStepIf=lambda step: savePackage(step, SAVED_PACKAGE_BRANCHES),
)
Expand All @@ -551,11 +521,11 @@ f_windows_msi.addStep(
mode=0o755,
url=util.Interpolate(
f'{os.getenv("ARTIFACTS_URL", default="https://ci.mariadb.org")}'
'/'
'%(prop:tarbuildnum)s'
'/'
'%(prop:buildername)s'
'/'
"/"
"%(prop:tarbuildnum)s"
"/"
"%(prop:buildername)s"
"/"
),
doStepIf=lambda step: savePackage(step, SAVED_PACKAGE_BRANCHES),
)
Expand Down Expand Up @@ -739,22 +709,26 @@ f_dockerlibrary.addStep(
f_dockerlibrary.addStep(
steps.SetPropertyFromCommand(
name="Extract the last tag created",
command=['bash','-c','if [ -f last_tag ]; then cat last_tag; else echo ""; fi'],
command=[
"bash",
"-c",
'if [ -f last_tag ]; then cat last_tag; else echo ""; fi',
],
property="lasttag",
)
)
f_dockerlibrary.addStep(
steps.SetPropertyFromCommand(
name="Determine sha for the last tag",
command=[
"bash",
"bash",
"-xc",
util.Interpolate(
'curl -s "https://quay.io/api/v1/repository/mariadb-foundation/mariadb-devel/tag/?filter_tag_name=eq:%(prop:lasttag)s&onlyActiveTags=true" | jq -r .tags[0].manifest_digest'
),
],
property="lastsha",
doStepIf=lambda step: (str(step.getProperty("lasttag")) != "")
doStepIf=lambda step: (str(step.getProperty("lasttag")) != ""),
)
)
f_dockerlibrary.addStep(
Expand All @@ -765,7 +739,9 @@ f_dockerlibrary.addStep(
"-xc",
"gh auth login --with-token < ~/gh_auth",
],
doStepIf=lambda step: (str(step.getProperty("lastsha")) != "null" and step.hasProperty("lastsha")),
doStepIf=lambda step: (
str(step.getProperty("lastsha")) != "null" and step.hasProperty("lastsha")
),
)
)

Expand Down Expand Up @@ -1132,24 +1108,12 @@ c["builders"].append(
"cxx_compiler": "clang++",
"mtr_env": {
"WSREP_PROVIDER": "/usr/local/lib/libgalera_smm.so",
}
},
},
nextBuild=nextBuild,
factory=f_freebsd,
)
)

# Add a Janitor configurator that removes old logs
c["configurators"] = [util.JanitorConfigurator(logHorizon=timedelta(weeks=6), hour=23)]

c["logEncoding"] = "utf-8"

c["multiMaster"] = True

c["mq"] = { # Need to enable multimaster aware mq. Wamp is the only option for now.
"type": "wamp",
"router_url": os.getenv("MQ_ROUTER_URL", default="ws://localhost:8085/ws"),
"realm": "realm1",
# valid are: none, critical, error, warn, info, debug, trace
"wamp_debug_level": "info",
}

0 comments on commit 19e99e9

Please sign in to comment.