Skip to content

Commit

Permalink
Move master-protected-branches to common function
Browse files Browse the repository at this point in the history
  • Loading branch information
@cvicentiu
cvicentiu committed Dec 4, 2024
1 parent ac723a0 commit 56ddb71
Showing 1 changed file with 73 additions and 90 deletions.
163 changes: 73 additions & 90 deletions master-protected-branches/master.cfg
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,85 +2,51 @@
# ex: set filetype=python:

import os
import sys
from collections import defaultdict
from datetime import timedelta

import docker
from buildbot.plugins import *
from buildbot.process.properties import Properties, Property
from buildbot.process.remotecommand import RemoteCommand
from buildbot.steps.mtrlogobserver import MTR, MtrLogObserver
from buildbot.steps.shell import Compile, SetPropertyFromCommand, ShellCommand, Test
from buildbot.steps.source.github import GitHub
from twisted.internet import defer

sys.setrecursionlimit(10000)

sys.path.insert(0, "/srv/buildbot/master")

from common_factories import *
from constants import *
from locks import *
from schedulers_definition import SCHEDULERS
from utils import *

# This is the dictionary that the buildmaster pays attention to. We also use
# a shorter alias to save typing.
c = BuildmasterConfig = {}
from buildbot.plugins import steps, util, worker
from buildbot.process.properties import Property
from common_factories import getLastNFailedBuildsFactory, getQuickBuildFactory
from locks import getLocks
from master_common import base_master_config
from utils import (
canStartBuild,
createWorker,
dockerfile,
isJepsenBranch,
isStagingBranch,
ls2list,
nextBuild,
waitIfStaging,
)

# Load the slave, database passwords and 3rd-party tokens from an external private file, so
# that the rest of the configuration can be public.
config = {"private": {}}
exec(open("master-private.cfg").read(), config, {})

c = BuildmasterConfig = base_master_config(
title=os.getenv("TITLE", default="MariaDB CI"),
title_url=os.getenv("TITLE_URL", default="https://github.com/MariaDB/server"),
buildbot_url=os.getenv("BUILDMASTER_URL", default="https://buildbot.mariadb.org/"),
github_access_token=config["private"]["gh_mdbci"]["access_token"],
# TODO(cvicentiu) undo this hardcoding
secrets_provider_file=os.getenv(
"MASTER_CREDENTIALS_DIR",
default="/srv/buildbot/master/master-credential-provider",
),
master_port=int(os.getenv("PORT", default=9994)),
db_url=config["private"]["db_url"],
mq_router_url=os.getenv("MQ_ROUTER_URL", default="ws://localhost:8085/ws"),
)

####### BUILDBOT SERVICES

# 'services' is a list of BuildbotService items like reporter targets. The
# status of each build will be pushed to these targets. buildbot/reporters/*.py
# has a variety to choose from, like IRC bots.


c["services"] = []
context = util.Interpolate("buildbot/%(prop:buildername)s")
gs = reporters.GitHubStatusPush(
token=config["private"]["gh_mdbci"]["access_token"],
context=context,
startDescription="Build started.",
endDescription="Build done.",
verbose=True,
builders=GITHUB_STATUS_BUILDERS,
)
c["services"].append(gs)
c['secretsProviders'] = [secrets.SecretInAFile(dirname=os.getenv("MASTER_CREDENTIALS_DIR", default="/srv/buildbot/master/master-credential-provider"))]
####### PROJECT IDENTITY

# the 'title' string will appear at the top of this buildbot installation's
# home pages (linked to the 'titleURL').
c["title"] = os.getenv("TITLE", default="MariaDB CI")
c["titleURL"] = os.getenv("TITLE_URL", default="https://github.com/MariaDB/server")

# the 'buildbotURL' string should point to the location where the buildbot's
# internal web server is visible. This typically uses the port number set in
# the 'www' entry below, but with an externally-visible host name which the
# buildbot cannot figure out without some help.
c["buildbotURL"] = os.getenv("BUILDMASTER_URL", default="https://buildbot.mariadb.org/")

# 'protocols' contains information about protocols which master will use for
# communicating with workers. You must define at least 'port' option that workers
# could connect to your master with this protocol.
# 'port' must match the value configured into the workers (with their
# --master option)
port = int(os.getenv("PORT", default="9994"))
c["protocols"] = {"pb": {"port": port}}

####### DB URL

c["db"] = {
# This specifies what database buildbot uses to store its state.
"db_url": config["private"]["db_url"]
}

mtrDbPool = util.EqConnectionPool(
"MySQLdb",
config["private"]["db_host"],
Expand All @@ -92,11 +58,6 @@ mtrDbPool = util.EqConnectionPool(
####### Disable net usage reports from being sent to buildbot.net
c["buildbotNetUsageData"] = None

####### SCHEDULERS

# Configure the Schedulers, which decide how to react to incoming changes.
c["schedulers"] = SCHEDULERS

####### WORKERS

# The 'workers' list defines the set of recognized workers. Each element is
Expand All @@ -116,7 +77,10 @@ c["workers"].append(
"hz-bbw1-docker-tarball-debian-10",
None,
docker_host=config["private"]["docker_workers"]["hz-bbw1-docker"],
image=os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + "debian10",
image=os.getenv(
"CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:"
)
+ "debian10",
followStartupLogs=False,
autopull=True,
alwaysPull=True,
Expand All @@ -134,7 +98,10 @@ c["workers"].append(
"hz-bbw4-docker-tarball-debian-10",
None,
docker_host=config["private"]["docker_workers"]["hz-bbw4-docker"],
image=os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + "debian10",
image=os.getenv(
"CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:"
)
+ "debian10",
followStartupLogs=False,
autopull=True,
alwaysPull=True,
Expand Down Expand Up @@ -180,39 +147,59 @@ for w_name in ["hz-bbw"]:
w_name,
i,
"debian-11-debug-ps-embed",
os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + "debian11",
os.getenv(
"CONTAINER_REGISTRY_URL",
default="quay.io/mariadb-foundation/bb-worker:",
)
+ "debian11",
jobs=14,
save_packages=False,
)
addWorker(
w_name,
i,
"debian-12",
os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + "debian12",
os.getenv(
"CONTAINER_REGISTRY_URL",
default="quay.io/mariadb-foundation/bb-worker:",
)
+ "debian12",
jobs=jobs,
save_packages=True,
)
addWorker(
w_name,
i,
"debian-12-debug-embed",
os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + "debian12",
os.getenv(
"CONTAINER_REGISTRY_URL",
default="quay.io/mariadb-foundation/bb-worker:",
)
+ "debian12",
jobs=14,
save_packages=False,
)
addWorker(
w_name,
i,
"fedora-40",
os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + "fedora40",
os.getenv(
"CONTAINER_REGISTRY_URL",
default="quay.io/mariadb-foundation/bb-worker:",
)
+ "fedora40",
jobs=jobs,
save_packages=True,
)
addWorker(
w_name,
i,
"last-N-failed",
os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + "rhel9",
os.getenv(
"CONTAINER_REGISTRY_URL",
default="quay.io/mariadb-foundation/bb-worker:",
)
+ "rhel9",
jobs=jobs,
save_packages=True,
)
Expand All @@ -228,15 +215,23 @@ for w_name in ["hz-bbw"]:
w_name,
i,
"ubuntu-2004-debug",
os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + "ubuntu20.04",
os.getenv(
"CONTAINER_REGISTRY_URL",
default="quay.io/mariadb-foundation/bb-worker:",
)
+ "ubuntu20.04",
jobs=14,
save_packages=True,
)
addWorker(
w_name,
i,
"ubuntu-2204-debug-ps",
os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + "ubuntu22.04",
os.getenv(
"CONTAINER_REGISTRY_URL",
default="quay.io/mariadb-foundation/bb-worker:",
)
+ "ubuntu22.04",
jobs=14,
save_packages=False,
)
Expand Down Expand Up @@ -560,15 +555,3 @@ c["builders"].append(
factory=getLastNFailedBuildsFactory("debug", mtrDbPool),
)
)

c["logEncoding"] = "utf-8"

c["multiMaster"] = True

c["mq"] = { # Need to enable multimaster aware mq. Wamp is the only option for now.
"type": "wamp",
"router_url": os.getenv("MQ_ROUTER_URL", default="ws://localhost:8085/ws"),
"realm": "realm1",
# valid are: none, critical, error, warn, info, debug, trace
"wamp_debug_level": "info",
}

0 comments on commit 56ddb71

Please sign in to comment.