diff --git a/locks.py b/locks.py index c80239c8..9f916227 100644 --- a/locks.py +++ b/locks.py @@ -1,47 +1,25 @@ -from twisted.internet import defer +import os -from buildbot.plugins import * -from buildbot.process.properties import Properties, Property -from buildbot.process.remotecommand import RemoteCommand -from buildbot.steps.mtrlogobserver import MTR, MtrLogObserver -from buildbot.steps.shell import Compile, SetPropertyFromCommand, ShellCommand, Test -from buildbot.steps.source.github import GitHub -from constants import * +import yaml -####### LOCKS -# main_master_lock = util.MasterLock('main_master_lock', maxCount=30) +from buildbot.plugins import util -hz_bbw1_lock = util.MasterLock("hz_bbw1_lock", maxCount=9) -hz_bbw2_lock = util.MasterLock("hz_bbw2_lock", maxCount=1) -hz_bbw4_lock = util.MasterLock("hz_bbw4_lock", maxCount=9) -hz_bbw5_lock = util.MasterLock("hz_bbw5_lock", maxCount=9) -amd_bbw1_lock = util.MasterLock("amd_bbw1_lock", maxCount=4) -amd_bbw2_lock = util.MasterLock("amd_bbw2_lock", maxCount=6) -intel_bbw1_lock = util.MasterLock("intel_bbw1_lock", maxCount=5) -p9_rhel8_bbw1_lock = util.MasterLock("p9_rhel8_bbw1_lock", maxCount=6) -p9_db_bbw1_lock = util.MasterLock("p9_db_bbw1_lock", maxCount=8) -p9_raptor_bbw1_lock = util.MasterLock("p9_raptor_bbw1_lock", maxCount=6) -aarch_bbw1_lock = util.MasterLock("aarch64_bbw1_lock", maxCount=2) -aarch_bbw2_lock = util.MasterLock("aarch64_bbw2_lock", maxCount=2) -aarch_bbw3_lock = util.MasterLock("aarch64_bbw3_lock", maxCount=2) -aarch_bbw4_lock = util.MasterLock("aarch64_bbw4_lock", maxCount=2) -aarch_bbw5_lock = util.MasterLock("aarch64_bbw5_lock", maxCount=15) -aarch_bbw6_lock = util.MasterLock("aarch64_bbw6_lock", maxCount=15) -aarch_bbw7_lock = util.MasterLock("aarch64_bbw7_lock", maxCount=15) -apexis_bbw1_lock = util.MasterLock("apexis_bbw1_lock", maxCount=1) -apexis_bbw2_lock = util.MasterLock("apexis_bbw2_lock", maxCount=1) -apexis_bbw3_lock = util.MasterLock("apexis_bbw3_lock", maxCount=6) -bg_bbw1_lock = util.MasterLock("bg_bbw1_lock", maxCount=3) -bg_bbw2_lock = util.MasterLock("bg_bbw2_lock", maxCount=2) -bg_bbw3_lock = util.MasterLock("bg_bbw3_lock", maxCount=2) -bg_bbw4_lock = util.MasterLock("bg_bbw4_lock", maxCount=2) -win_bbw1_lock = util.MasterLock("win_bbw1_lock", maxCount=1) -win_bbw2_lock = util.MasterLock("win_bbw2_lock", maxCount=4) -s390x_bbw1_lock = util.MasterLock("s390x_bbw1_lock", maxCount=3) -s390x_bbw2_lock = util.MasterLock("s390x_bbw2_lock", maxCount=3) -s390x_bbw3_lock = util.MasterLock("s390x_bbw3_lock", maxCount=3) -s390x_bbw4_lock = util.MasterLock("s390x_bbw4_lock", maxCount=3) -s390x_bbw5_lock = util.MasterLock("s390x_bbw5_lock", maxCount=3) +# Local +from constants import builders_install, builders_upgrade, github_status_builders + +LOCKS: dict[str, util.MasterLock] = {} +# worker_locks.yaml currently is in the same folder as locks.py. +# TODO: re-evaluate if this is the right place after multi-master +# is refactored to use a single base master.cfg. +with open( + os.path.join(os.path.dirname(os.path.realpath(__file__)), "worker_locks.yaml"), + encoding="utf-8", +) as file: + locks = yaml.safe_load(file) + for worker_name in locks: + LOCKS[worker_name] = util.MasterLock( + f"{worker_name}_lock", maxCount=locks[worker_name] + ) @util.renderer @@ -57,71 +35,7 @@ def getLocks(props): or builder_name in builders_upgrade ): return [] - locks = [] - # else: - # locks = [main_master_lock.access('counting')] - - if "hz-bbw1-docker" in worker_name: - locks = locks + [hz_bbw1_lock.access("counting")] - if "hz-bbw2-docker" in worker_name: - locks = locks + [hz_bbw2_lock.access("counting")] - if "hz-bbw4-docker" in worker_name: - locks = locks + [hz_bbw4_lock.access("counting")] - if "hz-bbw5-docker" in worker_name: - locks = locks + [hz_bbw5_lock.access("counting")] - if "intel-bbw1-docker" in worker_name: - locks = locks + [intel_bbw1_lock.access("counting")] - if "ppc64le-rhel8-bbw1-docker" in worker_name: - locks = locks + [p9_rhel8_bbw1_lock.access("counting")] - if "ppc64le-db-bbw1-docker" in worker_name: - locks = locks + [p9_db_bbw1_lock.access("counting")] - if "ppc64le-raptor-bbw1-docker" in worker_name: - locks = locks + [p9_raptor_bbw1_lock.access("counting")] - if "aarch64-bbw1-docker" in worker_name: - locks = locks + [aarch_bbw1_lock.access("counting")] - if "aarch64-bbw2-docker" in worker_name: - locks = locks + [aarch_bbw2_lock.access("counting")] - if "aarch64-bbw3-docker" in worker_name: - locks = locks + [aarch_bbw3_lock.access("counting")] - if "aarch64-bbw4-docker" in worker_name: - locks = locks + [aarch_bbw4_lock.access("counting")] - if "aarch64-bbw5-docker" in worker_name: - locks = locks + [aarch_bbw5_lock.access("counting")] - if "aarch64-bbw6-docker" in worker_name: - locks = locks + [aarch_bbw6_lock.access("counting")] - if "aarch64-bbw7-docker" in worker_name: - locks = locks + [aarch_bbw7_lock.access("counting")] - if "fjord1-docker" in worker_name: - locks = locks + [apexis_bbw1_lock.access("counting")] - if "fjord2-docker" in worker_name: - locks = locks + [apexis_bbw2_lock.access("counting")] - if "ns-x64-bbw1-docker" in worker_name: - locks = locks + [bg_bbw1_lock.access("counting")] - if "ns-x64-bbw2-docker" in worker_name: - locks = locks + [bg_bbw2_lock.access("counting")] - if "ns-x64-bbw3-docker" in worker_name: - locks = locks + [bg_bbw3_lock.access("counting")] - if "ns-x64-bbw4-docker" in worker_name: - locks = locks + [bg_bbw4_lock.access("counting")] - if "bbw1-docker-windows" in worker_name: - locks = locks + [win_bbw1_lock.access("counting")] - if "bbw2-docker-windows" in worker_name: - locks = locks + [win_bbw2_lock.access("counting")] - if "s390x-bbw1-docker" in worker_name: - locks = locks + [s390x_bbw1_lock.access("counting")] - if "s390x-bbw2-docker" in worker_name: - locks = locks + [s390x_bbw2_lock.access("counting")] - if "s390x-bbw3-docker" in worker_name: - locks = locks + [s390x_bbw3_lock.access("counting")] - if "s390x-bbw4-docker" in worker_name: - locks = locks + [s390x_bbw4_lock.access("counting")] - if "s390x-bbw5-docker" in worker_name: - locks = locks + [s390x_bbw5_lock.access("counting")] - if "amd-bbw1-docker" in worker_name: - locks = locks + [amd_bbw1_lock.access("counting")] - if "amd-bbw2-docker" in worker_name: - locks = locks + [amd_bbw2_lock.access("counting")] - if "apexis-bbw3-docker" in worker_name: - locks = locks + [apexis_bbw3_lock.access("counting")] - return locks + if worker_name not in LOCKS: + return [] + return [LOCKS[worker_name].access("counting")] diff --git a/worker_locks.yaml b/worker_locks.yaml new file mode 100644 index 00000000..03554653 --- /dev/null +++ b/worker_locks.yaml @@ -0,0 +1,35 @@ +# This file keeps a maximum count of concurrent builds per worker. +# Later, this should be refactored to have this information as part of +# other worker metadata, such as IP, connection string, etc. +# The keys are worker names, as used in locks.py. +# TODO: There should be a single place where these keys are defined, currently +# there are more places. +hz-bbw1-docker: 9 +hz-bbw2-docker: 1 +hz-bbw4-docker: 9 +hz-bbw5-docker: 9 +amd-bbw1-docker: 4 +amd-bbw2-docker: 6 +intel-bbw1-docker: 5 +ppc64le-rhel8-docker: 6 +ppc64le-db-docker: 8 +ppc64le-raptor-docker: 6 +aarch64-bbw1-docker: 2 +aarch64-bbw2-docker: 2 +aarch64-bbw3-docker: 2 +aarch64-bbw4-docker: 2 +aarch64-bbw5-docker: 15 +aarch64-bbw6-docker: 15 +aarch64-bbw7-docker: 15 +apexis-bbw3-docker: 6 +ns-x64-bbw1-docker: 3 +ns-x64-bbw2-docker: 2 +ns-x64-bbw3-docker: 2 +ns-x64-bbw4-docker: 2 +bbw1-docker-windows: 1 +bbw2-docker-windows: 4 +s390x-bbw1-docker: 3 +s390x-bbw2-docker: 3 +s390x-bbw3-docker: 3 +s390x-bbw4-docker: 3 +s390x-bbw5-docker: 3