From 39f8aee6733ea85685a4018a1f61d8efdafacbc7 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 20 Oct 2023 21:31:25 +1100 Subject: [PATCH] MDBF-588 Buildbot packages accessable in fixed repo URL per major version --- common_factories.py | 81 +++++++++++++++------------------------- master-galera/master.cfg | 81 ++++++++++++++++++---------------------- 2 files changed, 67 insertions(+), 95 deletions(-) diff --git a/common_factories.py b/common_factories.py index 6d48c1ad..02d4f258 100644 --- a/common_factories.py +++ b/common_factories.py @@ -1,6 +1,3 @@ -import os -from utils import * -from constants import * from buildbot.plugins import * from buildbot.process.properties import Property, Properties from buildbot.steps.package.rpm.rpmlint import RpmLint @@ -10,6 +7,9 @@ from buildbot.process.remotecommand import RemoteCommand from twisted.internet import defer +from utils import * +from constants import * + def getQuickBuildFactory(mtrDbPool): f_quick_build = util.BuildFactory() @@ -221,20 +221,12 @@ def getQuickBuildFactory(mtrDbPool): timeout=7200, haltOnFailure=True, command=util.Interpolate( - "mkdir -p " - + "/packages/" - + "%(prop:tarbuildnum)s" - + "/" - + "%(prop:buildername)s" - + " && sha256sum %(prop:mariadb_binary)s >> sha256sums.txt && cp " - + "%(prop:mariadb_binary)s sha256sums.txt" - + " /packages/" - + "%(prop:tarbuildnum)s" - + "/" - + "%(prop:buildername)s" - + "/" - + " && sync /packages/" - + "%(prop:tarbuildnum)s" + """ + mkdir -p /packages/%(prop:tarbuildnum)s/%(prop:buildername)s \ + && sha256sum %(prop:mariadb_binary)s >> sha256sums.txt \ + && cp %(prop:mariadb_binary)s sha256sums.txt /packages/%(prop:tarbuildnum)s/%(prop:buildername)s/ \ + && sync /packages/%(prop:tarbuildnum)s + """ ), doStepIf=savePackage, ) @@ -285,18 +277,17 @@ def getRpmAutobakeFactory(mtrDbPool): ) f_rpm_autobake.addStep( steps.ShellCommand( - name="Fetch packages for MariaDB-compat", - command=util.Interpolate( - "wget --no-check-certificate -cO ../MariaDB-shared-5.3.%(kw:arch)s.rpm " - + os.getenv("ARTIFACTS_URL", default="https://ci.mariadb.org") - + "/helper_files/mariadb-shared-5.3-%(kw:arch)s.rpm" - + "&&" - + "wget --no-check-certificate -cO ../MariaDB-shared-10.1.%(kw:arch)s.rpm " - + os.getenv("ARTIFACTS_URL", default="https://ci.mariadb.org") - + "/helper_files/mariadb-shared-10.1-kvm-rpm-%(kw:rpm_type)s-%(kw:arch)s.rpm", - arch=getArch, - rpm_type=util.Property("rpm_type"), - ), + name="fetch packages for MariaDB-compat", + command=[ + "sh", + "-c", + util.Interpolate( + 'wget --no-check-certificate -cO ../MariaDB-shared-5.3.%(kw:arch)s.rpm "%(kw:url)s/helper_files/mariadb-shared-5.3-%(kw:arch)s.rpm" && wget -cO ../MariaDB-shared-10.1.%(kw:arch)s.rpm "%(kw:url)s/helper_files/mariadb-shared-10.1-kvm-rpm-%(kw:rpm_type)s-%(kw:arch)s.rpm"', + arch=getArch, + url=os.getenv("ARTIFACTS_URL", default="https://ci.mariadb.org"), + rpm_type=util.Property("rpm_type"), + ), + ], doStepIf=hasCompat, ) ) @@ -381,15 +372,14 @@ def getRpmAutobakeFactory(mtrDbPool): cat << EOF > MariaDB.repo [MariaDB-%(prop:branch)s] name=MariaDB %(prop:branch)s repo (build %(prop:tarbuildnum)s) -baseurl=""" - + os.getenv("ARTIFACTS_URL", default="https://ci.mariadb.org") - + """/%(prop:tarbuildnum)s/%(prop:buildername)s/rpms +baseurl=%(kw:url)s/%(prop:tarbuildnum)s/%(prop:buildername)s/rpms gpgcheck=0 EOF if [ "%(prop:rpm_type)s" = rhel8 ] || [ "%(prop:rpm_type)s" = centosstream8 ]; then echo "module_hotfixes = 1" >> MariaDB.repo fi - """ + """, + url=os.getenv("ARTIFACTS_URL", default="https://ci.mariadb.org"), ), ] ) @@ -400,27 +390,18 @@ def getRpmAutobakeFactory(mtrDbPool): timeout=7200, haltOnFailure=True, command=util.Interpolate( - "mkdir -p " - + "/packages/" - + "%(prop:tarbuildnum)s" - + "/" - + "%(prop:buildername)s" - + " && cp -r MariaDB.repo rpms srpms /packages/" - + "%(prop:tarbuildnum)s" - + "/" - + "%(prop:buildername)s" - + "/" - + " && ln -sf %(prop:tarbuildnum)s/%(prop:buildername)s/MariaDB.repo %(prop:branch)s-latest-%(prop:buildername)s.repo " - + " && sync /packages/" - + "%(prop:tarbuildnum)s" + """" + mkdir -p /packages/%(prop:tarbuildnum)s/%(prop:buildername)s && + cp -r MariaDB.repo rpms srpms /packages/%(prop:tarbuildnum)s/%(prop:buildername)s/ && + ln -sf %(prop:tarbuildnum)s/%(prop:buildername)s/MariaDB.repo %(prop:branch)s-latest-%(prop:buildername)s.repo && + sync /packages/%(prop:tarbuildnum)s +""" ), doStepIf=lambda step: hasFiles(step) and savePackage(step), descriptionDone=util.Interpolate( """ -Repository available with: curl """ - + os.getenv("ARTIFACTS_URL", default="https://ci.mariadb.org") - + """/%(prop:tarbuildnum)s/%(prop:buildername)s/MariaDB.repo -o /etc/yum.repos.d/MariaDB.repo - """ +Repository available with: curl %(kw:url)s/%(prop:tarbuildnum)s/%(prop:buildername)s/MariaDB.repo -o /etc/yum.repos.d/MariaDB.repo""", + url=os.getenv("ARTIFACTS_URL", default="https://ci.mariadb.org"), ), ) ) diff --git a/master-galera/master.cfg b/master-galera/master.cfg index 656ccf29..f6410e5d 100644 --- a/master-galera/master.cfg +++ b/master-galera/master.cfg @@ -7,12 +7,11 @@ from buildbot.steps.shell import ShellCommand, Compile, Test, SetPropertyFromCom from buildbot.steps.mtrlogobserver import MTR, MtrLogObserver from buildbot.steps.source.github import GitHub from buildbot.process.remotecommand import RemoteCommand -from datetime import timedelta from twisted.internet import defer - -import docker -import os import sys +import os +import docker +from datetime import timedelta sys.setrecursionlimit(10000) @@ -169,12 +168,12 @@ for platform in all_platforms: for w_name in machines: for i in worker_ids: - for os_i in os_info: - if platform in os_info[os_i]["arch"]: + for os_str in os_info: + if platform in os_info[os_str]["arch"]: quay_name = "quay.io/mariadb-foundation/bb-worker:" + "".join( - os_i.split("-") + os_str.split("-") ) - os_name = os_i + os_name = os_str if "ubuntu" in quay_name: quay_name = quay_name[:-2] + "." + quay_name[-2:] if platform == "x86": @@ -270,24 +269,22 @@ f_deb_build.addStep( timeout=7200, haltOnFailure=True, command=util.Interpolate( - "mkdir -p " - + "/packages/" - + "%(prop:branch)s" - + "/" - + "%(prop:revision)s" - + "/" - + "%(prop:buildername)s" - + " && cp -r debs/ sha256sums.txt /packages/" - + "%(prop:branch)s" - + "/" - + "%(prop:revision)s" - + "/" - + "%(prop:buildername)s" - + "/" - + " && sync /packages/" - + "%(prop:branch)s" - + "/" - + "%(prop:revision)s" + """ + source /etc/os-release; \ + mkdir -p /packages/%(prop:branch)s/%(prop:revision)s/%(prop:buildername)s \ + && cp -r debs/ sha256sums.txt /packages/%(prop:branch)s/%(prop:revision)s/%(prop:buildername)s/ \ + && cat << EOF > /packages/%(prop:branch)s/%(prop:revision)s/%(prop:buildername)s/galera.sources \ +X-Repolib-Name: Galera +Types: deb +URIs: %(kw:url)s/%(prop:branch)s/%(prop:revision)s/%(prop:buildername)s +Suites: $VERSION_CODENAME +Components: main main/debug +Trusted: yes +EOF + && ln -s %(prop:branch)s/%(prop:revision)s/%(prop:buildername)s/galera.sources /packages/%(prop:branch)s-latest-%(prop:buildername)s.sources \ + && sync /packages/%(prop:branch)s/%(prop:revision)s +""", + url=os.getenv("ARTIFACTS_URL", default="https://ci.mariadb.org"), ), doStepIf=lambda step: savePackage(step, savedPackageBranches), ) @@ -326,25 +323,19 @@ f_rpm_build.addStep( timeout=7200, haltOnFailure=True, command=util.Interpolate( - "mkdir -p " - + "/packages/" - + "%(prop:branch)s" - + "/" - + "%(prop:revision)s" - + "/" - + "%(prop:buildername)s" - + " && cp -r rpms srpms sha256sums.txt" - + " /packages/" - + "%(prop:branch)s" - + "/" - + "%(prop:revision)s" - + "/" - + "%(prop:buildername)s" - + "/" - + " && sync /packages/" - + "%(prop:branch)s" - + "/" - + "%(prop:revision)s" + """ + mkdir -p /packages/%(prop:branch)s/%(prop:revision)s/%(prop:buildername)s \ + cat << EOF > packages/%(prop:branch)s/%(prop:revision)s/%(prop:buildername)s/galera.repo +[Galera-%(prop:branch)s] +name=Galera %(prop:branch)s repo (build %(prop:tarbuildnum)s) +baseurl=%(kw:url)s/%(prop:branch)s/%(prop:revision)s/%(prop:buildername)s/rpms +gpgcheck=0 +EOF + && cp -r rpms srpms sha256sums.txt /packages/%(prop:branch)s/%(prop:revision)s/%(prop:buildername)s/ \ + && ln -sf %(prop:branch)s/%(prop:revision)s/%(prop:buildername)s/galera.repo %(prop:branch)s-latest-%(prop:buildername)s.repo \ + && sync /packages/%(prop:branch)s/%(prop:revision)s +""", + url=os.getenv("ARTIFACTS_URL", default="https://ci.mariadb.org"), ), doStepIf=lambda step: savePackage(step, savedPackageBranches), )