MDBF-588 Buildbot packages accessable in fixed repo URL per major ver…

…sion

common_factories.py

@@ -1,6 +1,3 @@
-import os
-from utils import *
-from constants import *
 from buildbot.plugins import *
 from buildbot.process.properties import Property, Properties
 from buildbot.steps.package.rpm.rpmlint import RpmLint
@@ -10,6 +7,9 @@
 from buildbot.process.remotecommand import RemoteCommand
 from twisted.internet import defer
 
+from utils import *
+from constants import *
+
 
 def getQuickBuildFactory(mtrDbPool):
     f_quick_build = util.BuildFactory()
@@ -221,20 +221,12 @@ def getQuickBuildFactory(mtrDbPool):
             timeout=7200,
             haltOnFailure=True,
             command=util.Interpolate(
-                "mkdir -p "
-                + "/packages/"
-                + "%(prop:tarbuildnum)s"
-                + "/"
-                + "%(prop:buildername)s"
-                + " && sha256sum %(prop:mariadb_binary)s >> sha256sums.txt  && cp "
-                + "%(prop:mariadb_binary)s sha256sums.txt"
-                + " /packages/"
-                + "%(prop:tarbuildnum)s"
-                + "/"
-                + "%(prop:buildername)s"
-                + "/"
-                + " && sync /packages/"
-                + "%(prop:tarbuildnum)s"
+                """
+        mkdir -p /packages/%(prop:tarbuildnum)s/%(prop:buildername)s \
+        && sha256sum %(prop:mariadb_binary)s >> sha256sums.txt \
+        && cp %(prop:mariadb_binary)s sha256sums.txt /packages/%(prop:tarbuildnum)s/%(prop:buildername)s/ \
+        && sync /packages/%(prop:tarbuildnum)s
+        """
             ),
             doStepIf=savePackage,
         )
@@ -285,18 +277,17 @@ def getRpmAutobakeFactory(mtrDbPool):
     )
     f_rpm_autobake.addStep(
         steps.ShellCommand(
-            name="Fetch packages for MariaDB-compat",
-            command=util.Interpolate(
-                "wget --no-check-certificate -cO ../MariaDB-shared-5.3.%(kw:arch)s.rpm "
-                + os.getenv("ARTIFACTS_URL", default="https://ci.mariadb.org")
-                + "/helper_files/mariadb-shared-5.3-%(kw:arch)s.rpm"
-                + "&&"
-                + "wget --no-check-certificate -cO ../MariaDB-shared-10.1.%(kw:arch)s.rpm "
-                + os.getenv("ARTIFACTS_URL", default="https://ci.mariadb.org")
-                + "/helper_files/mariadb-shared-10.1-kvm-rpm-%(kw:rpm_type)s-%(kw:arch)s.rpm",
-                arch=getArch,
-                rpm_type=util.Property("rpm_type"),
-            ),
+            name="fetch packages for MariaDB-compat",
+            command=[
+                "sh",
+                "-c",
+                util.Interpolate(
+                    'wget --no-check-certificate -cO ../MariaDB-shared-5.3.%(kw:arch)s.rpm "%(kw:url)s/helper_files/mariadb-shared-5.3-%(kw:arch)s.rpm" && wget -cO ../MariaDB-shared-10.1.%(kw:arch)s.rpm "%(kw:url)s/helper_files/mariadb-shared-10.1-kvm-rpm-%(kw:rpm_type)s-%(kw:arch)s.rpm"',
+                    arch=getArch,
+                    url=os.getenv("ARTIFACTS_URL", default="https://ci.mariadb.org"),
+                    rpm_type=util.Property("rpm_type"),
+                ),
+            ],
             doStepIf=hasCompat,
         )
     )
@@ -381,15 +372,14 @@ def getRpmAutobakeFactory(mtrDbPool):
             cat << EOF > MariaDB.repo
 [MariaDB-%(prop:branch)s]
 name=MariaDB %(prop:branch)s repo (build %(prop:tarbuildnum)s)
-baseurl="""
-                    + os.getenv("ARTIFACTS_URL", default="https://ci.mariadb.org")
-                    + """/%(prop:tarbuildnum)s/%(prop:buildername)s/rpms
+baseurl=%(kw:url)s/%(prop:tarbuildnum)s/%(prop:buildername)s/rpms
 gpgcheck=0
 EOF
             if [ "%(prop:rpm_type)s" = rhel8 ] || [ "%(prop:rpm_type)s" = centosstream8 ]; then
                 echo "module_hotfixes = 1" >> MariaDB.repo
             fi
-        """
+        """,
+                    url=os.getenv("ARTIFACTS_URL", default="https://ci.mariadb.org"),
                 ),
             ]
         )
@@ -400,27 +390,18 @@ def getRpmAutobakeFactory(mtrDbPool):
             timeout=7200,
             haltOnFailure=True,
             command=util.Interpolate(
-                "mkdir -p "
-                + "/packages/"
-                + "%(prop:tarbuildnum)s"
-                + "/"
-                + "%(prop:buildername)s"
-                + " && cp -r MariaDB.repo rpms srpms /packages/"
-                + "%(prop:tarbuildnum)s"
-                + "/"
-                + "%(prop:buildername)s"
-                + "/"
-                + " && ln -sf %(prop:tarbuildnum)s/%(prop:buildername)s/MariaDB.repo %(prop:branch)s-latest-%(prop:buildername)s.repo "
-                + " && sync /packages/"
-                + "%(prop:tarbuildnum)s"
+                """"
+                mkdir -p /packages/%(prop:tarbuildnum)s/%(prop:buildername)s &&
+                cp -r MariaDB.repo rpms srpms /packages/%(prop:tarbuildnum)s/%(prop:buildername)s/ &&
+                ln -sf %(prop:tarbuildnum)s/%(prop:buildername)s/MariaDB.repo %(prop:branch)s-latest-%(prop:buildername)s.repo &&
+                sync /packages/%(prop:tarbuildnum)s
+"""
             ),
             doStepIf=lambda step: hasFiles(step) and savePackage(step),
             descriptionDone=util.Interpolate(
                 """
-Repository available with: curl """
-                + os.getenv("ARTIFACTS_URL", default="https://ci.mariadb.org")
-                + """/%(prop:tarbuildnum)s/%(prop:buildername)s/MariaDB.repo -o /etc/yum.repos.d/MariaDB.repo
-        """
+Repository available with: curl %(kw:url)s/%(prop:tarbuildnum)s/%(prop:buildername)s/MariaDB.repo -o /etc/yum.repos.d/MariaDB.repo""",
+                url=os.getenv("ARTIFACTS_URL", default="https://ci.mariadb.org"),
             ),
         )
     )

master-galera/master.cfg

@@ -7,12 +7,11 @@ from buildbot.steps.shell import ShellCommand, Compile, Test, SetPropertyFromCom
 from buildbot.steps.mtrlogobserver import MTR, MtrLogObserver
 from buildbot.steps.source.github import GitHub
 from buildbot.process.remotecommand import RemoteCommand
-from datetime import timedelta
 from twisted.internet import defer
-
-import docker
-import os
 import sys
+import os
+import docker
+from datetime import timedelta
 
 sys.setrecursionlimit(10000)
 
@@ -169,12 +168,12 @@ for platform in all_platforms:
 
     for w_name in machines:
         for i in worker_ids:
-            for os_i in os_info:
-                if platform in os_info[os_i]["arch"]:
+            for os_str in os_info:
+                if platform in os_info[os_str]["arch"]:
                     quay_name = "quay.io/mariadb-foundation/bb-worker:" + "".join(
-                        os_i.split("-")
+                        os_str.split("-")
                     )
-                    os_name = os_i
+                    os_name = os_str
                     if "ubuntu" in quay_name:
                         quay_name = quay_name[:-2] + "." + quay_name[-2:]
                     if platform == "x86":
@@ -270,24 +269,22 @@ f_deb_build.addStep(
         timeout=7200,
         haltOnFailure=True,
         command=util.Interpolate(
-            "mkdir -p "
-            + "/packages/"
-            + "%(prop:branch)s"
-            + "/"
-            + "%(prop:revision)s"
-            + "/"
-            + "%(prop:buildername)s"
-            + " && cp -r debs/ sha256sums.txt /packages/"
-            + "%(prop:branch)s"
-            + "/"
-            + "%(prop:revision)s"
-            + "/"
-            + "%(prop:buildername)s"
-            + "/"
-            + " && sync /packages/"
-            + "%(prop:branch)s"
-            + "/"
-            + "%(prop:revision)s"
+            """
+        source /etc/os-release; \
+        mkdir -p /packages/%(prop:branch)s/%(prop:revision)s/%(prop:buildername)s \
+        && cp -r debs/ sha256sums.txt /packages/%(prop:branch)s/%(prop:revision)s/%(prop:buildername)s/ \
+        && cat << EOF > /packages/%(prop:branch)s/%(prop:revision)s/%(prop:buildername)s/galera.sources \
+X-Repolib-Name: Galera
+Types: deb
+URIs: %(kw:url)s/%(prop:branch)s/%(prop:revision)s/%(prop:buildername)s
+Suites: $VERSION_CODENAME
+Components: main main/debug
+Trusted: yes
+EOF
+        && ln -s %(prop:branch)s/%(prop:revision)s/%(prop:buildername)s/galera.sources /packages/%(prop:branch)s-latest-%(prop:buildername)s.sources \
+        && sync /packages/%(prop:branch)s/%(prop:revision)s
+""",
+            url=os.getenv("ARTIFACTS_URL", default="https://ci.mariadb.org"),
         ),
         doStepIf=lambda step: savePackage(step, savedPackageBranches),
     )
@@ -326,25 +323,19 @@ f_rpm_build.addStep(
         timeout=7200,
         haltOnFailure=True,
         command=util.Interpolate(
-            "mkdir -p "
-            + "/packages/"
-            + "%(prop:branch)s"
-            + "/"
-            + "%(prop:revision)s"
-            + "/"
-            + "%(prop:buildername)s"
-            + " && cp -r rpms srpms sha256sums.txt"
-            + " /packages/"
-            + "%(prop:branch)s"
-            + "/"
-            + "%(prop:revision)s"
-            + "/"
-            + "%(prop:buildername)s"
-            + "/"
-            + " && sync /packages/"
-            + "%(prop:branch)s"
-            + "/"
-            + "%(prop:revision)s"
+            """
+        mkdir -p /packages/%(prop:branch)s/%(prop:revision)s/%(prop:buildername)s \
+        cat << EOF > packages/%(prop:branch)s/%(prop:revision)s/%(prop:buildername)s/galera.repo
+[Galera-%(prop:branch)s]
+name=Galera %(prop:branch)s repo (build %(prop:tarbuildnum)s)
+baseurl=%(kw:url)s/%(prop:branch)s/%(prop:revision)s/%(prop:buildername)s/rpms
+gpgcheck=0
+EOF
+        && cp -r rpms srpms sha256sums.txt /packages/%(prop:branch)s/%(prop:revision)s/%(prop:buildername)s/ \
+        && ln -sf %(prop:branch)s/%(prop:revision)s/%(prop:buildername)s/galera.repo %(prop:branch)s-latest-%(prop:buildername)s.repo \
+        && sync /packages/%(prop:branch)s/%(prop:revision)s
+""",
+            url=os.getenv("ARTIFACTS_URL", default="https://ci.mariadb.org"),
         ),
         doStepIf=lambda step: savePackage(step, savedPackageBranches),
     )