Change abichecker generation to use gz-collection.yaml (#1038)

* Update abichecker to work with gz-*
* Rename abichecker bash to gz
* Implement gazebo_libs jobs for abichecker
* Return true from all ignition abicheckerq previous jobs
* Add excludes for abichecker

* Added a check for main branches
* Added a check for abichecker old jobs

----

Signed-off-by: Jose Luis Rivero <[email protected]>
Signed-off-by: Addisu Z. Taddese <[email protected]>

.github/workflows/ci.yaml

@@ -52,6 +52,33 @@ jobs:
           sudo chown -R ${USER} /var/lib/jenkins
           cd jenkins-scripts/dsl
           WRITE_JOB_LOG=1 java -jar tools/jobdsl.jar *.dsl
+      - name: Checks for abichecker jobs
+        if: steps.dsl_check.outputs.run_job == 'true'
+        run: |
+          cd jenkins-scripts/dsl
+          main=$(grep '<branch>main</branch>' *-abichecker-*.xml || true)
+          if [[ -n ${main} ]]; then
+            echo "Found a main branch in an abichecker job:"
+            echo "${main}"
+            echo "Main branches are not target of abichecking. Fix the code."
+            exit 1
+          fi
+          # Filter out the previous auto jobs
+          mkdir /tmp/abichecker_filtered_jobs
+          cp *-abichecker-*.xml /tmp/abichecker_filtered_jobs
+          cd /tmp/abichecker_filtered_jobs
+          rm *ubuntu_auto*.xml
+          repeated=$(grep '\<branch>' *-abichecker-*.xml | awk '{ print $2 }' | sort | uniq -d)
+          if [[ -n ${repeated} ]]; then
+            echo "Found a duplicate in an abichecker branch:"
+            echo "${repeated}"
+            echo "please exclude one of the versions in the yaml file to reduce the server workload"
+            exit 1
+          fi
+      - name: Export XML generated configuration for diff
+        if: steps.dsl_check.outputs.run_job == 'true'
+        run: |
+          cd jenkins-scripts/dsl
           # export files for later diff
           mkdir /tmp/pr_xml_configuration && mkdir /tmp/pr_log_generated/
           mv *.xml /tmp/pr_xml_configuration/

jenkins-scripts/docker/ignition-abichecker.bash → jenkins-scripts/docker/gz-abichecker.bash

@@ -14,13 +14,13 @@ if [[ -z ${DISTRO} ]]; then
   exit 1
 fi
 
-# In ignition this variable is passed directly from the DSL job creation
+# In Gazebo this variable is passed directly from the DSL job creation
 if [[ -z ${ABI_JOB_SOFTWARE_NAME} ]]; then
   echo "ABI_JOB_SOFTWARE_NAME not set!"
   exit 1
 fi
 
-# convert from ign-package to GZ_PACKAGE_DEPENDENCIES
+# convert from gz-package to GZ_PACKAGE_DEPENDENCIES
 GZ_NAME_PREFIX=$(\
   echo ${ABI_JOB_SOFTWARE_NAME} | tr '[:lower:]-' '[:upper:]_')
 ABI_JOB_PKG_DEPENDENCIES_VAR_NAME=${GZ_NAME_PREFIX}_DEPENDENCIES
@@ -42,7 +42,7 @@ export ${GZ_NAME_PREFIX}_MAJOR_VERSION=${GZ_NAME_PREFIX_MAJOR_VERSION}
 #
 # Set ABI_JOB_HEADER_PREFIX to match the last subfolder:
 #   gazebo6, math6, math7, sim7
-if [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-gazebo" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -ge 7 ]]
+if [[ "${ABI_JOB_SOFTWARE_NAME}" = "gz-sim" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -ge 7 ]]
 then
   # special case for gz-sim7+
   export ABI_JOB_HEADER_PREFIX=sim[0-9]*
@@ -51,29 +51,29 @@ else
   export ABI_JOB_HEADER_PREFIX=${ABI_JOB_SOFTWARE_NAME/[ignz]*-/}[0-9]*
 fi
 
-# check if NEED_C17_COMPILER should be set
+# check if NEED_C17_COMPILER should be set. Only needed for Bionic so only care about Citadel
+# versions
 if [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-gazebo" ]] || \
   [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-physics" ]] || \
   [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-sensors" ]] || \
-  [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-common"    && ${GZ_NAME_PREFIX_MAJOR_VERSION} -ge 3 ]] || \
-  [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-gui"       && ${GZ_NAME_PREFIX_MAJOR_VERSION} -ge 1 ]] || \
-  [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-math"      && ${GZ_NAME_PREFIX_MAJOR_VERSION} -ge 6 ]] || \
-  [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-msgs"      && ${GZ_NAME_PREFIX_MAJOR_VERSION} -ge 3 ]] || \
-  [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-plugin"    && ${GZ_NAME_PREFIX_MAJOR_VERSION} -ge 1 ]] || \
-  [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-rendering" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -ge 1 ]] || \
-  [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-transport" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -ge 6 ]]
+  [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-common" ]] || \
+  [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-gui" ]] || \
+  [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-math" ]] || \
+  [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-msgs" ]] || \
+  [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-plugin" ]] || \
+  [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-rendering" ]] || \
+  [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-transport" ]]
 then
   export NEED_C17_COMPILER=true
 fi
 
 # check if OGRE-2.2 include paths are needed
-if [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-gazebo"   && ${GZ_NAME_PREFIX_MAJOR_VERSION} -eq 6 ]] || \
-  [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-gui"       && ${GZ_NAME_PREFIX_MAJOR_VERSION} -eq 6 ]] || \
-  [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-launch"    && ${GZ_NAME_PREFIX_MAJOR_VERSION} -eq 5 ]] || \
-  [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-sensors"   && ${GZ_NAME_PREFIX_MAJOR_VERSION} -eq 6 ]] || \
-  [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-rendering" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -eq 6 ]]
+if [[ "${ABI_JOB_SOFTWARE_NAME}" = "gz-sim"      && ${GZ_NAME_PREFIX_MAJOR_VERSION} -eq 6 ]] || \
+  [[ "${ABI_JOB_SOFTWARE_NAME}" = "gz-gui"       && ${GZ_NAME_PREFIX_MAJOR_VERSION} -eq 6 ]] || \
+  [[ "${ABI_JOB_SOFTWARE_NAME}" = "gz-launch"    && ${GZ_NAME_PREFIX_MAJOR_VERSION} -eq 5 ]] || \
+  [[ "${ABI_JOB_SOFTWARE_NAME}" = "gz-sensors"   && ${GZ_NAME_PREFIX_MAJOR_VERSION} -eq 6 ]] || \
+  [[ "${ABI_JOB_SOFTWARE_NAME}" = "gz-rendering" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -eq 6 ]]
 then
-
   # OGRE 2.2 is packaged as "Ogre-Next" on jammy
   if [[ "${DISTRO}" == "jammy" ]]
   then
@@ -92,11 +92,11 @@ then
   fi
 fi
 
-if [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-gazebo"   && ${GZ_NAME_PREFIX_MAJOR_VERSION} -ge 6 ]] || \
-  [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-gui"       && ${GZ_NAME_PREFIX_MAJOR_VERSION} -ge 7 ]] || \
-  [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-launch"    && ${GZ_NAME_PREFIX_MAJOR_VERSION} -ge 6 ]] || \
-  [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-sensors"   && ${GZ_NAME_PREFIX_MAJOR_VERSION} -ge 7 ]] || \
-  [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-rendering" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -ge 7 ]]
+if [[ "${ABI_JOB_SOFTWARE_NAME}" = "gz-sim"      && ${GZ_NAME_PREFIX_MAJOR_VERSION} -ge 6 ]] || \
+  [[ "${ABI_JOB_SOFTWARE_NAME}" = "gz-gui"       && ${GZ_NAME_PREFIX_MAJOR_VERSION} -ge 7 ]] || \
+  [[ "${ABI_JOB_SOFTWARE_NAME}" = "gz-launch"    && ${GZ_NAME_PREFIX_MAJOR_VERSION} -ge 6 ]] || \
+  [[ "${ABI_JOB_SOFTWARE_NAME}" = "gz-sensors"   && ${GZ_NAME_PREFIX_MAJOR_VERSION} -ge 7 ]] || \
+  [[ "${ABI_JOB_SOFTWARE_NAME}" = "gz-rendering" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -ge 7 ]]
 then
   # -fPIC needed to compile Qt
   export ABI_JOB_EXTRA_GCC_OPTIONS="-fPIC"

jenkins-scripts/dsl/gazebo_libs.dsl

@@ -149,25 +149,25 @@ ciconf_per_lib_index.each { lib_name, lib_configs ->
     def ci_config = gz_collections_yaml.ci_configs.find{ it.name == config_name }
     def branches_with_collections = ci_configs.getValue()
     def branch_names = branches_with_collections.collect { it.branch }.unique()
-    if (ci_config.exclude.contains(lib_name))
+    if (ci_config.exclude.all?.contains(lib_name))
       return
     assert(lib_name)
     assert(branch_names)
     assert(ci_config)
 
-    if (ci_config.exclude.contains(lib_name))
-      return
-
 
     // Main PR jobs (-ci-pr_any-) (pulling check every 5 minutes)
     // --------------------------------------------------------------
     def distro = ci_config.system.version
     def arch = ci_config.system.arch
     def gz_job_name_prefix = lib_name.replaceAll('-','_')
-    def gz_ci_job_name = "${gz_job_name_prefix}-ci-pr_any-${distro}-${arch}"
-    def gz_ci_any_job = job(gz_ci_job_name)
     def pre_setup_script = ci_config.pre_setup_script_hook?.get(lib_name)?.join('\n')
     def extra_cmd = pre_setup_script ?: ""
+
+    // Main PR jobs (-ci-pr_any-) (pulling check every 5 minutes)
+    // --------------------------------------------------------------
+    def gz_ci_job_name = "${gz_job_name_prefix}-ci-pr_any-${distro}-${arch}"
+    def gz_ci_any_job = job(gz_ci_job_name)
     OSRFLinuxCompilationAnyGitHub.create(gz_ci_any_job,
                                          "gazebosim/${lib_name}",
                                          is_testing_enabled(lib_name, ci_config),
@@ -193,6 +193,37 @@ ciconf_per_lib_index.each { lib_name, lib_configs ->
       } // end of steps
     } // end of ci_any_job
 
+    if (! ci_config.exclude.abichecker?.contains(lib_name)) {
+      // ABI branch jobs (-ci-abichecker-) for non main branches
+      def abi_job_name = "${gz_job_name_prefix}-abichecker-any_to_any-ubuntu-${distro}-${arch}"
+      def abi_job = job(abi_job_name)
+      OSRFLinuxABIGitHub.create(abi_job)
+      GenericAnyJobGitHub.create(abi_job,
+                        "gazebosim/${lib_name}",
+                        branch_names - [ 'main'])
+      generate_label_by_requirements(abi_job, lib_name, ci_config.requirements)
+      abi_job.with
+      {
+        steps {
+          shell("""\
+                #!/bin/bash -xe
+
+                export DISTRO=${distro}
+
+                ${GLOBAL_SHELL_CMD}
+                ${extra_cmd}
+
+                export ARCH=${arch}
+                export DEST_BRANCH=\${DEST_BRANCH:-\$ghprbTargetBranch}
+                export SRC_BRANCH=\${SRC_BRANCH:-\$ghprbSourceBranch}
+                export SRC_REPO=\${SRC_REPO:-\$ghprbAuthorRepoGitUrl}
+                export ABI_JOB_SOFTWARE_NAME=${lib_name}
+                /bin/bash -xe ./scripts/jenkins-scripts/docker/gz-abichecker.bash
+                """.stripIndent())
+        } // end of steps
+      }  // end of with
+    }
+
     // CI branch jobs (-ci-$branch-) (pulling check every 5 minutes)
     branches_with_collections.each { branch_and_collection ->
       // TODO: remove after testing
@@ -212,7 +243,6 @@ ciconf_per_lib_index.each { lib_name, lib_configs ->
         [collection: branch_and_collection.collection,
          job_name: gz_ci_job.name])
     } // end_of_branch
-
   } //en of lib_configs
 } // end of lib
 

jenkins-scripts/dsl/gz-collections.yaml

@@ -487,7 +487,7 @@ collections:
       configs:
         - jammy
       exclude:
-        - __upcoming__
+          - __upcoming__
 ci_configs:
   - name: bionic
     system:
@@ -496,7 +496,11 @@ ci_configs:
       version: bionic
       arch: amd64
     exclude:
-      - ign-citadel
+      all:
+        - ign-citadel
+      abichecker:
+        - ign-cmake
+        - ign-tools
     requirements:
       large_memory:
         - ign-physics
@@ -524,8 +528,17 @@ ci_configs:
         - gz-rendering
         - gz-sensors
     exclude:
-      - gz-fortress
-      - gz-garden
+      all:
+        - gz-fortress
+        - gz-garden
+      abichecker:
+        - gz-cmake
+        - gz-tools
+        # These have a Jammy counterpart in Harmonic
+        - gz-common
+        - gz-math
+        - gz-plugin
+        - gz-utils
     pre_setup_script_hook:
       ign-physics:
         - "export MAKE_JOBS=1"
@@ -545,8 +558,12 @@ ci_configs:
         - gz-rendering
         - gz-sensors
     exclude:
-      - gz-harmonic
-      - __upcoming__
+      all:
+        - gz-harmonic
+        - __upcoming__
+      abichecker:
+        - gz-cmake
+        - gz-tools
     pre_setup_script_hook:
       gz-physics:
         - "export MAKE_JOBS=1"

jenkins-scripts/dsl/ignition.dsl

@@ -389,9 +389,9 @@ void generate_install_job(prefix, gz_sw, major_version, distro, arch)
 gz_software.each { gz_sw ->
   supported_arches.each { arch ->
     // 1 Per library and per linux arch
-    //   1.1 Per abi_distro
+    //   1.1 DEPRECATED Per abi_distro
     //     1.1.1 [job] ABI checker for main branches
-    //   1.2 Per ci_str_distro
+    //   1.2 DEPRECATED Per ci_str_distro
     //     1.2.1 [job] Main PR jobs (-ci-pr_any-)
     //   1.3 Per all supported_distros
     //     1.3.1 Per all supported branches on each library
@@ -412,39 +412,12 @@ gz_software.each { gz_sw ->
 
       abi_job_names[software_name] = "ignition_${software_name}-abichecker-any_to_any-ubuntu_auto-${arch}"
       def abi_job = job(abi_job_names[software_name])
-      checkout_subdir = "ign-${software_name}"
-      OSRFLinuxABIGitHub.create(abi_job)
       GenericAnyJobGitHub.create(abi_job,
                         "gazebosim/ign-${software_name}",
                         all_branches(software_name) - [ 'main'])
       abi_job.with
       {
-        extra_str=""
-        if (gz_sw == 'physics')
-        {
-          label Globals.nontest_label("large-memory")
-          // on ARM native nodes in buildfarm we need to restrict to 1 the
-          // compilation threads to avoid OOM killer
-          extra_str += '\nexport MAKE_JOBS=1'
-        }
-
-        steps {
-          shell("""\
-                #!/bin/bash -xe
-
-                export DISTRO=${distro}
-
-                ${GLOBAL_SHELL_CMD}
-                ${extra_str}
-
-                export ARCH=${arch}
-                export DEST_BRANCH=\${DEST_BRANCH:-\$ghprbTargetBranch}
-                export SRC_BRANCH=\${SRC_BRANCH:-\$ghprbSourceBranch}
-                export SRC_REPO=\${SRC_REPO:-\$ghprbAuthorRepoGitUrl}
-                export ABI_JOB_SOFTWARE_NAME=${checkout_subdir}
-                /bin/bash -xe ./scripts/jenkins-scripts/docker/ignition-abichecker.bash
-                """.stripIndent())
-        } // end of steps
+        description 'Automatic generated job by DSL jenkins. Stub job for migration, not doing any check'
       }  // end of with
     } // end of abi_distro
 
@@ -542,12 +515,12 @@ void generate_asan_ci_job(gz_ci_job, gz_sw, branch, distro, arch)
   generate_ci_job(gz_ci_job, gz_sw, branch, distro, arch,
                   '-DGZ_SANITIZER=Address',
                   Globals.MAKETEST_SKIP_GZ,
-                  'export ASAN_OPTIONS=check_initialization_order=true:strict_init_order=true')
+                  ['export ASAN_OPTIONS=check_initialization_order=true:strict_init_order=true'])
 }
 
 
 void generate_ci_job(gz_ci_job, gz_sw, branch, distro, arch,
-                     extra_cmake = '', extra_test = '', extra_cmd = '')
+                     extra_cmake = '', extra_test = '', extra_cmd = [])
 {
   OSRFLinuxCompilation.create(gz_ci_job, enable_testing(software_name))
   OSRFGitHub.create(gz_ci_job,
@@ -559,17 +532,16 @@ void generate_ci_job(gz_ci_job, gz_sw, branch, distro, arch,
   {
     if (gz_sw == 'physics') {
       label Globals.nontest_label("large-memory")
-      extra_str += '\nexport MAKE_JOBS=1'
+      extra_cmd += "export MAKE_JOBS=1"
     }
     if (gz_sw == 'gazebo')
       gz_sw = 'sim'
 
     steps {
-      shell("""\
-            #!/bin/bash -xe
+      shell("""#!/bin/bash -xe
 
             ${GLOBAL_SHELL_CMD}
-            ${extra_cmd}
+            ${extra_cmd.join('\n')}
             export BUILDING_EXTRA_CMAKE_PARAMS="${extra_cmake}"
             export BUILDING_EXTRA_MAKETEST_PARAMS="${extra_test}"
             export DISTRO=${distro}

jenkins-scripts/dsl/ignition_collection.dsl

@@ -491,7 +491,7 @@ gz_collections_yaml.collections.each { collection ->
     distro = ci_config.system.version
     arch = ci_config.system.arch
 
-    if (ci_config.exclude.contains(gz_collection_name))
+    if (ci_config.exclude.all?.contains(gz_collection_name))
       return
 
     // INSTALL JOBS: