Merge branch 'master' into jrivero/cleanup_release.py

.github/workflows/ci.yaml

@@ -15,8 +15,9 @@ jobs:
       - name: Idenfify files changed in this PR
         id: files
         run: |
-            git diff --name-only origin/${{ github.base_ref }}...origin/${{ github.head_ref }}
-            echo "changed-files=$(git diff --name-only origin/${{ github.base_ref }}...origin/${{ github.head_ref }}| tr '\n' ' ')"  >> $GITHUB_OUTPUT
+            git fetch origin ${{ github.ref }}
+            git diff --name-only origin/${{ github.base_ref }}...FETCH_HEAD
+            echo "changed-files=$(git diff --name-only origin/${{ github.base_ref }}...FETCH_HEAD| tr '\n' ' ')"  >> $GITHUB_OUTPUT
       - name: Run testing on changed config files
         id: dsl_check
         run: |

bloom/ros_gz/Dockerfile

@@ -17,11 +17,11 @@ RUN apt-get update && apt-get install -y \
       wget \
     && rm -rf /var/lib/apt/lists/*
 
-# Install osrf-rosdep for Gazebo distributions not in ROS
+# Install osrf-rosdep for Gazebo distributions not in ROS.
+# Note that 01 is important to give priority to the replace list below
 RUN sudo bash -c \
-  'wget https://raw.githubusercontent.com/osrf/osrf-rosdep/master/gz/00-gazebo.list -O /etc/ros/rosdep/sources.list.d/00-gazebo.list'
-# Replace url
-# since it does not support arbitrary environment variables
+  'wget https://raw.githubusercontent.com/osrf/osrf-rosdep/master/gz/00-gazebo.list -O /etc/ros/rosdep/sources.list.d/01-gazebo.list'
+# Bloom does not support environment vars, use the trick or replacing one rosdep keys by others
 RUN sudo bash -c \
   "wget $URL_OSRF_ROSDEP_REPLACE -O /etc/ros/rosdep/sources.list.d/00-replace-gz.list"
 

bloom/ros_gz/README.md

@@ -27,6 +27,14 @@ Although using the officially supported version is the recommended way
 specially for non experienced users, some use cases might need to use a
 newer version of Gazebo than the one selected in REP-2000.
 
+### List of active relases
+
+| Gazebo Release | ROS / ROS 2 Release | status     | ros_gz branch | -release repository |
+| ---------------|---------------------|------------|---------------|---------------------|
+| Garden         | Humble              | stable     | humble        | https://github.com/gazebo-release/ros_ign-release |
+| Garden         | Iron                | prerelease | iron          | https://github.com/gazebo-release/ros_ign-release |
+| Harmonic       | Iron                | prerelease | ros2          | https://github.com/j-rivero/ros_ign-gzharmonic-release |
+
 ### Upstream versions released using this tutorial
 
 The `gbp -release repository` hosts the latest version released by the
@@ -35,6 +43,7 @@ version the version of `ros_gz` released will be the latest one existing in the
 official `gbp -release repository`. The version would be the same but the
 release number will start on 1000.
 
+
 ## 2. Initial setup
 
 To release a modified version of `ros_gz` which supports a different major
@@ -55,6 +64,19 @@ package name.
     - Usage: `$ rename-ros_gz-pkgs.bash <desired_gz_version> <space separted list of rosdistros to release>`
     - Example: `$ rename-ros_gz-pkgs.bash garden humble`
 
+The script supports to inject a custom RELEASE_REPO_URL that points to a bloom gbp
+repository different than https://github.com/gazebo-release/ros_ign-release.
+
+```
+i.e use a https://github.com/gazebo-testing/ros_ign-gzharmonic-release as gbp testing repository
+
+RELEASE_REPO_URL=https://github.com/gazebo-testing/ros_ign-gzharmonic-release \
+  ./bloom_from_special_env.bash \
+    humble \
+    garden \
+    https://raw.githubusercontent.com/osrf/osrf-rosdep/master/gz/replace_fortress_with_garden/00-replace-gz-fortress-with-garden.list
+```
+
 
 ### 2.2 Create a custom track in tracks.yml
 

bloom/ros_gz/rename-ros_gz-pkgs.bash

@@ -47,7 +47,9 @@ for pkg in ${PKGS}; do
     git commit debian/control.em debian/changelog.em -m "Patch name to release ${GZ_RELEASE} version"
     # Include conflict with initial package name in ROS
     sed -i -e '/^Depends/a\Conflicts: \@(Package)' debian/control.em
-
+    if [[ -n ${GZ_RELEASE_TO_CONFLICT} ]]; then
+      sed -i -e "s/Conflicts: @(Package)/Conflicts: @(Package.replace('-gz','-gz${GZ_RELEASE_TO_CONFLICT}'))/" debian/control.em
+    fi
     git commit debian/control.em -m "Set up a conflict with official ROS packages"
     git push origin "debian/$distro/$pkg"
   done

jenkins-scripts/docker/reprepro_updater.bash

@@ -40,13 +40,12 @@ git clone "${REPREPRO_GIT_REPO}" -b "${REPREPRO_GIT_BRANCH}" "${REPREPRO_REPO_PA
 echo '# END SECTION'
 
 export PYTHONPATH="${REPREPRO_REPO_PATH}/src"
-export GNUPGHOME=/var/lib/jenkins/.gnupg 
 
 echo '# BEGIN SECTION: run reprepro'
 cd "${REPREPRO_REPO_PATH}/scripts"
-sudo -E bash -c "PYTHONPATH=${REPREPRO_REPO_PATH}/src python3 import_upstream.py ${REPREPRO_PARAMS} \
-  ${UPLOAD_TO_REPO:-:_} \
-  ${REPREPRO_REPO_PATH}/config/packages.osrfoundation.org/${REPREPRO_IMPORT_YAML_FILE}"
+PYTHONPATH=${REPREPRO_REPO_PATH}/src python3 import_upstream.py ${REPREPRO_PARAMS} \
+  "${UPLOAD_TO_REPO:-:_}" \
+  "${REPREPRO_REPO_PATH}/config/packages.osrfoundation.org/${REPREPRO_IMPORT_YAML_FILE}"
 echo '# END SECTION'
 
 echo '#BEGIN: exit the venv'

jenkins-scripts/dsl/_configs_/Globals.groovy

@@ -131,4 +131,28 @@ class Globals
    static String nontest_label(String original_label) {
     return "(${original_label}) && !test-instance"
    }
+
+   static String get_canonical_package_name(String package_name) {
+    return package_name.replaceAll('\\d*$', '')
+  }
+
+   static String _s3_releases_dir(String package_name) {
+    return get_canonical_package_name(package_name) + '/releases'
+   }
+
+   static String _s3_build_tarball_name(String package_name, String version) {
+    // canonical_name + version
+    return package_name.replaceAll('\\d*$', '') + '-' + version
+   }
+
+   static String s3_upload_tarball_path(String package_name) {
+    return 's3://osrf-distributions/' + _s3_releases_dir(package_name)
+   }
+
+   // Not yet in use. Requires changing release.py
+   static String s3_download_uri(String package_name, String version) {
+    return 'https://osrf-distributions.s3.amazonaws.com/' + \
+            _s3_releases_dir(package_name) + \
+            _s3_build_tarball_name(package_name, version)
+   }
 }

jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy

@@ -5,43 +5,48 @@ import _configs_.Globals
 
 class OSRFSourceCreation
 {
+  static String properties_file = "package_name.prop"
+  static String package_name = ""
+
   static void addParameters(Job job, Map default_params = [:])
   {
+    package_name = default_params.find{ it.key == "PACKAGE"}?.value
+
     job.with
     {
       parameters {
-        stringParam("PACKAGE_NAME",
-                    default_params.find{ it.key == "PACKAGE_NAME"}?.value,
-                    "Software name (i.e gz-cmake3)")
-        stringParam("SOURCE_REPO_URI",
-                    default_params.find{ it.key == "SOURCE_REPO_URI"}?.value,
-                    "GitHub URI to release the sources from (i.e: https://github.com/gazebosim/gz-cmake.git)")
+        choiceParam('PACKAGE',
+                    [default_params.find{ it.key == "PACKAGE"}?.value],
+                    "Package name (can not be modified)")
+        choiceParam('SOURCE_REPO_URI',
+                    [default_params.find{ it.key == "SOURCE_REPO_URI"}?.value],
+                    "Software repository URL (can not be modified)")
         stringParam("VERSION",
                     default_params.find{ it.key == "VERSION"}?.value,
                     "Packages version to be built or nightly (enable nightly build mode)")
         stringParam("RELEASE_VERSION",
                     default_params.find{ it.key == "RELEASE_VERSION"}?.value,
-                    "Packages release version")
+                    "For downstream jobs: Packages release version")
         stringParam("RELEASE_REPO_BRANCH",
                     default_params.find{ it.key == "RELEASE_REPO_BRANCH"}?.value,
-                    "Branch from the -release repo to be used")
+                    "For downstream jobs: Branch from the -release repo to be used")
         stringParam("UPLOAD_TO_REPO",
                     default_params.find{ it.key == "UPLOAD_TO_REPO"}?.value,
-                    "OSRF repo name to upload the package to: stable | prerelease | nightly | none (for testing proposes)")
+                    "For downstream jobs: OSRF repo name to upload the package to: stable | prerelease | nightly | none (for testing proposes)")
       }
     }
   }
 
-  static void create(Job job, Map default_params = [:])
+  static void create(Job job, Map default_params = [:], Map default_hidden_params = [:])
   {
     OSRFLinuxBuildPkgBase.create(job)
     GenericRemoteToken.create(job)
     OSRFSourceCreation.addParameters(job, default_params)
 
+    def pkg_sources_dir="pkgs"
+
     job.with
     {
-      label Globals.nontest_label("docker")
-
       wrappers {
         preBuildCleanup()
       }
@@ -50,6 +55,9 @@ class OSRFSourceCreation
         priority 100
       }
 
+      def canonical_package_name = Globals.get_canonical_package_name(
+        default_params.find{ it.key == "PACKAGE"}.value)
+
       steps {
         systemGroovyCommand("""\
           build.setDescription(
@@ -62,14 +70,78 @@ class OSRFSourceCreation
           'RTOOLS_BRANCH: ' + build.buildVariableResolver.resolve('RTOOLS_BRANCH'));
           """.stripIndent()
         )
+        shell("""\
+          #!/bin/bash -xe
+
+          # Use Jammy/amd64 as base image to generate sources
+          export DISTRO=jammy
+          export ARCH=amd64
 
+          /bin/bash -x ./scripts/jenkins-scripts/docker/gz-source-generation.bash
+          """.stripIndent()
+        )
         shell("""\
-            #!/bin/bash -xe
-            export DISTRO=jammy
-            export ARCH=amd64
+          #!/bin/bash -xe
+
+          # Export information from the build in properties_files. The tarball extraction helps to
+          # deal with changes in the compression of the tarballs.
+          tarball=\$(find \${WORKSPACE}/${pkg_sources_dir} \
+                       -type f \
+                       -name ${canonical_package_name}-\${VERSION}.tar.* \
+                       -printf "%f\\n")
+          if [[ -z \${tarball} ]] || [[ \$(wc -w <<< \${tarball}) != 1 ]]; then
+            echo "Tarball name extraction returned \${tarball} which is not a one word string"
+            exit 1
+          fi
 
-            /bin/bash -x ./scripts/jenkins-scripts/docker/gz-source-generation.bash
-            """.stripIndent())
+          echo "TARBALL_NAME=\${tarball}" >> ${properties_file}
+          """.stripIndent()
+        )
+      }
+    }
+  }
+
+  // Useful to inject testing jobs
+  static void call_uploader_and_releasepy(Job job,
+                                          String repository_uploader_jobname,
+                                          String releasepy_jobname)
+  {
+    job.with
+    {
+      publishers {
+        postBuildScripts {
+          steps {
+            conditionalSteps {
+             condition {
+              not {
+                expression('none|None|^$','${ENV,var="UPLOAD_TO_REPO"}')
+                }
+              }
+              steps {
+                // Invoke repository_uploader
+                downstreamParameterized {
+                  trigger(repository_uploader_jobname) {
+                    parameters {
+                      currentBuild()
+                      predefinedProps([PROJECT_NAME_TO_COPY_ARTIFACTS: '${JOB_NAME}',
+                                       S3_UPLOAD_PATH: Globals.s3_upload_tarball_path(package_name)])
+                      propertiesFile(properties_file) // TARBALL_NAME
+                    }
+                  }
+                }
+                downstreamParameterized {
+                  trigger(releasepy_jobname) {
+                    parameters {
+                      currentBuild()
+                      predefinedProps([PROJECT_NAME_TO_COPY_ARTIFACTS: "\${JOB_NAME}"])
+                      propertiesFile(properties_file) // TARBALL_NAME
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }
       }
     }
   }

jenkins-scripts/dsl/brew_release.dsl

@@ -4,7 +4,7 @@ import javaposse.jobdsl.dsl.Job
 Globals.default_emails = "[email protected], [email protected]"
 
 // first distro in list is used as touchstone
-brew_supported_distros         = [ "bigsur", "monterey", "ventura" ]
+brew_supported_distros         = [ "monterey", "ventura" ]
 bottle_hash_updater_job_name   = 'generic-release-homebrew_pr_bottle_hash_updater'
 bottle_builder_job_name        = 'generic-release-homebrew_triggered_bottle_builder'
 directory_for_bottles          = 'pkgs'