From 42a19d4e6521fcb90560c1632bdbc23a845120b7 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 7 May 2024 18:56:09 +0200 Subject: [PATCH 1/3] Make release.py to automatically deal with ign- Citadel packages (#1109) * Make release.py to automatically deal with ign- Citadel packages Signed-off-by: Jose Luis Rivero * Use gz- in SOURCE_REPO_URI Signed-off-by: Jose Luis Rivero * Be sure of not using github.org Signed-off-by: Jose Luis Rivero * Do not enforce the PACKAGE name in OSRFSourceCreation Signed-off-by: Jose Luis Rivero * Add test to check SOURCE_REPO_URI Signed-off-by: Jose Luis Rivero * Disable safety check on package-name tarball Signed-off-by: Jose Luis Rivero * repository_uploader uses PACKAGE_ALIAS not PACKAGE Signed-off-by: Jose Luis Rivero * PACKAGE needs to be in OSRFSourceCreation Signed-off-by: Jose Luis Rivero * Fix dealing with ign-gazebo to be gz-sim Signed-off-by: Jose Luis Rivero --------- Signed-off-by: Jose Luis Rivero --- check_releasepy.bash | 29 +++++++++- .../dsl/_configs_/OSRFSourceCreation.groovy | 21 +++++--- jenkins-scripts/dsl/test.dsl | 3 +- release.py | 53 +++++++++---------- 4 files changed, 70 insertions(+), 36 deletions(-) diff --git a/check_releasepy.bash b/check_releasepy.bash index dbef90108..921153017 100755 --- a/check_releasepy.bash +++ b/check_releasepy.bash @@ -25,6 +25,16 @@ exec_ignition_releasepy_test() ign-foo 1.2.3 token ${test_params}"" } +exec_ignition_gazebo_releasepy_test() +{ + test_params=${1} + + ./release.py \ + --dry-run \ + --no-sanity-checks \ + ign-gazebo 1.2.3 token ${test_params}"" +} + expect_job_run() { output="${1}" job="${2}" @@ -66,10 +76,11 @@ expect_param() } -source_repo_uri_test=$(exec_releasepy_test "--source-repo-uri https://github.org/gazebosim/gz-foo") +source_repo_uri_test=$(exec_releasepy_test "--source-repo-uri https://github.com/gazebosim/gz-foo.git") expect_job_run "${source_repo_uri_test}" "gz-foo-source" expect_job_not_run "${source_repo_uri_test}" "gz-foo-debbuilder" expect_number_of_jobs "${source_repo_uri_test}" "1" +expect_param "${source_repo_uri_test}" "SOURCE_REPO_URI=https%3A%2F%2Fgithub.com%2Fgazebosim%2Fgz-foo.git" source_tarball_uri_test=$(exec_releasepy_test "--source-tarball-uri https://gazebosim/gz-foo-1.2.3.tar.gz") expect_job_run "${source_tarball_uri_test}" "gz-foo-debbuilder" @@ -92,10 +103,13 @@ expect_job_not_run "${bump_linux_test}" "gz-foo-source" expect_number_of_jobs "${bump_linux_test}" "6" expect_param "${bump_linux_test}" "RELEASE_VERSION=2" -ignition_test=$(exec_ignition_releasepy_test "--source-repo-uri https://github.org/gazebosim/gz-foo") +ignition_test=$(exec_ignition_releasepy_test "--source-repo-uri https://github.com/gazebosim/gz-foo.git") expect_job_run "${ignition_test}" "gz-foo-source" expect_job_not_run "${ignition_test}" "ignition-foo-source" expect_number_of_jobs "${ignition_test}" "1" +expect_param "${ignition_test}" "PACKAGE=ign-foo" +expect_param "${ignition_test}" "PACKAGE_ALIAS=ignition-foo" +expect_param "${ignition_test}" "SOURCE_REPO_URI=https%3A%2F%2Fgithub.com%2Fgazebosim%2Fgz-foo.git" ignition_source_tarball_uri_test=$(exec_ignition_releasepy_test "--source-tarball-uri https://gazebosim/gz-foo-1.2.3.tar.gz") expect_job_run "${ignition_source_tarball_uri_test}" "gz-foo-debbuilder" @@ -103,3 +117,14 @@ expect_job_run "${ignition_source_tarball_uri_test}" "generic-release-homebrew_p expect_job_not_run "${ignition_source_tarball_uri_test}" "gz-foo-source" expect_number_of_jobs "${ignition_source_tarball_uri_test}" "7" expect_param "${ignition_source_tarball_uri_test}" "SOURCE_TARBALL_URI=https%3A%2F%2Fgazebosim%2Fgz-foo-1.2.3.tar.gz" +expect_param "${ignition_source_tarball_uri_test}" "PACKAGE=ign-foo" +expect_param "${ignition_source_tarball_uri_test}" "PACKAGE_ALIAS=ignition-foo" + +ign_gazebo_source_tarball_uri_test=$(exec_ignition_gazebo_releasepy_test "--source-tarball-uri https://gazebosim/ign-gazebo-1.2.3.tar.gz") +expect_job_run "${ign_gazebo_source_tarball_uri_test}" "gz-sim-debbuilder" +expect_job_run "${ign_gazebo_source_tarball_uri_test}" "generic-release-homebrew_pull_request_updater" +expect_job_not_run "${ign_gazebo_source_tarball_uri_test}" "gz-sim-source" +expect_number_of_jobs "${ign_gazebo_source_tarball_uri_test}" "7" +expect_param "${ign_gazebo_source_tarball_uri_test}" "SOURCE_TARBALL_URI=https%3A%2F%2Fgazebosim%2Fign-gazebo-1.2.3.tar.gz" +expect_param "${ign_gazebo_source_tarball_uri_test}" "PACKAGE=ign-gazebo" +expect_param "${ign_gazebo_source_tarball_uri_test}" "PACKAGE_ALIAS=ignition-gazebo" diff --git a/jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy b/jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy index 4020a006f..2f8f490bb 100644 --- a/jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy +++ b/jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy @@ -33,9 +33,11 @@ class OSRFSourceCreation stringParam("DISTRO", default_params.find{ it.key == "DISTRO"}?.value, "Linux release inside LINUX_DISTRO to generate sources on") - choiceParam('PACKAGE', - [default_params.find{ it.key == "PACKAGE"}?.value], - "For downstream use: Package name (can not be modified)") + // Not using choiceParam here to support Citadel/Fortress ign-* packages not only + // gz-* packages + stringParam('PACKAGE', + default_params.find{ it.key == "PACKAGE"}?.value, + "For downstream use: Package name") stringParam("RELEASE_VERSION", default_params.find{ it.key == "RELEASE_VERSION"}?.value, "For downstream jobs: Packages release version") @@ -70,8 +72,6 @@ class OSRFSourceCreation priority 100 } - def canonical_package_name = Globals.get_canonical_package_name( - default_params.find{ it.key == "PACKAGE"}.value) def s3_download_url_basedir = Globals.s3_download_url_basedir( default_params.find{ it.key == "PACKAGE"}?.value) @@ -94,6 +94,14 @@ class OSRFSourceCreation /bin/bash -x ./scripts/jenkins-scripts/docker/gz-source-generation.bash """.stripIndent() ) + + // This can be enabled after the complete deprecation of ign- names + // that uses ignition aliases. + // def canonical_package_name = Globals.get_canonical_package_name( + // default_params.find{ it.key == "PACKAGE"}.value) + // + // then improve the find command below with: + // -name ${canonical_package_name}-\${VERSION}.tar.* \ shell("""\ #!/bin/bash -xe @@ -101,7 +109,7 @@ class OSRFSourceCreation # deal with changes in the compression of the tarballs. tarball=\$(find \${WORKSPACE}/${pkg_sources_dir} \ -type f \ - -name ${canonical_package_name}-\${VERSION}.tar.* \ + -name *-\${VERSION}.tar.* \ -printf "%f\\n") if [[ -z \${tarball} ]] || [[ \$(wc -w <<< \${tarball}) != 1 ]]; then # There is one use case that can be valid but failed to find canonical_package_name @@ -148,6 +156,7 @@ class OSRFSourceCreation parameters { currentBuild() predefinedProps([PROJECT_NAME_TO_COPY_ARTIFACTS: '${JOB_NAME}', + PACKAGE_ALIAS: '${PACKAGE_ALIAS}', S3_UPLOAD_PATH: "${Globals.s3_releases_dir(package_name)}/"]) // relative path with a final / propertiesFile(properties_file) // S3_FILES_TO_UPLOAD } diff --git a/jenkins-scripts/dsl/test.dsl b/jenkins-scripts/dsl/test.dsl index 56af6a360..0247ad212 100644 --- a/jenkins-scripts/dsl/test.dsl +++ b/jenkins-scripts/dsl/test.dsl @@ -30,6 +30,7 @@ releasepy_job.with { def gz_source_job = job("_test_gz_source") OSRFSourceCreation.create(gz_source_job, [ PACKAGE: "gz-plugin2", + PACKAGE_ALIAS: "gz-plugin2", SOURCE_REPO_URI: "https://github.com/gazebosim/gz-plugin.git", SOURCE_REPO_REF: "gz-plugin2"]) OSRFSourceCreation.call_uploader_and_releasepy(gz_source_job, @@ -50,7 +51,7 @@ repo_uploader.with parameters { - stringParam('PACKAGE','','Package name') + stringParam('PACKAGE_ALIAS','','Used by real repository_uploader') stringParam('S3_UPLOAD_PATH','', 'S3 path to upload') stringParam('S3_FILES_TO_UPLOAD','', 'S3 file names to upload') stringParam('UPLOAD_TO_REPO','none','repo to upload') diff --git a/release.py b/release.py index 97568ebe8..265796308 100755 --- a/release.py +++ b/release.py @@ -31,22 +31,6 @@ IGNORE_DRY_RUN = True -GARDEN_IGN_PACKAGES = ['ign-cmake3', - 'ign-common5', - 'ign-fuel-tools8', - 'ign-sim7', - 'ign-gui7', - 'ign-launch6', - 'ign-math7', - 'ign-msgs9', - 'ign-physics6', - 'ign-plugin2', - 'ign-rendering7', - 'ign-sensors7', - 'ign-tools2', - 'ign-transport12', - 'ign-utils2'] - class ErrorNoPermsRepo(Exception): pass @@ -85,6 +69,12 @@ def get_canonical_package_name(pkg_name): return pkg_name.rstrip('1234567890') +def replace_ignition_gz(pkg_name): + return pkg_name \ + .replace('ignition-gazebo', 'gz-sim') \ + .replace('ignition-', 'gz-') + + def github_repo_exists(url): try: check_call(['git', 'ls-remote', '-q', '--exit-code', url], IGNORE_DRY_RUN) @@ -164,10 +154,6 @@ def parse_args(argv): args = parser.parse_args() - if args.package in GARDEN_IGN_PACKAGES: - print(f"Garden packages start with gz- prefix, changing {args.package} to {args.package.replace('ign-','gz-')}",) - args.package = args.package.replace('ign-', 'gz-') - args.package_alias = args.package if args.package.startswith('ign-'): args.package_alias = args.package.replace('ign-', 'ignition-') @@ -310,6 +296,11 @@ def sanity_check_source_repo_uri(source_repo_uri): if not parsed_uri.scheme == "https" or \ not parsed_uri.path.endswith(".git"): error("--source-repo-uri parameter should start with https:// and end with .git") + # Needs to be fully in sync for SOURCE_REPO_URI values in + # OSRFSourceCreation + # https://github.com/gazebo-tooling/release-tools/blob/master/jenkins-scripts/dsl/gazebo_libs.dsl#L513 + if parsed_uri.netloc.startswith("github.org"): + error("--source-repo-uri needs github.com instead of github.org") def sanity_check_bump_linux(source_tarball_uri): @@ -445,12 +436,20 @@ def generate_source_repository_uri(args): git_remote = out.decode().split('\n')[0] if org_repo not in git_remote: - print(f""" !! Automatic calculation of source_repo_uri failed.\ - \n * git remote origin is: {git_remote}\ - \n * Package name generated org/repo: {org_repo}\ - \n >> Please use --source-repo-uri parameter""") - sys.exit(1) + # Handle the special case for citadel ignition repositories + if replace_ignition_gz(org_repo) not in git_remote: + print(f""" !! Automatic calculation of the source repository URI\ + failed with different information:\ + \n * git remote origin in the local direcotry is: {git_remote}\ + \n * Package name generated org/repo: {org_repo}\ + \n >> Please use --source-repo-uri parameter""") + sys.exit(1) + else: + org_repo = replace_ignition_gz(org_repo) + print(' ~ Ignition found in generated org/repo assuming gz repo: ' + + org_repo) + # Always use github.com return f"https://github.com/{org_repo}.git" # NOQA @@ -506,7 +505,7 @@ def display_help_job_chain_for_source_calls(args): f'{JENKINS_URL}/job/repository_uploader_packages/?{url_search_params}' rel_search_params = urllib.parse.urlencode( {'search': - f'{args.package_alias}/{args.version}-{args.release_version}'}) + f'{args.package}/{args.version}-{args.release_version}'}) releasepy_check_url = \ f'{JENKINS_URL}/job/_releasepy/?{rel_search_params}' print('\tINFO: After the source job finished, the release process will trigger:\n' @@ -523,7 +522,7 @@ def go(argv): if not args.release_version: args.release_version = 1 - package_alias_force_gz = args.package_alias.replace('ignition-','gz-') + package_alias_force_gz = replace_ignition_gz(args.package_alias) print(f"Downloading releasing info for {args.package}") # Sanity checks and dicover supported distributions before proceed. From 352318ef1f2f23b9986aa498be5bffef9759792e Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 8 May 2024 16:47:30 +0200 Subject: [PATCH 2/3] In Noble user amrhf standard images (#1135) Signed-off-by: Jose Luis Rivero --- .../docker/lib/docker_generate_dockerfile.bash | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash b/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash index 44499fe1f..fc3e7c9eb 100644 --- a/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash +++ b/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash @@ -116,12 +116,10 @@ case ${ARCH} in fi ;; 'armhf') - # There is no osrf/jammy_armhf image. Trying new - # platform support in docker - if [[ ${DISTRO} == 'jammy' ]]; then - FROM_VALUE=${LINUX_DISTRO}:${DISTRO} - else + if [[ ${DISTRO} == 'focal' ]]; then FROM_VALUE=osrf/${LINUX_DISTRO}_${ARCH}:${DISTRO} + else + FROM_VALUE=${LINUX_DISTRO}:${DISTRO} fi ;; 'arm64') From 602a1c0f2bcb75b1a227db44a9366b59783a135c Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 8 May 2024 17:58:16 +0200 Subject: [PATCH 3/3] Support noble for armhf (#1136) Signed-off-by: Jose Luis Rivero --- jenkins-scripts/docker/lib/docker_run.bash | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/jenkins-scripts/docker/lib/docker_run.bash b/jenkins-scripts/docker/lib/docker_run.bash index 5c96eaef5..f646c73eb 100644 --- a/jenkins-scripts/docker/lib/docker_run.bash +++ b/jenkins-scripts/docker/lib/docker_run.bash @@ -26,7 +26,9 @@ USER=$(whoami) # platform support starts on versions greater than 17.07 PLAFTORM_PARAM= DOCKER_CLI_PLUGIN= -if [[ ${DISTRO} == 'jammy' && ${ARCH} == 'armhf' ]]; then +if [[ ${LINUX_DISTRO} == 'ubuntu' && \ + ${DISTRO} != 'focal' && \ + ${ARCH} == 'armhf' ]]; then PLAFTORM_PARAM="--platform=linux/armhf" DOCKER_CLI_PLUGIN="buildx" fi