From f392d30813b5097229b22413f56a6733556d34f4 Mon Sep 17 00:00:00 2001 From: "Addisu Z. Taddese" Date: Fri, 5 Apr 2024 13:52:11 -0500 Subject: [PATCH 01/13] brew: pip install protobuf in CI (#1127) Add protobuf from pip since the homebrew bottle dropped support for python bindings Signed-off-by: Addisu Z. Taddese --- jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash | 3 +++ 1 file changed, 3 insertions(+) diff --git a/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash b/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash index 0e01b32bc..91817a4d2 100644 --- a/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash +++ b/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash @@ -78,6 +78,9 @@ brew install $(brew deps --1 --include-build ${PROJECT_FORMULA}) # pytest is needed to run python tests with junit xml output PIP_PACKAGES_NEEDED="${PIP_PACKAGES_NEEDED} pytest" +# Add protobuf since the homebrew protobuf bottle dropped support for python bindings +PIP_PACKAGES_NEEDED="${PIP_PACKAGES_NEEDED} protobuf" + if [[ "${RERUN_FAILED_TESTS}" -gt 0 ]]; then # Install lxml for flaky_junit_merge.py PIP_PACKAGES_NEEDED="${PIP_PACKAGES_NEEDED} lxml" From 83fc94066d68a2511ede1cddf3f86d241561941e Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 10 Apr 2024 16:07:13 +0200 Subject: [PATCH 02/13] Update and fix gzdev jobs in Jenkins (#1128) * Modernize DSL distributions * Update and fix the docker-in-docker support until the point of being able to run rocker * Remove old docker install code --------- Signed-off-by: Jose Luis Rivero --- .../docker/lib/_install_nvidia_docker.sh | 22 +------- .../lib/docker_generate_dockerfile.bash | 12 +++++ .../docker/lib/gzdev-base-linux.bash | 50 +++++++++---------- jenkins-scripts/dsl/gzdev.dsl | 5 +- 4 files changed, 42 insertions(+), 47 deletions(-) diff --git a/jenkins-scripts/docker/lib/_install_nvidia_docker.sh b/jenkins-scripts/docker/lib/_install_nvidia_docker.sh index 8e766acbf..517665ceb 100644 --- a/jenkins-scripts/docker/lib/_install_nvidia_docker.sh +++ b/jenkins-scripts/docker/lib/_install_nvidia_docker.sh @@ -1,22 +1,7 @@ -INSTALL_NVIDIA_DOCKER1=""" -echo '# BEGIN SECTION: install docker (in docker)' -curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - -sudo add-apt-repository \"deb [arch=amd64] https://download.docker.com/linux/ubuntu \$(lsb_release -cs) stable\" -sudo apt-get update -sudo apt-get install -y docker-ce -echo '# END SECTION' - -echo '# BEGIN SECTION: install nvidia-docker1 (in docker)' -sudo apt-get install -y wget nvidia-340 nvidia-modprobe -wget -P /tmp https://github.com/NVIDIA/nvidia-docker/releases/download/v1.0.1/nvidia-docker_1.0.1-1_amd64.deb -sudo dpkg -i /tmp/nvidia-docker*.deb && rm /tmp/nvidia-docker*.deb -echo '# END SECTION' -""" - INSTALL_NVIDIA_DOCKER2=""" echo '# BEGIN SECTION: install docker (in docker)' curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - -sudo add-apt-repository \"deb [arch=amd64] https://download.docker.com/linux/ubuntu \$(lsb_release -cs) stable\" +sudo add-apt-repository -y \"deb [arch=amd64] https://download.docker.com/linux/ubuntu \$(lsb_release -cs) stable\" sudo apt-get update # sudo apt-get install -y docker-ce echo '# END SECTION' @@ -29,11 +14,8 @@ curl -s -L https://nvidia.github.io/nvidia-docker/\$distribution/nvidia-docker.l sudo tee /etc/apt/sources.list.d/nvidia-docker.list sudo apt-get update sudo apt-get install -y nvidia-docker2 -# systemctl daemon-reload -# systemctl restart docker +sudo docker info echo '# END SECTION' """ -DOCKER2_CMD="sudo docker run --privileged -e DISPLAY=unix:0 -v /sys:/sys:ro -v /var/run/docker.sock:/var/run/docker.sock -v /tmp/.X11-unix:/tmp/.X11-unix:rw --runtime=nvidia -e DOCKER_FIX= -v /dev/log:/dev/log:ro -v /run/log:/run/log:ro -v /sys/fs/cgroup:/sys/fs/cgroup:ro --device /dev/snd --tty --rm" - DEPENDENCY_PKGS="${DEPENDENCY_PKGS} curl software-properties-common" diff --git a/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash b/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash index e7f66c419..b78f8f432 100644 --- a/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash +++ b/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash @@ -413,6 +413,13 @@ DELIM_DISPLAY fi fi +if ${USE_DOCKER_IN_DOCKER}; then +cat >> Dockerfile << DELIM_WORKAROUND_DOCKER_IN_DOCKER_HOOK +# Avoid ERROR: invoke-rc.d: policy-rc.d denied execution of start. +RUN sed -i "s/^exit 101$/exit 0/" /usr/sbin/policy-rc.d +DELIM_WORKAROUND_DOCKER_IN_DOCKER_HOOK +fi + if [ `expr length "${DOCKER_POSTINSTALL_HOOK}"` -gt 1 ]; then cat >> Dockerfile << DELIM_WORKAROUND_POST_HOOK RUN ${DOCKER_POSTINSTALL_HOOK} @@ -468,11 +475,16 @@ RUN adduser --uid \$USERID --gid \$GID --gecos "Developer" --disabled-password \ RUN adduser \$USER sudo RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers RUN chown -R \$USER:\$USER /home/\$USER +# Needed if USE_DOCKER_IN_DOCKER is active. Harmless to be here +RUN groupadd docker +RUN gpasswd -a \$USER docker +RUN newgrp docker # permit access to USER variable inside docker ENV USER \$USER USER \$USER # Must use sudo where necessary from this point on + DELIM_DOCKER_USER if [[ -n ${SOFTWARE_DIR} ]]; then diff --git a/jenkins-scripts/docker/lib/gzdev-base-linux.bash b/jenkins-scripts/docker/lib/gzdev-base-linux.bash index 9acbe12f8..7deb198a9 100644 --- a/jenkins-scripts/docker/lib/gzdev-base-linux.bash +++ b/jenkins-scripts/docker/lib/gzdev-base-linux.bash @@ -13,49 +13,49 @@ set -ex export MAKE_JOBS=${MAKE_JOBS} export DISPLAY=${DISPLAY} -${INSTALL_NVIDIA_DOCKER1} +${INSTALL_NVIDIA_DOCKER2} echo '# BEGIN SECTION: install pip requirements' cd ${WORKSPACE}/gzdev -pip3 install -r requirements.txt +# pip3 install -r requirements.txt echo '# END SECTION' -echo '# BEGIN SECTION: run gzdev for gazebo8 with nvidia' -cd ${WORKSPACE}/gzdev -./gzdev.py spawn --gzv=8 --nvidia -echo '# END SECTION' +echo '# BEGIN SECTION: smoke tests for ign-docker-env' +sudo pip3 install git+https://github.com/adlarkin/ign-rocker.git -echo '# BEGIN SECTION: Disply log file.' -cat ./gz8.log -echo '# END SECTION' +# TODO: rocker can not play well docker-in-docker installations +# out of the box. Needs more work. + +# xvfb :1 -ac -noreset -core -screen 0 1280x1024x24 & +# export display=:1.0 +# export mesa_gl_version_override=3.3 + +# test_timeout=300 +# test_start=\`date +%s\` +# sudo bash -c "timeout --preserve-status \$test_timeout ./gzdev.py ign-docker-env dome --linux-distro ubuntu:bionic" +# test_end=\`date +%s\` +# diff=\$(expr \$test_end - \$test_start) -echo '# BEGIN SECTION: check that gazebo is running' -gazebo_detection=false -seconds_waiting=0 -cat gz8.log | grep "Connected to gazebo master" && gazebo_detection=true -while (! \$gazebo_detection); do - sleep 1 - docker top gz8 | grep gazebo && gazebo_detection=true - docker top gz8 | grep gzserver && gazebo_detection=true - seconds_waiting=\$((seconds_waiting+1)) - [ \$seconds_waiting -gt 30 ] && break -done -# clean up gazebo instances -docker rm -f gz8 || true -killall -9 gazebo gzserver gzclient || true -! \${gazebo_detection} && exit 1 +# if [ \$diff -lt \$test_timeout ]; then +# echo 'the test took less than \$test_timeout. something bad happened' +# exit 1 +# fi echo '# END SECTION' DELIM export USE_DOCKER_IN_DOCKER=true export OSRF_REPOS_TO_USE="stable" +export USE_ROS_REPO=true +export ROS2=true export DEPENDENCY_PKGS="python3-pip \ bash \ apt-transport-https \ ca-certificates \ curl \ software-properties-common \ - psmisc" # killall + python3-rocker \ + psmisc \ + xvfb" . "${SCRIPT_DIR}/lib/docker_generate_dockerfile.bash" . "${SCRIPT_DIR}/lib/docker_run.bash" diff --git a/jenkins-scripts/dsl/gzdev.dsl b/jenkins-scripts/dsl/gzdev.dsl index 1137eac9b..ed954c4a1 100644 --- a/jenkins-scripts/dsl/gzdev.dsl +++ b/jenkins-scripts/dsl/gzdev.dsl @@ -1,7 +1,7 @@ import _configs_.* import javaposse.jobdsl.dsl.Job -def supported_distros = [ 'bionic' ] +def supported_distros = [ 'jammy' ] def supported_arches = [ 'amd64' ] @@ -17,6 +17,8 @@ supported_distros.each { distro -> gzdev_ci_job.with { + + label Globals.nontest_label("large-memory") scm { git { @@ -54,7 +56,6 @@ supported_distros.each { distro -> gzdev_any_job.with { - // use only the most powerful nodes label Globals.nontest_label("large-memory") steps { From b74f628b7e865955866d58bea260608d158e88a4 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 12 Apr 2024 18:51:42 +0200 Subject: [PATCH 03/13] Update gzdev to use "purge" and remove the deprecated "--keyserver" (#1129) * Remove keyserver from gzdev * Use purge before dealing with gzdev installation --------- Signed-off-by: Jose Luis Rivero --- .../docker/lib/docker_generate_dockerfile.bash | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash b/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash index b78f8f432..44499fe1f 100644 --- a/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash +++ b/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash @@ -45,7 +45,7 @@ GZDEV_BRANCH=${GZDEV_BRANCH:-master} if python3 ${SCRIPT_DIR}/../tools/detect_ci_matching_branch.py "${ghprbSourceBranch}"; then GZDEV_TRY_BRANCH=$ghprbSourceBranch fi - + KEYSERVER="keyserver.ubuntu.com" if [[ "${DISTRO}" == 'bionic' || "${DISTRO}" == 'focal' ]]; then KEYSERVER="hkps://pgp.surf.nl" @@ -62,23 +62,26 @@ RUN if [ -n $GZDEV_TRY_BRANCH ]; then \ fi || true # print branch for informational purposes RUN git -C ${GZDEV_DIR} branch +# clean all _gzdev_ repository installations from the system before handling the configuration +# otherwise the docker cache could contain unexpected repositories +RUN ${GZDEV_DIR}/gzdev.py repository purge DELIM_OSRF_REPO_GIT if [[ -n ${GZDEV_PROJECT_NAME} ]]; then # debian sid docker images does not return correct name so we need to use # force-linux-distro cat >> Dockerfile << DELIM_OSRF_REPO_GZDEV -RUN ${GZDEV_DIR}/gzdev.py repository enable --project=${GZDEV_PROJECT_NAME} --force-linux-distro=${DISTRO} --keyserver ${KEYSERVER} || ( git -C ${GZDEV_DIR} pull origin ${GZDEV_BRANCH} && \ +RUN ${GZDEV_DIR}/gzdev.py repository enable --project=${GZDEV_PROJECT_NAME} --force-linux-distro=${DISTRO} || ( git -C ${GZDEV_DIR} pull origin ${GZDEV_BRANCH} && \ if [ -n $GZDEV_TRY_BRANCH ]; then git -C ${GZDEV_DIR} checkout $GZDEV_TRY_BRANCH; fi || true && \ - ${GZDEV_DIR}/gzdev.py repository enable --project=${GZDEV_PROJECT_NAME} --force-linux-distro=${DISTRO} --keyserver ${KEYSERVER}) + ${GZDEV_DIR}/gzdev.py repository enable --project=${GZDEV_PROJECT_NAME} --force-linux-distro=${DISTRO} ) DELIM_OSRF_REPO_GZDEV fi # This could duplicate repositories enabled in the step above. gzdev should warn about it without failing. for repo in ${OSRF_REPOS_TO_USE}; do cat >> Dockerfile << DELIM_OSRF_REPO -RUN ${GZDEV_DIR}/gzdev.py repository enable osrf ${repo} --force-linux-distro=${DISTRO} --keyserver ${KEYSERVER} || ( git -C ${GZDEV_DIR} pull origin ${GZDEV_BRANCH} && \ +RUN ${GZDEV_DIR}/gzdev.py repository enable osrf ${repo} --force-linux-distro=${DISTRO} || ( git -C ${GZDEV_DIR} pull origin ${GZDEV_BRANCH} && \ if [ -n $GZDEV_TRY_BRANCH ]; then git -C ${GZDEV_DIR} checkout $GZDEV_TRY_BRANCH; fi || true && \ - ${GZDEV_DIR}/gzdev.py repository enable osrf ${repo} --force-linux-distro=${DISTRO} --keyserver ${KEYSERVER}) + ${GZDEV_DIR}/gzdev.py repository enable osrf ${repo} --force-linux-distro=${DISTRO} ) DELIM_OSRF_REPO done } From 3da89b7a888f5933b3289d45104e76b12fbbd108 Mon Sep 17 00:00:00 2001 From: "Addisu Z. Taddese" Date: Wed, 17 Apr 2024 10:35:10 -0500 Subject: [PATCH 04/13] Inject test output into junit files generated by gtest (#1130) * Add script to parse system output from cmake junit files and inject them into junit files created by gtest * Ignore test names that start with check_ * Document the output when test file is not expected * Inject error in testsuite if failures appear --------- Signed-off-by: Addisu Z. Taddese Signed-off-by: Jose Luis Rivero Co-authored-by: Jose Luis Rivero --- .../_generic_linux_compilation_build.sh.bash | 5 +- .../tools/cmake_to_gtest_junit_output.py | 105 ++++++++++++++++++ 2 files changed, 109 insertions(+), 1 deletion(-) create mode 100644 jenkins-scripts/tools/cmake_to_gtest_junit_output.py diff --git a/jenkins-scripts/docker/lib/_generic_linux_compilation_build.sh.bash b/jenkins-scripts/docker/lib/_generic_linux_compilation_build.sh.bash index 9d63185d9..344431001 100644 --- a/jenkins-scripts/docker/lib/_generic_linux_compilation_build.sh.bash +++ b/jenkins-scripts/docker/lib/_generic_linux_compilation_build.sh.bash @@ -96,7 +96,10 @@ if $GENERIC_ENABLE_TESTS; then echo '# BEGIN SECTION: running tests' init_stopwatch TEST mkdir -p \$HOME - make test ARGS="-VV ${BUILDING_EXTRA_MAKETEST_PARAMS}" || true + make test ARGS="-VV ${BUILDING_EXTRA_MAKETEST_PARAMS} --output-junit cmake_junit_output.xml" || true + if [ -f cmake_junit_output.xml ]; then + python3 $WORKSPACE/scripts/jenkins-scripts/tools/cmake_to_gtest_junit_output.py cmake_junit_output.xml test_results || true + fi stop_stopwatch TEST echo '# END SECTION' else diff --git a/jenkins-scripts/tools/cmake_to_gtest_junit_output.py b/jenkins-scripts/tools/cmake_to_gtest_junit_output.py new file mode 100644 index 000000000..a255df892 --- /dev/null +++ b/jenkins-scripts/tools/cmake_to_gtest_junit_output.py @@ -0,0 +1,105 @@ +# Copyright (C) 2024 Open Source Robotics Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os +import sys +from xml.etree import ElementTree as ET + +SYSTEM_OUT_TAG = "system-out" + + +def main(argv=sys.argv[1:]): + parser = argparse.ArgumentParser( + description="Parse system output from junit file generated by CTest" + "and inject it into existing junit files generated " + "by gtest. Useful in some cases like ASAN reports." + ) + parser.add_argument( + "cmake_junit_file", + help="Junit file generated by 'cmake --output-junit='", + ) + parser.add_argument( + "gtest_results_dir", + help="Directory where gtest junit files were output" + ) + args = parser.parse_args(argv) + + doc = ET.parse(args.cmake_junit_file) + + for testcase in doc.findall("testcase"): + # Ignore tests that start with 'check_'. Those are tests that simply + # run the check_test_run.py script, so we don't need their output. + if testcase.attrib["name"].startswith("check_"): + continue + + results_file_name = os.path.join( + args.gtest_results_dir, testcase.attrib["name"] + ".xml" + ) + system_output = testcase.find(SYSTEM_OUT_TAG) + testcase_failures = 0 + assert system_output is not None + try: + print(f"Processing {results_file_name}") + results_doc = ET.parse(results_file_name) + should_write_to_file = False + for result_testsuite in results_doc.findall("testsuite"): + if testcase.attrib["status"] == 'fail': + testcase_failures += 1 + print(f" - failure {testcase_failures} in {testcase.attrib['name']}") + result_testsuite.attrib['failures'] = \ + f'{testcase_failures}' + error_tag = ET.Element('error', {'message': + f'Failure injected from CTest by tools/{os.path.basename(__file__)}'}) + error_tag.text = system_output.text + print(f" - inject error in {testcase.attrib['name']}") + result_testsuite.append(error_tag) + elif result_testsuite.find(SYSTEM_OUT_TAG) is None: + print(f" - inject system-out in {testcase.attrib['name']}") + result_testsuite.append(system_output) + should_write_to_file = True + + # junit files created by gtest start with `` (plural). + # If the root tag is ``, this indicates that the junit file was + # created by our check_test_run.py script, which means the test crashed, in which + # case, we will just write the output to the first testcase. + if results_doc.getroot().tag == "testsuite": + result_test_case = results_doc.find("testcase") + assert result_test_case is not None + if result_test_case.find(SYSTEM_OUT_TAG) is None: + result_test_case.append(system_output) + should_write_to_file = True + + if testcase_failures > 0: + print(f' - update root testsuites failures tag to {testcase_failures}') + a = results_doc.getroot() + a.attrib["failures"] = f'{testcase_failures}' + should_write_to_file = True + + if should_write_to_file: + results_doc.write(results_file_name) + + except Exception as e: + print(e) if str(e).startswith('[Errno 2]') else exit(1) + + print(""" + Possible failures above in the script are expected due to: + - Tests written directly in CTest and not using gtest + (INTEGRATION_versioned_symbols as an example) + - Tests in Python using pytest have different name: + right name in CTest vs filepath name in current setup + """) + +if __name__ == "__main__": + main() From f4c65105497245e51f939a0f5976ced0b855ee1f Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Wed, 17 Apr 2024 21:56:49 -0700 Subject: [PATCH 05/13] detect_cmake_version.py: use raw regex strings Some of the sequences in the regex patterns are not valid in python string literals (such as `\(`, so use raw strings instead. Signed-off-by: Steve Peters --- jenkins-scripts/tools/detect_cmake_major_version.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/jenkins-scripts/tools/detect_cmake_major_version.py b/jenkins-scripts/tools/detect_cmake_major_version.py index 448e02f09..b93ae9ef3 100644 --- a/jenkins-scripts/tools/detect_cmake_major_version.py +++ b/jenkins-scripts/tools/detect_cmake_major_version.py @@ -12,11 +12,11 @@ txt = f.read() f.close() -old_version = re.search('set *\( *PROJECT_MAJOR_VERSION +(\d+)', txt) -gazebo_version = re.search('set *\( *GAZEBO_MAJOR_VERSION +(\d+)', txt) -sdformat_version = re.search('set *\( *SDF_MAJOR_VERSION +(\d+)', txt) -ign_cmake_version = re.search('project *\( *ignition-[a-z\-_]+(\d+)', txt) -cmake_project_version = re.search('project *\(.*VERSION +(\d+)', txt) +old_version = re.search(r'set *\( *PROJECT_MAJOR_VERSION +(\d+)', txt) +gazebo_version = re.search(r'set *\( *GAZEBO_MAJOR_VERSION +(\d+)', txt) +sdformat_version = re.search(r'set *\( *SDF_MAJOR_VERSION +(\d+)', txt) +ign_cmake_version = re.search(r'project *\( *ignition-[a-z\-_]+(\d+)', txt) +cmake_project_version = re.search(r'project *\(.*VERSION +(\d+)', txt) if old_version: print(old_version.group(1) ) elif gazebo_version: From b9dd06af50c7ea1853fa31c4fcb03029fc179040 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 30 Apr 2024 16:18:44 +0200 Subject: [PATCH 06/13] Add support for Noble -ci-{branch}- jobs on Harmonic (#1125) * Add support for Noble -ci-{branch}- jobs on Harmonic Signed-off-by: Jose Luis Rivero * Automated change: update logs [skip ci] --------- Signed-off-by: Jose Luis Rivero Co-authored-by: j-rivero --- jenkins-scripts/dsl/gz-collections.yaml | 28 +++++++++++++++++++++ jenkins-scripts/dsl/logs/generated_jobs.txt | 16 ++++++++++++ 2 files changed, 44 insertions(+) diff --git a/jenkins-scripts/dsl/gz-collections.yaml b/jenkins-scripts/dsl/gz-collections.yaml index 3dc9dc5fc..f91dee7e5 100644 --- a/jenkins-scripts/dsl/gz-collections.yaml +++ b/jenkins-scripts/dsl/gz-collections.yaml @@ -338,6 +338,7 @@ collections: ci: configs: - jammy + - noble - brew - win packaging: @@ -502,6 +503,33 @@ ci_configs: - pr_abichecker - stable_branches - stable_branches_asan + - name: noble + system: + so: linux + distribution: ubuntu + version: noble + arch: amd64 + requirements: + large_memory: + - gz-physics + nvidia_gpu: + - gz-sim + - gz-gui + - gz-rendering + - gz-sensors + exclude: + all: + - gz-harmonic + - __upcoming__ + abichecker: + - gz-cmake + - gz-tools + pre_setup_script_hook: + gz-physics: + - "export MAKE_JOBS=1" + tests_disabled: + ci_categories_enabled: + - stable_branches - name: brew system: so: darwin diff --git a/jenkins-scripts/dsl/logs/generated_jobs.txt b/jenkins-scripts/dsl/logs/generated_jobs.txt index 94825d5c3..38eb393fc 100644 --- a/jenkins-scripts/dsl/logs/generated_jobs.txt +++ b/jenkins-scripts/dsl/logs/generated_jobs.txt @@ -223,50 +223,66 @@ branch_ci garden sdformat-sdf13-win branch_ci harmonic gz_cmake-3-win branch_ci harmonic gz_cmake-ci-gz-cmake3-homebrew-amd64 branch_ci harmonic gz_cmake-ci-gz-cmake3-jammy-amd64 +branch_ci harmonic gz_cmake-ci-gz-cmake3-noble-amd64 branch_ci harmonic gz_common-5-win branch_ci harmonic gz_common-ci-gz-common5-homebrew-amd64 branch_ci harmonic gz_common-ci-gz-common5-jammy-amd64 +branch_ci harmonic gz_common-ci-gz-common5-noble-amd64 branch_ci harmonic gz_fuel_tools-9-win branch_ci harmonic gz_fuel_tools-ci-gz-fuel-tools9-homebrew-amd64 branch_ci harmonic gz_fuel_tools-ci-gz-fuel-tools9-jammy-amd64 +branch_ci harmonic gz_fuel_tools-ci-gz-fuel-tools9-noble-amd64 branch_ci harmonic gz_gui-8-win branch_ci harmonic gz_gui-ci-gz-gui8-homebrew-amd64 branch_ci harmonic gz_gui-ci-gz-gui8-jammy-amd64 +branch_ci harmonic gz_gui-ci-gz-gui8-noble-amd64 branch_ci harmonic gz_launch-7-win branch_ci harmonic gz_launch-ci-gz-launch7-homebrew-amd64 branch_ci harmonic gz_launch-ci-gz-launch7-jammy-amd64 +branch_ci harmonic gz_launch-ci-gz-launch7-noble-amd64 branch_ci harmonic gz_math-7-win branch_ci harmonic gz_math-ci-gz-math7-homebrew-amd64 branch_ci harmonic gz_math-ci-gz-math7-jammy-amd64 +branch_ci harmonic gz_math-ci-gz-math7-noble-amd64 branch_ci harmonic gz_msgs-10-win branch_ci harmonic gz_msgs-ci-gz-msgs10-homebrew-amd64 branch_ci harmonic gz_msgs-ci-gz-msgs10-jammy-amd64 +branch_ci harmonic gz_msgs-ci-gz-msgs10-noble-amd64 branch_ci harmonic gz_physics-7-win branch_ci harmonic gz_physics-ci-gz-physics7-homebrew-amd64 branch_ci harmonic gz_physics-ci-gz-physics7-jammy-amd64 +branch_ci harmonic gz_physics-ci-gz-physics7-noble-amd64 branch_ci harmonic gz_plugin-2-win branch_ci harmonic gz_plugin-ci-gz-plugin2-homebrew-amd64 branch_ci harmonic gz_plugin-ci-gz-plugin2-jammy-amd64 +branch_ci harmonic gz_plugin-ci-gz-plugin2-noble-amd64 branch_ci harmonic gz_rendering-8-win branch_ci harmonic gz_rendering-ci-gz-rendering8-homebrew-amd64 branch_ci harmonic gz_rendering-ci-gz-rendering8-jammy-amd64 +branch_ci harmonic gz_rendering-ci-gz-rendering8-noble-amd64 branch_ci harmonic gz_sensors-8-win branch_ci harmonic gz_sensors-ci-gz-sensors8-homebrew-amd64 branch_ci harmonic gz_sensors-ci-gz-sensors8-jammy-amd64 +branch_ci harmonic gz_sensors-ci-gz-sensors8-noble-amd64 branch_ci harmonic gz_sim-8-win branch_ci harmonic gz_sim-ci-gz-sim8-homebrew-amd64 branch_ci harmonic gz_sim-ci-gz-sim8-jammy-amd64 +branch_ci harmonic gz_sim-ci-gz-sim8-noble-amd64 branch_ci harmonic gz_tools-2-win branch_ci harmonic gz_tools-ci-gz-tools2-homebrew-amd64 branch_ci harmonic gz_tools-ci-gz-tools2-jammy-amd64 +branch_ci harmonic gz_tools-ci-gz-tools2-noble-amd64 branch_ci harmonic gz_transport-13-win branch_ci harmonic gz_transport-ci-gz-transport13-homebrew-amd64 branch_ci harmonic gz_transport-ci-gz-transport13-jammy-amd64 +branch_ci harmonic gz_transport-ci-gz-transport13-noble-amd64 branch_ci harmonic gz_utils-2-win branch_ci harmonic gz_utils-ci-gz-utils2-homebrew-amd64 branch_ci harmonic gz_utils-ci-gz-utils2-jammy-amd64 +branch_ci harmonic gz_utils-ci-gz-utils2-noble-amd64 branch_ci harmonic sdformat-ci-sdf14-homebrew-amd64 branch_ci harmonic sdformat-ci-sdf14-jammy-amd64 +branch_ci harmonic sdformat-ci-sdf14-noble-amd64 branch_ci harmonic sdformat-sdf14-win branch_ci ionic gz_cmake-ci-main-homebrew-amd64 branch_ci ionic gz_cmake-ci-main-jammy-amd64 From 42a19d4e6521fcb90560c1632bdbc23a845120b7 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 7 May 2024 18:56:09 +0200 Subject: [PATCH 07/13] Make release.py to automatically deal with ign- Citadel packages (#1109) * Make release.py to automatically deal with ign- Citadel packages Signed-off-by: Jose Luis Rivero * Use gz- in SOURCE_REPO_URI Signed-off-by: Jose Luis Rivero * Be sure of not using github.org Signed-off-by: Jose Luis Rivero * Do not enforce the PACKAGE name in OSRFSourceCreation Signed-off-by: Jose Luis Rivero * Add test to check SOURCE_REPO_URI Signed-off-by: Jose Luis Rivero * Disable safety check on package-name tarball Signed-off-by: Jose Luis Rivero * repository_uploader uses PACKAGE_ALIAS not PACKAGE Signed-off-by: Jose Luis Rivero * PACKAGE needs to be in OSRFSourceCreation Signed-off-by: Jose Luis Rivero * Fix dealing with ign-gazebo to be gz-sim Signed-off-by: Jose Luis Rivero --------- Signed-off-by: Jose Luis Rivero --- check_releasepy.bash | 29 +++++++++- .../dsl/_configs_/OSRFSourceCreation.groovy | 21 +++++--- jenkins-scripts/dsl/test.dsl | 3 +- release.py | 53 +++++++++---------- 4 files changed, 70 insertions(+), 36 deletions(-) diff --git a/check_releasepy.bash b/check_releasepy.bash index dbef90108..921153017 100755 --- a/check_releasepy.bash +++ b/check_releasepy.bash @@ -25,6 +25,16 @@ exec_ignition_releasepy_test() ign-foo 1.2.3 token ${test_params}"" } +exec_ignition_gazebo_releasepy_test() +{ + test_params=${1} + + ./release.py \ + --dry-run \ + --no-sanity-checks \ + ign-gazebo 1.2.3 token ${test_params}"" +} + expect_job_run() { output="${1}" job="${2}" @@ -66,10 +76,11 @@ expect_param() } -source_repo_uri_test=$(exec_releasepy_test "--source-repo-uri https://github.org/gazebosim/gz-foo") +source_repo_uri_test=$(exec_releasepy_test "--source-repo-uri https://github.com/gazebosim/gz-foo.git") expect_job_run "${source_repo_uri_test}" "gz-foo-source" expect_job_not_run "${source_repo_uri_test}" "gz-foo-debbuilder" expect_number_of_jobs "${source_repo_uri_test}" "1" +expect_param "${source_repo_uri_test}" "SOURCE_REPO_URI=https%3A%2F%2Fgithub.com%2Fgazebosim%2Fgz-foo.git" source_tarball_uri_test=$(exec_releasepy_test "--source-tarball-uri https://gazebosim/gz-foo-1.2.3.tar.gz") expect_job_run "${source_tarball_uri_test}" "gz-foo-debbuilder" @@ -92,10 +103,13 @@ expect_job_not_run "${bump_linux_test}" "gz-foo-source" expect_number_of_jobs "${bump_linux_test}" "6" expect_param "${bump_linux_test}" "RELEASE_VERSION=2" -ignition_test=$(exec_ignition_releasepy_test "--source-repo-uri https://github.org/gazebosim/gz-foo") +ignition_test=$(exec_ignition_releasepy_test "--source-repo-uri https://github.com/gazebosim/gz-foo.git") expect_job_run "${ignition_test}" "gz-foo-source" expect_job_not_run "${ignition_test}" "ignition-foo-source" expect_number_of_jobs "${ignition_test}" "1" +expect_param "${ignition_test}" "PACKAGE=ign-foo" +expect_param "${ignition_test}" "PACKAGE_ALIAS=ignition-foo" +expect_param "${ignition_test}" "SOURCE_REPO_URI=https%3A%2F%2Fgithub.com%2Fgazebosim%2Fgz-foo.git" ignition_source_tarball_uri_test=$(exec_ignition_releasepy_test "--source-tarball-uri https://gazebosim/gz-foo-1.2.3.tar.gz") expect_job_run "${ignition_source_tarball_uri_test}" "gz-foo-debbuilder" @@ -103,3 +117,14 @@ expect_job_run "${ignition_source_tarball_uri_test}" "generic-release-homebrew_p expect_job_not_run "${ignition_source_tarball_uri_test}" "gz-foo-source" expect_number_of_jobs "${ignition_source_tarball_uri_test}" "7" expect_param "${ignition_source_tarball_uri_test}" "SOURCE_TARBALL_URI=https%3A%2F%2Fgazebosim%2Fgz-foo-1.2.3.tar.gz" +expect_param "${ignition_source_tarball_uri_test}" "PACKAGE=ign-foo" +expect_param "${ignition_source_tarball_uri_test}" "PACKAGE_ALIAS=ignition-foo" + +ign_gazebo_source_tarball_uri_test=$(exec_ignition_gazebo_releasepy_test "--source-tarball-uri https://gazebosim/ign-gazebo-1.2.3.tar.gz") +expect_job_run "${ign_gazebo_source_tarball_uri_test}" "gz-sim-debbuilder" +expect_job_run "${ign_gazebo_source_tarball_uri_test}" "generic-release-homebrew_pull_request_updater" +expect_job_not_run "${ign_gazebo_source_tarball_uri_test}" "gz-sim-source" +expect_number_of_jobs "${ign_gazebo_source_tarball_uri_test}" "7" +expect_param "${ign_gazebo_source_tarball_uri_test}" "SOURCE_TARBALL_URI=https%3A%2F%2Fgazebosim%2Fign-gazebo-1.2.3.tar.gz" +expect_param "${ign_gazebo_source_tarball_uri_test}" "PACKAGE=ign-gazebo" +expect_param "${ign_gazebo_source_tarball_uri_test}" "PACKAGE_ALIAS=ignition-gazebo" diff --git a/jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy b/jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy index 4020a006f..2f8f490bb 100644 --- a/jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy +++ b/jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy @@ -33,9 +33,11 @@ class OSRFSourceCreation stringParam("DISTRO", default_params.find{ it.key == "DISTRO"}?.value, "Linux release inside LINUX_DISTRO to generate sources on") - choiceParam('PACKAGE', - [default_params.find{ it.key == "PACKAGE"}?.value], - "For downstream use: Package name (can not be modified)") + // Not using choiceParam here to support Citadel/Fortress ign-* packages not only + // gz-* packages + stringParam('PACKAGE', + default_params.find{ it.key == "PACKAGE"}?.value, + "For downstream use: Package name") stringParam("RELEASE_VERSION", default_params.find{ it.key == "RELEASE_VERSION"}?.value, "For downstream jobs: Packages release version") @@ -70,8 +72,6 @@ class OSRFSourceCreation priority 100 } - def canonical_package_name = Globals.get_canonical_package_name( - default_params.find{ it.key == "PACKAGE"}.value) def s3_download_url_basedir = Globals.s3_download_url_basedir( default_params.find{ it.key == "PACKAGE"}?.value) @@ -94,6 +94,14 @@ class OSRFSourceCreation /bin/bash -x ./scripts/jenkins-scripts/docker/gz-source-generation.bash """.stripIndent() ) + + // This can be enabled after the complete deprecation of ign- names + // that uses ignition aliases. + // def canonical_package_name = Globals.get_canonical_package_name( + // default_params.find{ it.key == "PACKAGE"}.value) + // + // then improve the find command below with: + // -name ${canonical_package_name}-\${VERSION}.tar.* \ shell("""\ #!/bin/bash -xe @@ -101,7 +109,7 @@ class OSRFSourceCreation # deal with changes in the compression of the tarballs. tarball=\$(find \${WORKSPACE}/${pkg_sources_dir} \ -type f \ - -name ${canonical_package_name}-\${VERSION}.tar.* \ + -name *-\${VERSION}.tar.* \ -printf "%f\\n") if [[ -z \${tarball} ]] || [[ \$(wc -w <<< \${tarball}) != 1 ]]; then # There is one use case that can be valid but failed to find canonical_package_name @@ -148,6 +156,7 @@ class OSRFSourceCreation parameters { currentBuild() predefinedProps([PROJECT_NAME_TO_COPY_ARTIFACTS: '${JOB_NAME}', + PACKAGE_ALIAS: '${PACKAGE_ALIAS}', S3_UPLOAD_PATH: "${Globals.s3_releases_dir(package_name)}/"]) // relative path with a final / propertiesFile(properties_file) // S3_FILES_TO_UPLOAD } diff --git a/jenkins-scripts/dsl/test.dsl b/jenkins-scripts/dsl/test.dsl index 56af6a360..0247ad212 100644 --- a/jenkins-scripts/dsl/test.dsl +++ b/jenkins-scripts/dsl/test.dsl @@ -30,6 +30,7 @@ releasepy_job.with { def gz_source_job = job("_test_gz_source") OSRFSourceCreation.create(gz_source_job, [ PACKAGE: "gz-plugin2", + PACKAGE_ALIAS: "gz-plugin2", SOURCE_REPO_URI: "https://github.com/gazebosim/gz-plugin.git", SOURCE_REPO_REF: "gz-plugin2"]) OSRFSourceCreation.call_uploader_and_releasepy(gz_source_job, @@ -50,7 +51,7 @@ repo_uploader.with parameters { - stringParam('PACKAGE','','Package name') + stringParam('PACKAGE_ALIAS','','Used by real repository_uploader') stringParam('S3_UPLOAD_PATH','', 'S3 path to upload') stringParam('S3_FILES_TO_UPLOAD','', 'S3 file names to upload') stringParam('UPLOAD_TO_REPO','none','repo to upload') diff --git a/release.py b/release.py index 97568ebe8..265796308 100755 --- a/release.py +++ b/release.py @@ -31,22 +31,6 @@ IGNORE_DRY_RUN = True -GARDEN_IGN_PACKAGES = ['ign-cmake3', - 'ign-common5', - 'ign-fuel-tools8', - 'ign-sim7', - 'ign-gui7', - 'ign-launch6', - 'ign-math7', - 'ign-msgs9', - 'ign-physics6', - 'ign-plugin2', - 'ign-rendering7', - 'ign-sensors7', - 'ign-tools2', - 'ign-transport12', - 'ign-utils2'] - class ErrorNoPermsRepo(Exception): pass @@ -85,6 +69,12 @@ def get_canonical_package_name(pkg_name): return pkg_name.rstrip('1234567890') +def replace_ignition_gz(pkg_name): + return pkg_name \ + .replace('ignition-gazebo', 'gz-sim') \ + .replace('ignition-', 'gz-') + + def github_repo_exists(url): try: check_call(['git', 'ls-remote', '-q', '--exit-code', url], IGNORE_DRY_RUN) @@ -164,10 +154,6 @@ def parse_args(argv): args = parser.parse_args() - if args.package in GARDEN_IGN_PACKAGES: - print(f"Garden packages start with gz- prefix, changing {args.package} to {args.package.replace('ign-','gz-')}",) - args.package = args.package.replace('ign-', 'gz-') - args.package_alias = args.package if args.package.startswith('ign-'): args.package_alias = args.package.replace('ign-', 'ignition-') @@ -310,6 +296,11 @@ def sanity_check_source_repo_uri(source_repo_uri): if not parsed_uri.scheme == "https" or \ not parsed_uri.path.endswith(".git"): error("--source-repo-uri parameter should start with https:// and end with .git") + # Needs to be fully in sync for SOURCE_REPO_URI values in + # OSRFSourceCreation + # https://github.com/gazebo-tooling/release-tools/blob/master/jenkins-scripts/dsl/gazebo_libs.dsl#L513 + if parsed_uri.netloc.startswith("github.org"): + error("--source-repo-uri needs github.com instead of github.org") def sanity_check_bump_linux(source_tarball_uri): @@ -445,12 +436,20 @@ def generate_source_repository_uri(args): git_remote = out.decode().split('\n')[0] if org_repo not in git_remote: - print(f""" !! Automatic calculation of source_repo_uri failed.\ - \n * git remote origin is: {git_remote}\ - \n * Package name generated org/repo: {org_repo}\ - \n >> Please use --source-repo-uri parameter""") - sys.exit(1) + # Handle the special case for citadel ignition repositories + if replace_ignition_gz(org_repo) not in git_remote: + print(f""" !! Automatic calculation of the source repository URI\ + failed with different information:\ + \n * git remote origin in the local direcotry is: {git_remote}\ + \n * Package name generated org/repo: {org_repo}\ + \n >> Please use --source-repo-uri parameter""") + sys.exit(1) + else: + org_repo = replace_ignition_gz(org_repo) + print(' ~ Ignition found in generated org/repo assuming gz repo: ' + + org_repo) + # Always use github.com return f"https://github.com/{org_repo}.git" # NOQA @@ -506,7 +505,7 @@ def display_help_job_chain_for_source_calls(args): f'{JENKINS_URL}/job/repository_uploader_packages/?{url_search_params}' rel_search_params = urllib.parse.urlencode( {'search': - f'{args.package_alias}/{args.version}-{args.release_version}'}) + f'{args.package}/{args.version}-{args.release_version}'}) releasepy_check_url = \ f'{JENKINS_URL}/job/_releasepy/?{rel_search_params}' print('\tINFO: After the source job finished, the release process will trigger:\n' @@ -523,7 +522,7 @@ def go(argv): if not args.release_version: args.release_version = 1 - package_alias_force_gz = args.package_alias.replace('ignition-','gz-') + package_alias_force_gz = replace_ignition_gz(args.package_alias) print(f"Downloading releasing info for {args.package}") # Sanity checks and dicover supported distributions before proceed. From 352318ef1f2f23b9986aa498be5bffef9759792e Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 8 May 2024 16:47:30 +0200 Subject: [PATCH 08/13] In Noble user amrhf standard images (#1135) Signed-off-by: Jose Luis Rivero --- .../docker/lib/docker_generate_dockerfile.bash | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash b/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash index 44499fe1f..fc3e7c9eb 100644 --- a/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash +++ b/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash @@ -116,12 +116,10 @@ case ${ARCH} in fi ;; 'armhf') - # There is no osrf/jammy_armhf image. Trying new - # platform support in docker - if [[ ${DISTRO} == 'jammy' ]]; then - FROM_VALUE=${LINUX_DISTRO}:${DISTRO} - else + if [[ ${DISTRO} == 'focal' ]]; then FROM_VALUE=osrf/${LINUX_DISTRO}_${ARCH}:${DISTRO} + else + FROM_VALUE=${LINUX_DISTRO}:${DISTRO} fi ;; 'arm64') From 602a1c0f2bcb75b1a227db44a9366b59783a135c Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 8 May 2024 17:58:16 +0200 Subject: [PATCH 09/13] Support noble for armhf (#1136) Signed-off-by: Jose Luis Rivero --- jenkins-scripts/docker/lib/docker_run.bash | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/jenkins-scripts/docker/lib/docker_run.bash b/jenkins-scripts/docker/lib/docker_run.bash index 5c96eaef5..f646c73eb 100644 --- a/jenkins-scripts/docker/lib/docker_run.bash +++ b/jenkins-scripts/docker/lib/docker_run.bash @@ -26,7 +26,9 @@ USER=$(whoami) # platform support starts on versions greater than 17.07 PLAFTORM_PARAM= DOCKER_CLI_PLUGIN= -if [[ ${DISTRO} == 'jammy' && ${ARCH} == 'armhf' ]]; then +if [[ ${LINUX_DISTRO} == 'ubuntu' && \ + ${DISTRO} != 'focal' && \ + ${ARCH} == 'armhf' ]]; then PLAFTORM_PARAM="--platform=linux/armhf" DOCKER_CLI_PLUGIN="buildx" fi From 5a29f983185b2d79d44c409140c281b0bbcf2c2a Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 16 May 2024 16:17:18 +0200 Subject: [PATCH 10/13] Rewrite DSL generation to remove the approach based on Indexes (#1137) The DSL generation in gazebo_libs is based on an approach of building two indexes before looping over the gz-collections.yaml file. That approach does not respect the fact of having different ci_config (CI confgurations) that use the same platform and will overwrite the first one found if that situation occurs. The index build process also did not load the exclusion clauses included in the ci_configuration which are useful to be parsed when building the index. The changes in this PR modify it to mainly: * Loop over the gz-collections.yaml directly the first time to generate the jobs that are linked to lib branches gz-fooX. * While looping it build a branch_index that will be used to build the pr jobs that are in the form gz-foo-pr_any- not based on the branch directly. * Loop over the branch_index created to create the pr jobs. Note: the common jobs used for -pr- need the assumption of having the same metadata when ci_configs use the same platform. The restrictions needs to be coded or the design corrected with structural changes. -------- Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/gazebo_libs.dsl | 189 +++++++++++++++------------- 1 file changed, 104 insertions(+), 85 deletions(-) diff --git a/jenkins-scripts/dsl/gazebo_libs.dsl b/jenkins-scripts/dsl/gazebo_libs.dsl index af5097a43..3b3b18a0b 100644 --- a/jenkins-scripts/dsl/gazebo_libs.dsl +++ b/jenkins-scripts/dsl/gazebo_libs.dsl @@ -69,21 +69,12 @@ boolean are_cmake_warnings_enabled(lib_name, ci_config) } /* + * TODO: deprecated, migrate the pkgconf_per_src index to use new branch_index + * or generate a new one together with the branch_index + * * Generate an index that facilitates the operations with the yaml values, * avoiding to parse them several times. * - * # ci_configs_by_lib index structure: - * lib_name : [ ci_config_name : [ .branch .collection ] ] - * - * The index main keys are the lib names (i.e: gz-cmake) and associated them - * another map of CI configuration names supported as keys (i.e: jammy) with the - * list of associated items composed by a map: branch (and collection) that CI configuration - * (i.e [[branch:gz-cmake3, collection: harmonic], [branch: gz-cmake3, collection: garden]) - * as values. In a graphic; - * - * index[gz-cmake][jammy] -> [ branch: gz-cmake3, collection: garden , - * branch: gz-cmake3, collection: harmonic] - * * # pkgconf_per_src index structure: * pkg_src_name : [ packaging_config_name : [ .lib_name .collection ] ] * @@ -94,16 +85,12 @@ boolean are_cmake_warnings_enabled(lib_name, ci_config) * * index[gz-cmake3][jammy] -> [ lib_name: gz-cmake, collection: harmonic ] */ -void generate_ciconfigs_by_lib(config, ciconf_per_lib_index, pkgconf_per_src_index) +void generate_ciconfigs_by_lib(config, pkgconf_per_src_index) { config.collections.each { collection -> collection.libs.each { lib -> def libName = lib.name def branch = lib.repo.current_branch - collection.ci.configs.each { config_name -> - ciconf_per_lib_index[libName][config_name] = ciconf_per_lib_index[libName][config_name]?: [] - ciconf_per_lib_index[libName][config_name].contains(branch) ?: ciconf_per_lib_index[libName][config_name] << [branch: branch, collection: collection.name] - } def pkg_name = lib.name + lib.major_version if (collection.packaging.linux?.ignore_major_version?.contains(libName)) pkg_name = lib.name @@ -320,42 +307,57 @@ def generate_debbuilder_job(src_name, ArrayList pre_setup_script_hooks) } } -def ciconf_per_lib_index = [:].withDefault { [:] } def pkgconf_per_src_index = [:].withDefault { [:] } -generate_ciconfigs_by_lib(gz_collections_yaml, ciconf_per_lib_index, pkgconf_per_src_index) - -// Generate PR jobs: 1 per ci configuration on each lib -ciconf_per_lib_index.each { lib_name, lib_configs -> - lib_configs.each { ci_configs -> - def config_name = ci_configs.getKey() +generate_ciconfigs_by_lib(gz_collections_yaml, pkgconf_per_src_index) +/* + * + * Loop over each collection, inside each collection loop over the ci configurations assigned + * and finally loop over each collection library listed. + * + */ +def branch_index = [:].withDefault { [:] } +gz_collections_yaml.collections.each { collection -> + collection.ci.configs.each { config_name -> def ci_config = gz_collections_yaml.ci_configs.find{ it.name == config_name } - def branches_with_collections = ci_configs.getValue() - def branch_names = branches_with_collections.collect { it.branch }.unique() - def script_name_prefix = cleanup_library_name(lib_name) - def gz_job_name_prefix = lib_name.replaceAll('-','_') def distro = ci_config.system.version def arch = ci_config.system.arch def categories_enabled = ci_config.ci_categories_enabled - def ws_checkout_dir = lib_name - if (ci_config.exclude.all?.contains(lib_name)) - return - assert(lib_name) - assert(branch_names) - assert(ci_config) - assert(categories_enabled) - - // CI branch jobs (-ci-$branch-) (pulling check every 5 minutes) - branches_with_collections.each { branch_and_collection -> - def gz_ci_job - branch_name = branch_and_collection.branch - if (categories_enabled.contains('stable_branches')) + collection.libs.each { lib -> + def lib_name = lib.name + def branch_name = lib.repo.current_branch + def gz_job_name_prefix = lib_name.replaceAll('-','_') + if (ci_config.exclude.all?.contains(lib_name)) + return + + // Build the branch_index while going through all the libraries to avoid + // looping twice. + if (ci_config.system.so == 'linux') { + platform = distro + } else if (ci_config.system.so == 'darwin') { + platform = 'homebrew' + } else if (ci_config.system.so == 'windows') { + platform = 'windows' + } + branch_index[lib_name][platform] = branch_index[lib_name][platform]?: ['pr':[], 'pr_abichecker':[]] + if (categories_enabled.contains('pr')) + { + branch_index[lib_name][platform]['pr'].contains(branch_name) ?: + branch_index[lib_name][platform]['pr'] << [branch: branch_name, ci_name: config_name] + } + if (categories_enabled.contains('pr_abichecker') && + (! ci_config.exclude.abichecker?.contains(lib_name))) { + branch_index[lib_name][platform]['pr_abichecker'].contains(branch_name) ?: + branch_index[lib_name][platform]['pr_abichecker'] << [branch: branch_name, ci_name: config_name] + } + + // Generate jobs for the library entry being parsed + if (categories_enabled.contains('stable_branches')) { if (ci_config.system.so == 'linux') { gz_ci_job = job("${gz_job_name_prefix}-ci-${branch_name}-${distro}-${arch}") generate_ci_job(gz_ci_job, lib_name, branch_name, ci_config) if (categories_enabled.contains('stable_branches_asan')) { - // Generate asan jobs on Linux def gz_ci_asan_job = job("${gz_job_name_prefix}-ci_asan-${branch_name}-${distro}-${arch}") generate_asan_ci_job(gz_ci_asan_job, lib_name, branch_name, ci_config) gz_ci_asan_job.with @@ -364,8 +366,9 @@ ciconf_per_lib_index.each { lib_name, lib_configs -> scm(Globals.CRON_ON_WEEKEND) } } + logging_list['asan_ci'].add( - [collection: branch_and_collection.collection, + [collection: collection.name, job_name: gz_ci_asan_job.name]) } } else if (ci_config.system.so == 'darwin') { @@ -389,18 +392,29 @@ ciconf_per_lib_index.each { lib_name, lib_configs -> } logging_list['branch_ci'].add( - [collection: branch_and_collection.collection, + [collection: collection.name, job_name: gz_ci_job.name]) } // end of daily category enabled - } // end_of_branch + } + } +} - if (categories_enabled.contains('pr')) - { +branch_index.each { lib_name, distro_configs -> + distro_configs.each { distro, branches_with_ciconfig -> + if (branches_with_ciconfig['pr']) { + def branch_names = branches_with_ciconfig['pr'].collect { it.branch }.unique() + // Hack that assumes that pre_setup_script_hook, arch and requirements are equal on all + // ciconfigs associated to a given branch and distro combination + def config_name = branches_with_ciconfig['pr'].collect { it.ci_name }.unique()[0] + def ci_config = gz_collections_yaml.ci_configs.find{ it.name == config_name } + def script_name_prefix = cleanup_library_name(lib_name) + def gz_job_name_prefix = lib_name.replaceAll('-','_') + def arch = ci_config.system.arch + def ws_checkout_dir = lib_name if (ci_config.system.so == 'linux') { def pre_setup_script = ci_config.pre_setup_script_hook?.get(lib_name)?.join('\n') def extra_cmd = pre_setup_script ?: "" - def gz_ci_job_name = "${gz_job_name_prefix}-ci-pr_any-${distro}-${arch}" def gz_ci_any_job = job(gz_ci_job_name) OSRFLinuxCompilationAnyGitHub.create(gz_ci_any_job, @@ -427,39 +441,6 @@ ciconf_per_lib_index.each { lib_name, lib_configs -> """.stripIndent()) } // end of steps } // end of ci_any_job - - if (categories_enabled.contains('stable_branches') && \ - (! ci_config.exclude.abichecker?.contains(lib_name))) - { - // ABI branch jobs (-ci-abichecker-) for non main branches - def abi_job_name = "${gz_job_name_prefix}-abichecker-any_to_any-ubuntu-${distro}-${arch}" - def abi_job = job(abi_job_name) - OSRFLinuxABIGitHub.create(abi_job) - GenericAnyJobGitHub.create(abi_job, - "gazebosim/${lib_name}", - branch_names - [ 'main']) - generate_label_by_requirements(abi_job, lib_name, ci_config.requirements) - abi_job.with - { - steps { - shell("""\ - #!/bin/bash -xe - - export DISTRO=${distro} - - ${GLOBAL_SHELL_CMD} - ${extra_cmd} - - export ARCH=${arch} - export DEST_BRANCH=\${DEST_BRANCH:-\$ghprbTargetBranch} - export SRC_BRANCH=\${SRC_BRANCH:-\$ghprbSourceBranch} - export SRC_REPO=\${SRC_REPO:-\$ghprbAuthorRepoGitUrl} - export ABI_JOB_SOFTWARE_NAME=${lib_name} - /bin/bash -xe ./scripts/jenkins-scripts/docker/gz-abichecker.bash - """.stripIndent()) - } // end of steps - } // end of with - } } else if (ci_config.system.so == 'darwin') { // -------------------------------------------------------------- def gz_brew_ci_any_job_name = "${gz_job_name_prefix}-ci-pr_any-homebrew-amd64" @@ -484,9 +465,48 @@ ciconf_per_lib_index.each { lib_name, lib_configs -> add_win_devel_bat_call(gz_win_ci_any_job, lib_name, ws_checkout_dir) Globals.gazebodistro_branch = false } - } //end of pr enabled - } //end of lib_configs -} // end of lib + } + + if (branches_with_ciconfig['pr_abichecker']) { + def branch_names = branches_with_ciconfig['pr_abichecker'].collect { it.branch }.unique() + // Hack that assumes that pre_setup_script_hook, arch and requirements are equal on all + // ciconfigs associated to a given branch and distro combination + def config_name = branches_with_ciconfig['pr_abichecker'].collect { it.ci_name }.unique()[0] + def ci_config = gz_collections_yaml.ci_configs.find{ it.name == config_name } + def pre_setup_script = ci_config.pre_setup_script_hook?.get(lib_name)?.join('\n') + def extra_cmd = pre_setup_script ?: "" + def arch = ci_config.system.arch + def gz_job_name_prefix = lib_name.replaceAll('-','_') + def abi_job_name = "${gz_job_name_prefix}-abichecker-any_to_any-ubuntu-${distro}-${arch}" + def abi_job = job(abi_job_name) + OSRFLinuxABIGitHub.create(abi_job) + GenericAnyJobGitHub.create(abi_job, + "gazebosim/${lib_name}", + branch_names - [ 'main']) + generate_label_by_requirements(abi_job, lib_name, ci_config.requirements) + abi_job.with + { + steps { + shell("""\ + #!/bin/bash -xe + + export DISTRO=${distro} + + ${GLOBAL_SHELL_CMD} + ${extra_cmd} + + export ARCH=${arch} + export DEST_BRANCH=\${DEST_BRANCH:-\$ghprbTargetBranch} + export SRC_BRANCH=\${SRC_BRANCH:-\$ghprbSourceBranch} + export SRC_REPO=\${SRC_REPO:-\$ghprbAuthorRepoGitUrl} + export ABI_JOB_SOFTWARE_NAME=${lib_name} + /bin/bash -xe ./scripts/jenkins-scripts/docker/gz-abichecker.bash + """.stripIndent()) + } // end of steps + } // end of with + } + } +} pkgconf_per_src_index.each { pkg_src, pkg_src_configs -> // For each entry in the index perform two steps: @@ -589,7 +609,6 @@ collection_job_names.each { collection_name, job_names -> name(jobname) } def collection = gz_collections_yaml.collections.find { it.name == collection_name } - println(collection) if (collection.packaging?.linux?.nightly) { collection.libs.each { lib -> name(get_debbuilder_name(lib, collection.packaging)) From 9ba5e2701731e984800220d41eac3ef585e8b83c Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 16 May 2024 18:23:20 +0200 Subject: [PATCH 11/13] Remove old ignition.dsl: disable all -auto- fakea abicheckers (#1141) Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/ignition.dsl | 401 ------------------------------- 1 file changed, 401 deletions(-) delete mode 100644 jenkins-scripts/dsl/ignition.dsl diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl deleted file mode 100644 index 1cf0d26fd..000000000 --- a/jenkins-scripts/dsl/ignition.dsl +++ /dev/null @@ -1,401 +0,0 @@ -import _configs_.* -import javaposse.jobdsl.dsl.Job - -// IGNITION PACKAGES -gz_software = [ 'cmake', - 'common', - 'fuel-tools', - 'gazebo', - 'gui', - 'launch', - 'math', - 'msgs', - 'physics', - 'plugin', - 'rendering', - 'sensors', - 'sim', - 'tools', - 'transport', - 'utils' ] -// DESC: need gpu/display for tests -gz_gpu = [ 'gazebo', - 'gui', - 'rendering', - 'sim', - 'sensors' ] -// DESC: software does not support cmake warnings enabled -gz_no_cmake_warnings = [ 'cmake', - 'common', - 'fuel-tools', - 'gazebo', - 'gui', - 'launch', - 'math', - 'msgs', - 'physics', - 'rendering', - 'sensors', - 'tools', - 'transport', - 'utils' ] -// DESC: software does not have tests -gz_no_test = [ ] -// DESC: major series supported and released. The branches get CI, install pkg -// testing and debbuild job. -// No branches in gz_branches means no released branches (only CI on -// main, ABI check, install pkg) -ignition_branches = [ 'cmake' : [ '2' ], - 'common' : [ '1', '3', '4' ], - 'fuel-tools' : [ '1', '4', '7' ], - 'gazebo' : [ '3', '6' ], - 'gui' : [ '0', '3', '6' ], - 'launch' : [ '2', '5' ], - 'math' : [ '4', '6' ], - 'msgs' : [ '1', '5', '8' ], - 'physics' : [ '2', '5' ], - 'plugin' : [ '1' ], - 'rendering' : [ '3', '6' ], - 'sensors' : [ '3', '6' ], - 'tools' : [ '1' ], - 'transport' : [ '4', '8', '11' ], - 'utils' : [ '1' ]] - -gz_branches = [ 'cmake' : [ '3' ], - 'common' : [ '5' ], - 'fuel-tools' : [ '8', '9' ], - 'gui' : [ '7', '8' ], - 'launch' : [ '6', '7' ], - 'math' : [ '7' ], - 'msgs' : [ '9', '10'], - 'physics' : [ '6', '7' ], - 'plugin' : [ '2' ], - 'rendering' : [ '7', '8' ], - 'sensors' : [ '7', '8' ], - 'sim' : [ '7', '8' ], - 'tools' : [ '2' ], - 'transport' : [ '12', '13' ], - 'utils' : [ '2' ]] -// DESC: prerelease branches are managed as any other supported branches for -// special cases different to major branches: get compilation CI on the branch -// physics/sensors don't need to be included since they use main for gz11 -gz_prerelease_branches = [] -// DESC: versioned names to generate debbuild jobs for special cases that -// don't appear in gz_branches (like nightly builders or 0-debbuild -// jobs for the special cases of foo0 packages) -gz_extra_debbuild = [ 'cmake4', - 'common6', - 'fuel-tools10', - 'gui9', - 'launch8', - 'math8', - 'msgs11', - 'physics8', - 'plugin3', - 'rendering9', - 'sensors9', - 'sim9', - 'transport14', - 'utils3', - - 'utils1' // see comment https://github.com/gazebo-tooling/release-tools/pull/431#issuecomment-815099918 - ] -// DESC: exclude ignition from generate any install testing job -gz_no_pkg_yet = [ ] -// DESC: major versions that has a package in the prerelease repo. Should -// not appear in gz_no_pkg_yet nor in gz_branches -gz_prerelease_pkgs = [ 'placeholder' : [ - '1': [ 'bionic' ]], - ] -// packages using colcon for windows compilation while migrating all them to -// this solution -gz_colcon_win = [ 'cmake', - 'common', - 'fuel-tools', - 'gazebo', - 'gui', - 'launch', - 'math', - 'msgs', - 'physics', - 'plugin', - 'rendering', - 'sensors', - 'tools', - 'transport', - 'utils' ] - -// Main platform using for quick CI -def ci_distro = Globals.get_ci_distro() -def abi_distro = Globals.get_abi_distro() -// Other supported platform to be checked but no for quick -// CI integration. -def other_supported_distros = Globals.get_other_supported_distros() -def supported_arches = Globals.get_supported_arches() - -all_supported_distros = ci_distro + other_supported_distros - -// Map needed to be used in ci_pr -abi_job_names = [:] - -Globals.extra_emails = "caguero@osrfoundation.org" - -// shell command to inject in all bash steps -GLOBAL_SHELL_CMD='' -GITHUB_SUPPORT_ALL_BRANCHES = [] -ENABLE_GITHUB_PR_INTEGRATION = true - -String ci_distro_str = ci_distro[0] - -// Map of lists to use in CIWorkflow -ci_pr_any_list = [:] -gz_software.each { gz_sw -> - def list_empty = [] - ci_pr_any_list[gz_sw] = list_empty -} - -/** - * Deeply merges the contents of each Map in sources, merging from - * "right to left" and returning the merged Map. - * - * The source maps will not be modified. - * - * Original source code: https://gist.github.com/robhruska/4612278 - */ -Map merge_maps(Map[] sources) { - if (sources.length == 0) return [:] - if (sources.length == 1) return sources[0] - - sources.inject([:]) { result, source -> - source.each { k, v -> - result[k] = result[k] instanceof Map ? merge(result[k], v) : v - } - result - } -} - -// return major versions supported or empty if just 0,1 series under -// -dev package. -ArrayList supported_ign_branches(String ign_software) -{ - // sim was not used in ignition - if (ign_software == 'sim') - return [''] - - major_versions_registered = ignition_branches["${ign_software}"] - - if (major_versions_registered == null) - return [ '' ] - - return major_versions_registered -} - -// return major versions supported or empty if just 0,1 series under -// -dev package. -ArrayList supported_gz_branches(String gz_software) -{ - // sim was not used in ignition - if (gz_software == 'gazebo') - return [''] - - major_versions_registered = gz_branches["${gz_software}"] - - if (major_versions_registered == null) - return [ '' ] - - return major_versions_registered -} - -// return prerelease branch names -ArrayList prerelease_branches(String gz_software) -{ - pre_branches = gz_prerelease_branches["${gz_software}"] - - if (pre_branches == null) - return [ '' ] - - return pre_branches -} - -// return all ci branch names -ArrayList all_branches(String software_name) -{ - List branches = new ArrayList(); - supported_ign_branches("${software_name}").each { major_version -> - if ("${major_version}") { - branches.add("ign-${software_name}${major_version}") - } - } - prerelease_branches("${software_name}").each { branch -> - if ("${branch}") { - branches.add(branch) - } - } - if (software_name == 'gazebo') - software_name = 'sim' - supported_gz_branches("${software_name}").each { major_version -> - if ("${major_version}") { - branches.add("gz-${software_name}${major_version}") - } - } - branches.add('main') - return branches -} - - -ArrayList all_debbuilders() -{ - List branches = new ArrayList(); - // add all supported branches - gz_software.each { software_name -> - supported_ign_branches("${software_name}").each { major_version -> - if (major_version) { - // No 1-debbuild versions, they use the unversioned job - if ("${major_version}" == "0" || "${major_version}" == "1" ) - major_version = "" - - branches.add("ign-${software_name}${major_version}") - } - } - if (software_name == 'gazebo') - software_name = 'sim' - supported_gz_branches("${software_name}").each { major_version -> - if (major_version) { - // No 1-debbuild versions, they use the unversioned job - if ("${major_version}" == "0" || "${major_version}" == "1" ) - major_version = "" - - branches.add("gz-${software_name}${major_version}") - } - } - } - // add all extra debbuilders - gz_extra_debbuild.each { gz_name -> - // utils1 is still using the ign preffix - if (gz_name == 'utils1') - branches.add("ign-${gz_name}") - else - branches.add("gz-${gz_name}") - } - - return branches -} - -// return all ci branch names -// Map with the form of: major versions as keys. -// Lists of distros supported as values -Map supported_install_pkg_branches(String gz_software) -{ - major_versions_prerelease = gz_prerelease_pkgs["${gz_software}"] - - // construct a map of stable packages based on supported_gz_branches and - // all_supported_distros - map_of_stable_versions = [:] - map_of_stable_versions[gz_software] = [:] - (supported_ign_branches(gz_software) + supported_gz_branches(gz_software)).each { major_version -> - if (! major_version) - return false // keep looping - new_relation = [:] - new_relation[major_version] = all_supported_distros - map_of_stable_versions[gz_software] << new_relation - } - - if (major_versions_prerelease == null) - return map_of_stable_versions[gz_software] - - return merge_maps(map_of_stable_versions[gz_software], - major_versions_prerelease) -} - -void include_gpu_label_if_needed(Job job, String gz_software_name) -{ - job.with - { - - gz_gpu.each { gz_each -> - if (gz_software_name == gz_each) - { - label Globals.nontest_label("gpu-reliable") - - // unstable build if missing valid gpu display - publishers { - consoleParsing { - projectRules('scripts/jenkins-scripts/parser_rules/display_missing.parser') - unstableOnWarning() - failBuildOnError(false) - } - } - } - } - } -} - -boolean enable_cmake_warnings(String gz_software_name) -{ - if (gz_software_name in gz_no_cmake_warnings) - return false - - return true -} - -boolean enable_testing(String gz_software_name) -{ - if (gz_software_name in gz_no_test) - return false - - return true -} - -boolean is_a_colcon_package(String gz_software_name) -{ - if (gz_software_name in gz_colcon_win) - return true - - return false -} - -// Need to be before the ci-pr_any so the abi job name is defined -gz_software.each { gz_sw -> - supported_arches.each { arch -> - // 1 Per library and per linux arch - // 1.1 DEPRECATED Per abi_distro - // 1.1.1 [job] ABI checker for main branches - // 1.2 DEPRECATED Per ci_str_distro - // 1.2.1 [job] Main PR jobs (-ci-pr_any-) - // 1.3 Per all supported_distros - // 1.3.1 Per all supported branches on each library - // 1.3.1.1 [job] Branch jobs -ci-$branch- - - // 1.1.1 ABI checker for main branches - // -------------------------------------------------------------- - abi_distro.each { distro -> - // Packages without ABI - if (gz_sw == 'tools' || gz_sw == 'cmake') - return - - software_name = gz_sw // Necessary substitution. gz_sw won't overwrite - - if (gz_sw == 'sim') - software_name = "gazebo" - - abi_job_names[software_name] = "ignition_${software_name}-abichecker-any_to_any-ubuntu_auto-${arch}" - def abi_job = job(abi_job_names[software_name]) - GenericAnyJobGitHub.create(abi_job, - "gazebosim/ign-${software_name}", - all_branches(software_name) - [ 'main']) - abi_job.with - { - description 'Automatic generated job by DSL jenkins. Stub job for migration, not doing any check' - } // end of with - } // end of abi_distro - } // end of arch -} // end of gz_software - -// Main CI workflow -gz_software.each { gz_sw -> - if (gz_sw == 'sim') - return - def String ci_main_name = "ignition_${gz_sw}-ci-manual_any" - def gz_ci_main = pipelineJob(ci_main_name) - OSRFCIWorkFlowMultiAnyGitHub.create(gz_ci_main, ci_pr_any_list[gz_sw]) -} From c8caafe9b22795d7faa2de4fccbd7c8c70a05eb0 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 16 May 2024 18:23:39 +0200 Subject: [PATCH 12/13] Fix check for pub/sub in unofficial wrappers (#1140) * Fix check for pub/sub in unofficial wrappers * Rework the way of giving time to transport layers --------- Signed-off-by: Jose Luis Rivero --- jenkins-scripts/docker/ros_gz-install-test-job.bash | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/jenkins-scripts/docker/ros_gz-install-test-job.bash b/jenkins-scripts/docker/ros_gz-install-test-job.bash index 274356136..9f883784a 100644 --- a/jenkins-scripts/docker/ros_gz-install-test-job.bash +++ b/jenkins-scripts/docker/ros_gz-install-test-job.bash @@ -18,12 +18,15 @@ export INSTALL_JOB_POSTINSTALL_HOOK=""" echo '# BEGIN SECTION: ros_gz talker/listener' ros2 run ros_gz_bridge parameter_bridge /chatter@std_msgs/msg/String@gz.msgs.StringMsg 2>/dev/null & -sleep 1 +ros2 topic info /chatter +# ros2 listener ros2 topic echo /chatter > /tmp/echo_chatter & -sleep 1 +# print debug info the chatter topic +gz topic -i -t /chatter +# gz transport talker gz topic -t /chatter -m gz.msgs.StringMsg -p 'data:\"Hello\"' & -sleep 1 -if ! grep -v Hello /tmp/echo_chatter; then +sleep 2 +if ! grep -q Hello /tmp/echo_chatter; then echo 'chatter log file does not contain the expected Hello string' exit 1 fi From cf3b6dedc711e5bcd659e65eab239962b1fc6096 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 17 May 2024 16:14:17 +0200 Subject: [PATCH 13/13] Use noble on Ionic and __upcoming__ instead of Jammy for PR and daily (#1138) Replace values in gz-collections.yaml to use right platform --------- Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/gz-collections.yaml | 58 +++++++++++++++--- jenkins-scripts/dsl/logs/generated_jobs.txt | 68 ++++++++++----------- 2 files changed, 84 insertions(+), 42 deletions(-) diff --git a/jenkins-scripts/dsl/gz-collections.yaml b/jenkins-scripts/dsl/gz-collections.yaml index f91dee7e5..fdf0ba837 100644 --- a/jenkins-scripts/dsl/gz-collections.yaml +++ b/jenkins-scripts/dsl/gz-collections.yaml @@ -338,7 +338,7 @@ collections: ci: configs: - jammy - - noble + - noble_no_pr - brew - win packaging: @@ -346,7 +346,7 @@ collections: - jammy linux: ignore_major_version: - - gz-harmonic + - gz-harmonic - name: 'ionic' libs: - name: gz-cmake @@ -415,18 +415,18 @@ collections: current_branch: main ci: configs: - - jammy + - noble - brew - win packaging: configs: - - jammy + - noble linux: ignore_major_version: - gz-ionic nightly: distros: - - jammy + - noble - name: '__upcoming__' libs: - name: gz-tools @@ -435,12 +435,12 @@ collections: current_branch: main ci: configs: - - jammy + - noble - brew - win packaging: configs: - - jammy + - noble ci_configs: - name: focal system: @@ -507,7 +507,37 @@ ci_configs: system: so: linux distribution: ubuntu - version: noble + version: noble + arch: amd64 + requirements: + large_memory: + - gz-physics + nvidia_gpu: + - gz-sim + - gz-gui + - gz-rendering + - gz-sensors + exclude: + all: + - gz-harmonic + - __upcoming__ + abichecker: + - gz-cmake + - gz-tools + pre_setup_script_hook: + gz-physics: + - "export MAKE_JOBS=1" + tests_disabled: + ci_categories_enabled: + - pr + - pr_abichecker + - stable_branches + - stable_branches_asan + - name: noble_no_pr + system: + so: linux + distribution: ubuntu + version: noble arch: amd64 requirements: large_memory: @@ -650,3 +680,15 @@ packaging_configs: - "export MAKE_JOBS=1" exclude: - __upcoming__ + - name: noble + system: + so: linux + distribution: ubuntu + version: jammy + arch: + - amd64 + pre_setup_script_hook: + gz-physics: + - "export MAKE_JOBS=1" + exclude: + - __upcoming__ diff --git a/jenkins-scripts/dsl/logs/generated_jobs.txt b/jenkins-scripts/dsl/logs/generated_jobs.txt index 38eb393fc..bfbf662a7 100644 --- a/jenkins-scripts/dsl/logs/generated_jobs.txt +++ b/jenkins-scripts/dsl/logs/generated_jobs.txt @@ -1,4 +1,4 @@ -asan_ci __upcoming__ gz_tools-ci_asan-main-jammy-amd64 +asan_ci __upcoming__ gz_tools-ci_asan-main-noble-amd64 asan_ci citadel gz_cmake-ci_asan-ign-cmake2-focal-amd64 asan_ci citadel gz_common-ci_asan-ign-common3-focal-amd64 asan_ci citadel gz_fuel_tools-ci_asan-ign-fuel-tools4-focal-amd64 @@ -62,24 +62,24 @@ asan_ci harmonic gz_tools-ci_asan-gz-tools2-jammy-amd64 asan_ci harmonic gz_transport-ci_asan-gz-transport13-jammy-amd64 asan_ci harmonic gz_utils-ci_asan-gz-utils2-jammy-amd64 asan_ci harmonic sdformat-ci_asan-sdf14-jammy-amd64 -asan_ci ionic gz_cmake-ci_asan-main-jammy-amd64 -asan_ci ionic gz_common-ci_asan-main-jammy-amd64 -asan_ci ionic gz_fuel_tools-ci_asan-main-jammy-amd64 -asan_ci ionic gz_gui-ci_asan-main-jammy-amd64 -asan_ci ionic gz_launch-ci_asan-main-jammy-amd64 -asan_ci ionic gz_math-ci_asan-main-jammy-amd64 -asan_ci ionic gz_msgs-ci_asan-main-jammy-amd64 -asan_ci ionic gz_physics-ci_asan-main-jammy-amd64 -asan_ci ionic gz_plugin-ci_asan-main-jammy-amd64 -asan_ci ionic gz_rendering-ci_asan-main-jammy-amd64 -asan_ci ionic gz_sensors-ci_asan-main-jammy-amd64 -asan_ci ionic gz_sim-ci_asan-main-jammy-amd64 -asan_ci ionic gz_tools-ci_asan-gz-tools2-jammy-amd64 -asan_ci ionic gz_transport-ci_asan-main-jammy-amd64 -asan_ci ionic gz_utils-ci_asan-main-jammy-amd64 -asan_ci ionic sdformat-ci_asan-main-jammy-amd64 +asan_ci ionic gz_cmake-ci_asan-main-noble-amd64 +asan_ci ionic gz_common-ci_asan-main-noble-amd64 +asan_ci ionic gz_fuel_tools-ci_asan-main-noble-amd64 +asan_ci ionic gz_gui-ci_asan-main-noble-amd64 +asan_ci ionic gz_launch-ci_asan-main-noble-amd64 +asan_ci ionic gz_math-ci_asan-main-noble-amd64 +asan_ci ionic gz_msgs-ci_asan-main-noble-amd64 +asan_ci ionic gz_physics-ci_asan-main-noble-amd64 +asan_ci ionic gz_plugin-ci_asan-main-noble-amd64 +asan_ci ionic gz_rendering-ci_asan-main-noble-amd64 +asan_ci ionic gz_sensors-ci_asan-main-noble-amd64 +asan_ci ionic gz_sim-ci_asan-main-noble-amd64 +asan_ci ionic gz_tools-ci_asan-gz-tools2-noble-amd64 +asan_ci ionic gz_transport-ci_asan-main-noble-amd64 +asan_ci ionic gz_utils-ci_asan-main-noble-amd64 +asan_ci ionic sdformat-ci_asan-main-noble-amd64 branch_ci __upcoming__ gz_tools-ci-main-homebrew-amd64 -branch_ci __upcoming__ gz_tools-ci-main-jammy-amd64 +branch_ci __upcoming__ gz_tools-ci-main-noble-amd64 branch_ci __upcoming__ gz_tools-main-win branch_ci citadel gz_cmake-ci-ign-cmake2-focal-amd64 branch_ci citadel gz_cmake-ci-ign-cmake2-homebrew-amd64 @@ -285,52 +285,52 @@ branch_ci harmonic sdformat-ci-sdf14-jammy-amd64 branch_ci harmonic sdformat-ci-sdf14-noble-amd64 branch_ci harmonic sdformat-sdf14-win branch_ci ionic gz_cmake-ci-main-homebrew-amd64 -branch_ci ionic gz_cmake-ci-main-jammy-amd64 +branch_ci ionic gz_cmake-ci-main-noble-amd64 branch_ci ionic gz_cmake-main-win branch_ci ionic gz_common-ci-main-homebrew-amd64 -branch_ci ionic gz_common-ci-main-jammy-amd64 +branch_ci ionic gz_common-ci-main-noble-amd64 branch_ci ionic gz_common-main-win branch_ci ionic gz_fuel_tools-ci-main-homebrew-amd64 -branch_ci ionic gz_fuel_tools-ci-main-jammy-amd64 +branch_ci ionic gz_fuel_tools-ci-main-noble-amd64 branch_ci ionic gz_fuel_tools-main-win branch_ci ionic gz_gui-ci-main-homebrew-amd64 -branch_ci ionic gz_gui-ci-main-jammy-amd64 +branch_ci ionic gz_gui-ci-main-noble-amd64 branch_ci ionic gz_gui-main-win branch_ci ionic gz_launch-ci-main-homebrew-amd64 -branch_ci ionic gz_launch-ci-main-jammy-amd64 +branch_ci ionic gz_launch-ci-main-noble-amd64 branch_ci ionic gz_launch-main-win branch_ci ionic gz_math-ci-main-homebrew-amd64 -branch_ci ionic gz_math-ci-main-jammy-amd64 +branch_ci ionic gz_math-ci-main-noble-amd64 branch_ci ionic gz_math-main-win branch_ci ionic gz_msgs-ci-main-homebrew-amd64 -branch_ci ionic gz_msgs-ci-main-jammy-amd64 +branch_ci ionic gz_msgs-ci-main-noble-amd64 branch_ci ionic gz_msgs-main-win branch_ci ionic gz_physics-ci-main-homebrew-amd64 -branch_ci ionic gz_physics-ci-main-jammy-amd64 +branch_ci ionic gz_physics-ci-main-noble-amd64 branch_ci ionic gz_physics-main-win branch_ci ionic gz_plugin-ci-main-homebrew-amd64 -branch_ci ionic gz_plugin-ci-main-jammy-amd64 +branch_ci ionic gz_plugin-ci-main-noble-amd64 branch_ci ionic gz_plugin-main-win branch_ci ionic gz_rendering-ci-main-homebrew-amd64 -branch_ci ionic gz_rendering-ci-main-jammy-amd64 +branch_ci ionic gz_rendering-ci-main-noble-amd64 branch_ci ionic gz_rendering-main-win branch_ci ionic gz_sensors-ci-main-homebrew-amd64 -branch_ci ionic gz_sensors-ci-main-jammy-amd64 +branch_ci ionic gz_sensors-ci-main-noble-amd64 branch_ci ionic gz_sensors-main-win branch_ci ionic gz_sim-ci-main-homebrew-amd64 -branch_ci ionic gz_sim-ci-main-jammy-amd64 +branch_ci ionic gz_sim-ci-main-noble-amd64 branch_ci ionic gz_sim-main-win branch_ci ionic gz_tools-2-win branch_ci ionic gz_tools-ci-gz-tools2-homebrew-amd64 -branch_ci ionic gz_tools-ci-gz-tools2-jammy-amd64 +branch_ci ionic gz_tools-ci-gz-tools2-noble-amd64 branch_ci ionic gz_transport-ci-main-homebrew-amd64 -branch_ci ionic gz_transport-ci-main-jammy-amd64 +branch_ci ionic gz_transport-ci-main-noble-amd64 branch_ci ionic gz_transport-main-win branch_ci ionic gz_utils-ci-main-homebrew-amd64 -branch_ci ionic gz_utils-ci-main-jammy-amd64 +branch_ci ionic gz_utils-ci-main-noble-amd64 branch_ci ionic gz_utils-main-win branch_ci ionic sdformat-ci-main-homebrew-amd64 -branch_ci ionic sdformat-ci-main-jammy-amd64 +branch_ci ionic sdformat-ci-main-noble-amd64 branch_ci ionic sdformat-main-win install_ci __upcoming__ gz_tools3-install-pkg-jammy-amd64 install_ci __upcoming__ gz_tools3-install_bottle-homebrew-amd64