From 56a2eb9e4b8b1260962f7d255e93f81092f631f8 Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Tue, 12 Oct 2021 13:51:53 -0700 Subject: [PATCH 001/540] brew: set cmake args for keg-only qwt-qt5 (#535) Signed-off-by: Steve Peters --- .../lib/project-default-devel-homebrew-amd64.bash | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash b/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash index 0d8f1fd61..0a64789da 100644 --- a/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash +++ b/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash @@ -110,10 +110,15 @@ export DISPLAY=$(ps ax \ | sed -e 's@.*Xquartz @@' -e 's@ .*@@' ) +CMAKE_ARGS="" # set CMAKE_PREFIX_PATH if we are using qt@5 if brew ruby -e "exit ! '${PROJECT_FORMULA}'.f.recursive_dependencies.map(&:name).keep_if { |d| d == 'qt@5' }.empty?"; then export CMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH}:/usr/local/opt/qt@5 fi +# set cmake args if we are using qwt-qt5 +if brew ruby -e "exit ! '${PROJECT_FORMULA}'.f.recursive_dependencies.map(&:name).keep_if { |d| d == 'qwt-qt5' }.empty?"; then + CMAKE_ARGS='-DQWT_WIN_INCLUDE_DIR=/usr/local/opt/qwt-qt5/lib/qwt.framework/Headers -DQWT_WIN_LIBRARY_DIR=/usr/local/opt/qwt-qt5/lib' +fi # Workaround for tbb@2020_u3: set CPATH, LIBRARY_PATH, and CMAKE_PREFIX_PATH if brew ruby -e "exit ! '${PROJECT_FORMULA}'.f.recursive_dependencies.map(&:name).keep_if { |d| d == 'tbb@2020_u3' }.empty?"; then export CMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH}:/usr/local/opt/tbb@2020_u3 @@ -143,6 +148,7 @@ fi cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo \ -DCMAKE_INSTALL_PREFIX=/usr/local/Cellar/${PROJECT_FORMULA}/HEAD \ + ${CMAKE_ARGS} \ ${WORKSPACE}/${PROJECT_PATH} echo '# END SECTION' From 816e6c5ccef0a3832dd3dd94d867374d00f6d4d3 Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Thu, 14 Oct 2021 09:11:08 -0700 Subject: [PATCH 002/540] ignition.dsl: colcon for ign-msgs on windows (#536) Signed-off-by: Steve Peters --- jenkins-scripts/dsl/ignition.dsl | 1 + jenkins-scripts/dsl/ignition_collection.dsl | 8 ++++---- jenkins-scripts/ign_msgs-default-devel-windows-amd64.bat | 4 +++- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index 7667e0312..8c2accded 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -95,6 +95,7 @@ ignition_colcon_win = [ 'common', 'gazebo', 'gui', 'launch', + 'msgs', 'physics', 'plugin', 'rendering', diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index 5d3b91507..7ff87927b 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -49,6 +49,7 @@ ignition_collection_jobs = 'ign_fuel-tools-ign-4-win', 'ign_gazebo-ign-3-win', 'ign_gui-ign-3-win', + 'ign_msgs-ign-5-win', 'ign_physics-ign-2-win', 'ign_plugin-ign-1-win', 'ign_rendering-ign-3-win', @@ -89,7 +90,6 @@ ignition_collection_jobs = 'ignition_math6-install_bottle-homebrew-amd64', 'ignition_msgs-ci-ign-msgs5-bionic-amd64', 'ignition_msgs-ci-ign-msgs5-homebrew-amd64', - 'ignition_msgs-ci-ign-msgs5-windows7-amd64', 'ignition_msgs5-install-pkg-bionic-amd64', 'ignition_msgs5-install_bottle-homebrew-amd64', 'ignition_physics-ci-ign-physics2-bionic-amd64', @@ -127,6 +127,7 @@ ignition_collection_jobs = 'ign_fuel-tools-ign-5-win', 'ign_gazebo-ign-4-win', 'ign_gui-ign-4-win', + 'ign_msgs-ign-6-win', 'ign_physics-ign-3-win', 'ign_plugin-ign-1-win', 'ign_rendering-ign-4-win', @@ -167,7 +168,6 @@ ignition_collection_jobs = 'ignition_math6-install_bottle-homebrew-amd64', 'ignition_msgs-ci-ign-msgs6-bionic-amd64', 'ignition_msgs-ci-ign-msgs6-homebrew-amd64', - 'ignition_msgs-ci-ign-msgs6-windows7-amd64', 'ignition_msgs6-install-pkg-bionic-amd64', 'ignition_msgs6-install_bottle-homebrew-amd64', 'ignition_physics-ci-ign-physics3-bionic-amd64', @@ -205,6 +205,7 @@ ignition_collection_jobs = 'ign_fuel-tools-ign-6-win', 'ign_gazebo-ign-5-win', 'ign_gui-ign-5-win', + 'ign_msgs-ign-7-win', 'ign_physics-ign-4-win', 'ign_plugin-ign-1-win', 'ign_rendering-ign-5-win', @@ -246,7 +247,6 @@ ignition_collection_jobs = 'ignition_math6-install_bottle-homebrew-amd64', 'ignition_msgs-ci-ign-msgs7-bionic-amd64', 'ignition_msgs-ci-ign-msgs7-homebrew-amd64', - 'ignition_msgs-ci-ign-msgs7-windows7-amd64', 'ignition_msgs7-install-pkg-bionic-amd64', 'ignition_msgs7-install_bottle-homebrew-amd64', 'ignition_physics-ci-ign-physics4-bionic-amd64', @@ -289,6 +289,7 @@ ignition_collection_jobs = 'ign_gazebo-ign-6-win', 'ign_gui-ign-6-win', 'ign_launch-ign-5-win', + 'ign_msgs-ign-8-win', 'ign_physics-ign-5-win', 'ign_plugin-ign-1-win', 'ign_rendering-ign-6-win', @@ -330,7 +331,6 @@ ignition_collection_jobs = 'ignition_math6-install_bottle-homebrew-amd64', 'ignition_msgs-ci-ign-msgs8-bionic-amd64', 'ignition_msgs-ci-ign-msgs8-homebrew-amd64', - 'ignition_msgs-ci-ign-msgs8-windows7-amd64', 'ignition_msgs8-install-pkg-bionic-amd64', 'ignition_msgs8-install_bottle-homebrew-amd64', 'ignition_physics-ci-ign-physics5-bionic-amd64', diff --git a/jenkins-scripts/ign_msgs-default-devel-windows-amd64.bat b/jenkins-scripts/ign_msgs-default-devel-windows-amd64.bat index 82417524b..702dcb618 100644 --- a/jenkins-scripts/ign_msgs-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_msgs-default-devel-windows-amd64.bat @@ -5,5 +5,7 @@ set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true set DEPEN_PKGS=protobuf tinyxml2 +set COLCON_PACKAGE=ignition-msgs +set COLCON_AUTO_MAJOR_VERSION=true -call "%SCRIPT_DIR%/lib/generic-default-devel-windows.bat" +call "%SCRIPT_DIR%\lib\colcon-default-devel-windows.bat" From bbe94871ea4c125f418bdabda3ff988f052799a6 Mon Sep 17 00:00:00 2001 From: Louise Poubel Date: Wed, 20 Oct 2021 16:39:55 -0700 Subject: [PATCH 003/540] Windows CI for ign-gazebo6 (#539) Signed-off-by: Louise Poubel --- jenkins-scripts/dsl/ignition.dsl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index 8c2accded..a5e6abc44 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -610,7 +610,7 @@ ignition_software.each { ign_sw -> // ign-gazebo only support Windows from ign-gazebo5 if (ign_sw == 'gazebo') - supported_branches = [ 'ign-gazebo5', 'main' ] + supported_branches = [ 'ign-gazebo5', 'ign-gazebo6', 'main' ] // ign-launch only support Windows from ign-launch5 if (ign_sw == 'launch') From d83d624452c59062b280c094b71218ad1336aab6 Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Mon, 25 Oct 2021 10:38:14 -0700 Subject: [PATCH 004/540] ignition.dsl: colcon for ign-tools on windows (#537) * enable tests in ign-tools Signed-off-by: Steve Peters --- jenkins-scripts/dsl/ignition.dsl | 3 ++- jenkins-scripts/dsl/ignition_collection.dsl | 8 ++++---- jenkins-scripts/ign_tools-default-devel-windows-amd64.bat | 6 ++++-- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index a5e6abc44..617b8061c 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -38,7 +38,7 @@ ignition_no_cmake_warnings = [ 'cmake', 'transport', 'utils' ] // DESC: software does not have tests -ignition_no_test = [ 'tools' ] +ignition_no_test = [ ] // DESC: major series supported and released. The branches get CI, install pkg // testing and debbuild job. // No branches in ignition_branches means no released branches (only CI on @@ -100,6 +100,7 @@ ignition_colcon_win = [ 'common', 'plugin', 'rendering', 'sensors', + 'tools', 'transport', 'utils' ] diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index 7ff87927b..d754245f4 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -54,6 +54,7 @@ ignition_collection_jobs = 'ign_plugin-ign-1-win', 'ign_rendering-ign-3-win', 'ign_sensors-ign-3-win', + 'ign_tools-ign-1-win', 'ign_transport-ign-8-win', 'ignition_citadel-ci-main-homebrew-amd64', 'ignition_citadel-install-pkg-bionic-amd64', @@ -110,7 +111,6 @@ ignition_collection_jobs = 'ignition_sensors3-install_bottle-homebrew-amd64', 'ignition_tools-ci-ign-tools1-bionic-amd64', 'ignition_tools-ci-ign-tools1-homebrew-amd64', - 'ignition_tools-ci-ign-tools1-windows7-amd64', 'ignition_tools-install-pkg-bionic-amd64', 'ignition_tools1-install_bottle-homebrew-amd64', 'ignition_transport-ci-ign-transport8-bionic-amd64', @@ -132,6 +132,7 @@ ignition_collection_jobs = 'ign_plugin-ign-1-win', 'ign_rendering-ign-4-win', 'ign_sensors-ign-4-win', + 'ign_tools-ign-1-win', 'ign_transport-ign-9-win', 'ignition_cmake-ci-ign-cmake2-bionic-amd64', 'ignition_cmake-ci-ign-cmake2-homebrew-amd64', @@ -188,7 +189,6 @@ ignition_collection_jobs = 'ignition_sensors4-install_bottle-homebrew-amd64', 'ignition_tools-ci-ign-tools1-bionic-amd64', 'ignition_tools-ci-ign-tools1-homebrew-amd64', - 'ignition_tools-ci-ign-tools1-windows7-amd64', 'ignition_tools-install-pkg-bionic-amd64', 'ignition_tools1-install_bottle-homebrew-amd64', 'ignition_transport-ci-ign-transport9-bionic-amd64', @@ -210,6 +210,7 @@ ignition_collection_jobs = 'ign_plugin-ign-1-win', 'ign_rendering-ign-5-win', 'ign_sensors-ign-5-win', + 'ign_tools-ign-1-win', 'ign_transport-ign-10-win', 'ign_utils-ign-1-win', 'ignition_cmake-ci-ign-cmake2-bionic-amd64', @@ -267,7 +268,6 @@ ignition_collection_jobs = 'ignition_sensors5-install_bottle-homebrew-amd64', 'ignition_tools-ci-ign-tools1-bionic-amd64', 'ignition_tools-ci-ign-tools1-homebrew-amd64', - 'ignition_tools-ci-ign-tools1-windows7-amd64', 'ignition_tools-install-pkg-bionic-amd64', 'ignition_tools1-install_bottle-homebrew-amd64', 'ignition_transport-ci-ign-transport10-bionic-amd64', @@ -294,6 +294,7 @@ ignition_collection_jobs = 'ign_plugin-ign-1-win', 'ign_rendering-ign-6-win', 'ign_sensors-ign-6-win', + 'ign_tools-ign-1-win', 'ign_transport-ign-11-win', 'ign_utils-ign-1-win', 'ignition_cmake-ci-ign-cmake2-bionic-amd64', @@ -351,7 +352,6 @@ ignition_collection_jobs = 'ignition_sensors6-install_bottle-homebrew-amd64', 'ignition_tools-ci-ign-tools1-bionic-amd64', 'ignition_tools-ci-ign-tools1-homebrew-amd64', - 'ignition_tools-ci-ign-tools1-windows7-amd64', 'ignition_tools-install-pkg-bionic-amd64', 'ignition_tools1-install_bottle-homebrew-amd64', 'ignition_transport-ci-ign-transport11-bionic-amd64', diff --git a/jenkins-scripts/ign_tools-default-devel-windows-amd64.bat b/jenkins-scripts/ign_tools-default-devel-windows-amd64.bat index 579e24bf0..de4940990 100644 --- a/jenkins-scripts/ign_tools-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_tools-default-devel-windows-amd64.bat @@ -3,6 +3,8 @@ set SCRIPT_DIR=%~dp0 set VCS_DIRECTORY=ign-tools set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true -set ENABLE_TESTS=false -call "%SCRIPT_DIR%/lib/generic-default-devel-windows.bat" +set COLCON_PACKAGE=ignition-tools +set COLCON_AUTO_MAJOR_VERSION=true + +call "%SCRIPT_DIR%\lib\colcon-default-devel-windows.bat" From 1798c648f4e47f4c683ada43c6b4d96805013675 Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Mon, 25 Oct 2021 10:59:53 -0700 Subject: [PATCH 005/540] ign-tools windows CI: don't major version number (#540) The ignition-tools cmake package name doesn't include its major version, so disable COLCON_AUTO_MAJOR_VERSION Signed-off-by: Steve Peters --- jenkins-scripts/ign_tools-default-devel-windows-amd64.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/ign_tools-default-devel-windows-amd64.bat b/jenkins-scripts/ign_tools-default-devel-windows-amd64.bat index de4940990..2bb9073d1 100644 --- a/jenkins-scripts/ign_tools-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_tools-default-devel-windows-amd64.bat @@ -5,6 +5,6 @@ set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true set COLCON_PACKAGE=ignition-tools -set COLCON_AUTO_MAJOR_VERSION=true +set COLCON_AUTO_MAJOR_VERSION=false call "%SCRIPT_DIR%\lib\colcon-default-devel-windows.bat" From 12aacc89ad3aee7cd279bec1009129fede5bc861 Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Tue, 2 Nov 2021 17:17:35 -0700 Subject: [PATCH 006/540] Fix debbuild names for nightly scheduler (#542) Signed-off-by: Steve Peters --- jenkins-scripts/dsl/ignition_collection.dsl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index d754245f4..8109a495b 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -21,10 +21,10 @@ ignition_collections = [ // These are the branches currently targeted at the upcoming collection // They're in topological order nightly_jobs: [ - 'tools' : [ debbuild: 'ign-tools1' , branch: 'ign-tools1' ], + 'tools' : [ debbuild: 'ign-tools' , branch: 'ign-tools1' ], 'cmake' : [ debbuild: 'ign-cmake2' , branch: 'ign-cmake2' ], 'math' : [ debbuild: 'ign-math6' , branch: 'ign-math6' ], - 'plugin' : [ debbuild: 'ign-plugin1' , branch: 'ign-plugin1' ], + 'plugin' : [ debbuild: 'ign-plugin' , branch: 'ign-plugin1' ], 'utils' : [ debbuild: 'ign-utils1' , branch: 'ign-utils1' ], 'common' : [ debbuild: 'ign-common4' , branch: 'ign-common4' ], 'msgs' : [ debbuild: 'ign-msgs8' , branch: 'ign-msgs8' ], From a233252bd7bb2f6656166d829b27bc904181c800 Mon Sep 17 00:00:00 2001 From: Louise Poubel Date: Tue, 2 Nov 2021 18:06:42 -0700 Subject: [PATCH 007/540] Script to help check unreleased changes for each collection Signed-off-by: Louise Poubel --- terminal-dashboard/unreleased.bash | 118 +++++++++++++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100755 terminal-dashboard/unreleased.bash diff --git a/terminal-dashboard/unreleased.bash b/terminal-dashboard/unreleased.bash new file mode 100755 index 000000000..cbc06867e --- /dev/null +++ b/terminal-dashboard/unreleased.bash @@ -0,0 +1,118 @@ +#!/bin/bash + +# Helper script to check if there are unreleased changes for supported versions. +# The script prints a list of links that can be clicked to see if there are new +# changes since the latest release. +# +# Arguments +# +# 1. : Required: A supported collection, i.e. "citadel", "fortesss", etc. +# +# Usage +# +# bash table.bash +# +# For example +# +# bash table.bash edifice + +COLLECTION=$1 + +PACKAGE_REPO=${2:-stable} + +if [ "$COLLECTION" = "citadel" ]; then + LIBS=( + "ignition-cmake2" + "ignition-math6" + "ignition-tools" + "ignition-common3" + "ignition-msgs5" + "ignition-transport8" + "ignition-fuel-tools4" + "ignition-plugin" + "ignition-rendering3" + "sdformat9" + "ignition-physics2" + "ignition-sensors3" + "ignition-gui3" + "ignition-gazebo3" + "ignition-launch2" + ) +elif [ "$COLLECTION" = "dome" ]; then + LIBS=( + "ignition-cmake2" + "ignition-math6" + "ignition-tools" + "ignition-common3" + "ignition-msgs6" + "ignition-transport9" + "ignition-fuel-tools5" + "ignition-plugin" + "ignition-rendering4" + "sdformat10" + "ignition-physics3" + "ignition-sensors4" + "ignition-gui4" + "ignition-gazebo4" + "ignition-launch3" + ) +elif [ "$COLLECTION" = "edifice" ]; then + LIBS=( + "ignition-cmake2" + "ignition-math6" + "ignition-utils1" + "ignition-tools" + "ignition-common4" + "ignition-msgs7" + "ignition-transport10" + "ignition-fuel-tools6" + "ignition-plugin" + "ignition-rendering5" + "sdformat11" + "ignition-physics4" + "ignition-sensors5" + "ignition-gui5" + "ignition-gazebo5" + "ignition-launch4" + ) +elif [ "$COLLECTION" = "fortress" ]; then + LIBS=( + "ignition-cmake2" + "ignition-math6" + "ignition-utils1" + "ignition-tools" + "ignition-common4" + "ignition-msgs8" + "ignition-transport11" + "ignition-fuel-tools7" + "ignition-plugin" + "ignition-rendering6" + "sdformat12" + "ignition-physics5" + "ignition-sensors6" + "ignition-gui6" + "ignition-gazebo6" + "ignition-launch5" + ) +else + echo "Missing collection" + exit +fi + +for LIB in "${LIBS[@]}" +do + LIB_SHORT="${LIB/ignition-/ign-}" + LIB_NAME=${LIB_SHORT//[[:digit:]]/} + + PKG_VERSION=$(wget -qO- http://packages.osrfoundation.org/gazebo/ubuntu-${PACKAGE_REPO}/dists/focal/main/binary-amd64/Packages | \ + grep -1 -m 1 -e "Source: ${LIB}" -e "Package: ${LIB}" | \ + sed -n 's/^Version: \(.*\)/\1/p' | uniq) + + PKG_VERSION=${PKG_VERSION%%~*} + + COMPACT_VERSION=${PKG_VERSION#* } + COMPACT_VERSION=${COMPACT_VERSION%%-*} + + echo https://github.com/ignitionrobotics/${LIB_NAME}/compare/${LIB}_${COMPACT_VERSION}...${LIB_SHORT} + +done From 02ccf226327c83b42410eb29f5c32163a36eb131 Mon Sep 17 00:00:00 2001 From: Louise Poubel Date: Tue, 2 Nov 2021 18:07:30 -0700 Subject: [PATCH 008/540] Revert mistaken push Signed-off-by: Louise Poubel --- terminal-dashboard/unreleased.bash | 118 ----------------------------- 1 file changed, 118 deletions(-) delete mode 100755 terminal-dashboard/unreleased.bash diff --git a/terminal-dashboard/unreleased.bash b/terminal-dashboard/unreleased.bash deleted file mode 100755 index cbc06867e..000000000 --- a/terminal-dashboard/unreleased.bash +++ /dev/null @@ -1,118 +0,0 @@ -#!/bin/bash - -# Helper script to check if there are unreleased changes for supported versions. -# The script prints a list of links that can be clicked to see if there are new -# changes since the latest release. -# -# Arguments -# -# 1. : Required: A supported collection, i.e. "citadel", "fortesss", etc. -# -# Usage -# -# bash table.bash -# -# For example -# -# bash table.bash edifice - -COLLECTION=$1 - -PACKAGE_REPO=${2:-stable} - -if [ "$COLLECTION" = "citadel" ]; then - LIBS=( - "ignition-cmake2" - "ignition-math6" - "ignition-tools" - "ignition-common3" - "ignition-msgs5" - "ignition-transport8" - "ignition-fuel-tools4" - "ignition-plugin" - "ignition-rendering3" - "sdformat9" - "ignition-physics2" - "ignition-sensors3" - "ignition-gui3" - "ignition-gazebo3" - "ignition-launch2" - ) -elif [ "$COLLECTION" = "dome" ]; then - LIBS=( - "ignition-cmake2" - "ignition-math6" - "ignition-tools" - "ignition-common3" - "ignition-msgs6" - "ignition-transport9" - "ignition-fuel-tools5" - "ignition-plugin" - "ignition-rendering4" - "sdformat10" - "ignition-physics3" - "ignition-sensors4" - "ignition-gui4" - "ignition-gazebo4" - "ignition-launch3" - ) -elif [ "$COLLECTION" = "edifice" ]; then - LIBS=( - "ignition-cmake2" - "ignition-math6" - "ignition-utils1" - "ignition-tools" - "ignition-common4" - "ignition-msgs7" - "ignition-transport10" - "ignition-fuel-tools6" - "ignition-plugin" - "ignition-rendering5" - "sdformat11" - "ignition-physics4" - "ignition-sensors5" - "ignition-gui5" - "ignition-gazebo5" - "ignition-launch4" - ) -elif [ "$COLLECTION" = "fortress" ]; then - LIBS=( - "ignition-cmake2" - "ignition-math6" - "ignition-utils1" - "ignition-tools" - "ignition-common4" - "ignition-msgs8" - "ignition-transport11" - "ignition-fuel-tools7" - "ignition-plugin" - "ignition-rendering6" - "sdformat12" - "ignition-physics5" - "ignition-sensors6" - "ignition-gui6" - "ignition-gazebo6" - "ignition-launch5" - ) -else - echo "Missing collection" - exit -fi - -for LIB in "${LIBS[@]}" -do - LIB_SHORT="${LIB/ignition-/ign-}" - LIB_NAME=${LIB_SHORT//[[:digit:]]/} - - PKG_VERSION=$(wget -qO- http://packages.osrfoundation.org/gazebo/ubuntu-${PACKAGE_REPO}/dists/focal/main/binary-amd64/Packages | \ - grep -1 -m 1 -e "Source: ${LIB}" -e "Package: ${LIB}" | \ - sed -n 's/^Version: \(.*\)/\1/p' | uniq) - - PKG_VERSION=${PKG_VERSION%%~*} - - COMPACT_VERSION=${PKG_VERSION#* } - COMPACT_VERSION=${COMPACT_VERSION%%-*} - - echo https://github.com/ignitionrobotics/${LIB_NAME}/compare/${LIB}_${COMPACT_VERSION}...${LIB_SHORT} - -done From 6a5f89b80af764c38293acc0a2ee604f03ef2dc7 Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Wed, 3 Nov 2021 19:18:49 -0700 Subject: [PATCH 009/540] ignition_collection: fix ign-tools nightly branch (#544) Signed-off-by: Steve Peters --- jenkins-scripts/dsl/ignition_collection.dsl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index 8109a495b..ea281e992 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -592,6 +592,7 @@ nightly_scheduler_job.with rendering_branch = get_nightly_branch(collection_data, 'rendering') sensors_branch = get_nightly_branch(collection_data, 'sensors') sdformat_branch = get_nightly_branch(collection_data, 'sdformat') + tools_branch = get_nightly_branch(collection_data, 'tools') transport_branch = get_nightly_branch(collection_data, 'transport') utils_branch = get_nightly_branch(collection_data, 'utils') @@ -637,6 +638,8 @@ nightly_scheduler_job.with src_branch="${sdformat_branch}" elif [[ "\${n}" != "\${n/transport/}" ]]; then src_branch="${transport_branch}" + elif [[ "\${n}" != "\${n/tools/}" ]]; then + src_branch="${tools_branch}" elif [[ "\${n}" != "\${n/utils/}" ]]; then src_branch="${utils_branch}" else From 985be39b543a79bad58a4961f29b21b30eb0d66d Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Wed, 10 Nov 2021 07:46:35 -0800 Subject: [PATCH 010/540] ignition_collection.dsl: add Garden view (#546) Part of #527. Also build nightlies for Garden Signed-off-by: Steve Peters * Change fortress to garden in nightly_jobs Signed-off-by: Jose Luis Rivero Co-authored-by: Jose Luis Rivero --- jenkins-scripts/dsl/ignition_collection.dsl | 91 ++++++++++++++++++++- 1 file changed, 89 insertions(+), 2 deletions(-) diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index ea281e992..5b294ad79 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -4,7 +4,7 @@ import javaposse.jobdsl.dsl.Job // IGNITION COLLECTIONS arch = 'amd64' -ignition_nightly = 'fortress' +ignition_nightly = 'garden' ignition_collections = [ [ name : 'citadel', @@ -18,6 +18,9 @@ ignition_collections = [ ], [ name : 'fortress', distros : [ 'focal' ], + ], + [ name : 'garden', + distros : [ 'focal' ], // These are the branches currently targeted at the upcoming collection // They're in topological order nightly_jobs: [ @@ -37,7 +40,7 @@ ignition_collections = [ 'physics' : [ debbuild: 'ign-physics5' , branch: 'ign-physics5' ], 'gazebo' : [ debbuild: 'ign-gazebo6' , branch: 'ign-gazebo6' ], 'launch' : [ debbuild: 'ign-launch5' , branch: 'ign-launch5' ], - 'fortress' : [ debbuild: 'ign-fortress' , branch: 'main' ], + 'garden' : [ debbuild: 'ign-garden' , branch: 'main' ], ], ], ] @@ -367,6 +370,90 @@ ignition_collection_jobs = 'sdformat-ci-sdformat12-windows7-amd64', 'sdformat-install-sdformat12_pkg-bionic-amd64' ], + 'garden' : [ + 'ign_common-ign-4-win', + 'ign_fuel-tools-ign-7-win', + 'ign_gazebo-ign-6-win', + 'ign_gui-ign-6-win', + 'ign_launch-ign-5-win', + 'ign_msgs-ign-8-win', + 'ign_physics-ign-5-win', + 'ign_plugin-ign-1-win', + 'ign_rendering-ign-6-win', + 'ign_sensors-ign-6-win', + 'ign_tools-ign-1-win', + 'ign_transport-ign-11-win', + 'ign_utils-ign-1-win', + 'ignition_cmake-ci-ign-cmake2-bionic-amd64', + 'ignition_cmake-ci-ign-cmake2-homebrew-amd64', + 'ignition_cmake-ci-ign-cmake2-windows7-amd64', + 'ignition_cmake2-install-pkg-bionic-amd64', + 'ignition_cmake2-install_bottle-homebrew-amd64', + 'ignition_common-ci-ign-common4-bionic-amd64', + 'ignition_common-ci-ign-common4-homebrew-amd64', + 'ignition_common4-install-pkg-bionic-amd64', + 'ignition_common4-install_bottle-homebrew-amd64', + 'ignition_garden-ci-main-homebrew-amd64', + 'ignition_garden-install-pkg-bionic-amd64', + 'ignition_garden-install_bottle-homebrew-amd64', + 'ignition_fuel-tools-ci-ign-fuel-tools7-bionic-amd64', + 'ignition_fuel-tools-ci-ign-fuel-tools7-homebrew-amd64', + 'ignition_fuel-tools7-install-pkg-bionic-amd64', + 'ignition_fuel-tools7-install_bottle-homebrew-amd64', + 'ignition_gazebo-ci-ign-gazebo6-bionic-amd64', + 'ignition_gazebo-ci-ign-gazebo6-homebrew-amd64', + 'ignition_gazebo6-install-pkg-bionic-amd64', + 'ignition_gazebo6-install_bottle-homebrew-amd64', + 'ignition_gui-ci-ign-gui6-bionic-amd64', + 'ignition_gui-ci-ign-gui6-homebrew-amd64', + 'ignition_gui6-install-pkg-bionic-amd64', + 'ignition_gui6-install_bottle-homebrew-amd64', + 'ignition_launch-ci-ign-launch5-bionic-amd64', + 'ignition_launch-ci-ign-launch5-homebrew-amd64', + 'ignition_launch5-install-pkg-bionic-amd64', + 'ignition_launch5-install_bottle-homebrew-amd64', + 'ignition_math-ci-ign-math6-bionic-amd64', + 'ignition_math-ci-ign-math6-homebrew-amd64', + 'ignition_math-ci-ign-math6-windows7-amd64', + 'ignition_math6-install-pkg-bionic-amd64', + 'ignition_math6-install_bottle-homebrew-amd64', + 'ignition_msgs-ci-ign-msgs8-bionic-amd64', + 'ignition_msgs-ci-ign-msgs8-homebrew-amd64', + 'ignition_msgs8-install-pkg-bionic-amd64', + 'ignition_msgs8-install_bottle-homebrew-amd64', + 'ignition_physics-ci-ign-physics5-bionic-amd64', + 'ignition_physics-ci-ign-physics5-homebrew-amd64', + 'ignition_physics5-install-pkg-bionic-amd64', + 'ignition_physics5-install_bottle-homebrew-amd64', + 'ignition_plugin-ci-ign-plugin1-bionic-amd64', + 'ignition_plugin-ci-ign-plugin1-homebrew-amd64', + 'ignition_plugin-install-pkg-bionic-amd64', + 'ignition_plugin1-install_bottle-homebrew-amd64', + 'ignition_rendering-ci-ign-rendering6-bionic-amd64', + 'ignition_rendering-ci-ign-rendering6-homebrew-amd64', + 'ignition_rendering6-install-pkg-bionic-amd64', + 'ignition_rendering6-install_bottle-homebrew-amd64', + 'ignition_sensors-ci-ign-sensors6-bionic-amd64', + 'ignition_sensors-ci-ign-sensors6-homebrew-amd64', + 'ignition_sensors6-install-pkg-bionic-amd64', + 'ignition_sensors6-install_bottle-homebrew-amd64', + 'ignition_tools-ci-ign-tools1-bionic-amd64', + 'ignition_tools-ci-ign-tools1-homebrew-amd64', + 'ignition_tools-install-pkg-bionic-amd64', + 'ignition_tools1-install_bottle-homebrew-amd64', + 'ignition_transport-ci-ign-transport11-bionic-amd64', + 'ignition_transport-ci-ign-transport11-homebrew-amd64', + 'ignition_transport11-install-pkg-bionic-amd64', + 'ignition_transport11-install_bottle-homebrew-amd64', + 'ignition_utils-ci-ign-utils1-bionic-amd64', + 'ignition_utils-ci-ign-utils1-homebrew-amd64', + 'ignition_utils-install-pkg-bionic-amd64', + 'ignition_utils1-install_bottle-homebrew-amd64', + 'sdformat-ci-sdformat12-bionic-amd64', + 'sdformat-ci-sdformat12-homebrew-amd64', + 'sdformat-ci-sdformat12-windows7-amd64', + 'sdformat-install-sdformat12_pkg-bionic-amd64' + ], ] def DISABLE_TESTS = false From bae655ae406dc39184f2d95b34060d9667626d28 Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Wed, 10 Nov 2021 21:16:09 -0800 Subject: [PATCH 011/540] Bumps in garden : ign-gui7 (#548) Signed-off-by: Steve Peters --- jenkins-scripts/dsl/ignition_collection.dsl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index 5b294ad79..2a773ab8f 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -35,7 +35,7 @@ ignition_collections = [ 'sdformat' : [ debbuild: 'sdformat12' , branch: 'sdf12' ], 'fuel-tools': [ debbuild: 'ign-fuel-tools7' , branch: 'ign-fuel-tools7' ], 'transport' : [ debbuild: 'ign-transport11' , branch: 'ign-transport11' ], - 'gui' : [ debbuild: 'ign-gui6' , branch: 'ign-gui6' ], + 'gui' : [ debbuild: 'ign-gui7' , branch: 'main' ], 'sensors' : [ debbuild: 'ign-sensors6' , branch: 'ign-sensors6' ], 'physics' : [ debbuild: 'ign-physics5' , branch: 'ign-physics5' ], 'gazebo' : [ debbuild: 'ign-gazebo6' , branch: 'ign-gazebo6' ], From 4ede91f39621ece223374da868a7cb0362a9083b Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Wed, 10 Nov 2021 21:18:40 -0800 Subject: [PATCH 012/540] Bumps in garden : ign-sensors7 (#550) Signed-off-by: Steve Peters Co-authored-by: Louise Poubel --- jenkins-scripts/dsl/ignition_collection.dsl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index 2a773ab8f..755024f30 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -36,7 +36,7 @@ ignition_collections = [ 'fuel-tools': [ debbuild: 'ign-fuel-tools7' , branch: 'ign-fuel-tools7' ], 'transport' : [ debbuild: 'ign-transport11' , branch: 'ign-transport11' ], 'gui' : [ debbuild: 'ign-gui7' , branch: 'main' ], - 'sensors' : [ debbuild: 'ign-sensors6' , branch: 'ign-sensors6' ], + 'sensors' : [ debbuild: 'ign-sensors7' , branch: 'main' ], 'physics' : [ debbuild: 'ign-physics5' , branch: 'ign-physics5' ], 'gazebo' : [ debbuild: 'ign-gazebo6' , branch: 'ign-gazebo6' ], 'launch' : [ debbuild: 'ign-launch5' , branch: 'ign-launch5' ], From 60a2c82df84b7e642567ee50e4ad5d169001126d Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Wed, 10 Nov 2021 21:22:16 -0800 Subject: [PATCH 013/540] Bumps in garden : ign-gazebo7 (#547) Signed-off-by: Steve Peters --- jenkins-scripts/dsl/ignition_collection.dsl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index 755024f30..87691c48e 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -38,7 +38,7 @@ ignition_collections = [ 'gui' : [ debbuild: 'ign-gui7' , branch: 'main' ], 'sensors' : [ debbuild: 'ign-sensors7' , branch: 'main' ], 'physics' : [ debbuild: 'ign-physics5' , branch: 'ign-physics5' ], - 'gazebo' : [ debbuild: 'ign-gazebo6' , branch: 'ign-gazebo6' ], + 'gazebo' : [ debbuild: 'ign-gazebo7' , branch: 'main' ], 'launch' : [ debbuild: 'ign-launch5' , branch: 'ign-launch5' ], 'garden' : [ debbuild: 'ign-garden' , branch: 'main' ], ], From e623a782dba47127bbb5f367a663775e06737ab8 Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Wed, 10 Nov 2021 21:26:33 -0800 Subject: [PATCH 014/540] Bumps in garden : ign-launch6 (#549) Signed-off-by: Steve Peters Co-authored-by: Louise Poubel --- jenkins-scripts/dsl/ignition_collection.dsl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index 87691c48e..d7525aa57 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -39,7 +39,7 @@ ignition_collections = [ 'sensors' : [ debbuild: 'ign-sensors7' , branch: 'main' ], 'physics' : [ debbuild: 'ign-physics5' , branch: 'ign-physics5' ], 'gazebo' : [ debbuild: 'ign-gazebo7' , branch: 'main' ], - 'launch' : [ debbuild: 'ign-launch5' , branch: 'ign-launch5' ], + 'launch' : [ debbuild: 'ign-launch6' , branch: 'main' ], 'garden' : [ debbuild: 'ign-garden' , branch: 'main' ], ], ], From d9bb19b378a0b3b159913f6ca1faf449f452cbbe Mon Sep 17 00:00:00 2001 From: Louise Poubel Date: Thu, 11 Nov 2021 06:14:03 -0800 Subject: [PATCH 015/540] Script to help check unreleased changes for each collection (#543) * Script to help check unreleased changes for each collection Signed-off-by: Louise Poubel * Refactor collection libs * Add note about getting the package list Co-authored-by: Jose Luis Rivero --- terminal-dashboard/_dashboard_lib.sh | 84 ++++++++++++++++++++++++ terminal-dashboard/table.bash | 96 +++------------------------- terminal-dashboard/unreleased.bash | 45 +++++++++++++ 3 files changed, 137 insertions(+), 88 deletions(-) create mode 100644 terminal-dashboard/_dashboard_lib.sh create mode 100755 terminal-dashboard/unreleased.bash diff --git a/terminal-dashboard/_dashboard_lib.sh b/terminal-dashboard/_dashboard_lib.sh new file mode 100644 index 000000000..87a5ed80a --- /dev/null +++ b/terminal-dashboard/_dashboard_lib.sh @@ -0,0 +1,84 @@ +#!/bin/bash + +# TODO:jrivero https://github.com/ignition-tooling/release-tools/issues/551 + +get_libraries_by_collection() +{ + local COLLECTION=${1} + + if [ "$COLLECTION" = "citadel" ]; then + LIBS=" + ignition-cmake2 + ignition-math6 + ignition-tools + ignition-common3 + ignition-msgs5 + ignition-transport8 + ignition-fuel-tools4 + ignition-plugin + ignition-rendering3 + sdformat9 + ignition-physics2 + ignition-sensors3 + ignition-gui3 + ignition-gazebo3 + ignition-launch2" + elif [ "$COLLECTION" = "dome" ]; then + LIBS=" + ignition-cmake2 + ignition-math6 + ignition-tools + ignition-common3 + ignition-msgs6 + ignition-transport9 + ignition-fuel-tools5 + ignition-plugin + ignition-rendering4 + sdformat10 + ignition-physics3 + ignition-sensors4 + ignition-gui4 + ignition-gazebo4 + ignition-launch3" + elif [ "$COLLECTION" = "edifice" ]; then + LIBS=" + ignition-cmake2 + ignition-math6 + ignition-utils1 + ignition-tools + ignition-common4 + ignition-msgs7 + ignition-transport10 + ignition-fuel-tools6 + ignition-plugin + ignition-rendering5 + sdformat11 + ignition-physics4 + ignition-sensors5 + ignition-gui5 + ignition-gazebo5 + ignition-launch4" + elif [ "$COLLECTION" = "fortress" ]; then + LIBS=" + ignition-cmake2 + ignition-math6 + ignition-utils1 + ignition-tools + ignition-common4 + ignition-msgs8 + ignition-transport11 + ignition-fuel-tools7 + ignition-plugin + ignition-rendering6 + sdformat12 + ignition-physics5 + ignition-sensors6 + ignition-gui6 + ignition-gazebo6 + ignition-launch5" + else + return 1 + fi + + echo "$LIBS" +} diff --git a/terminal-dashboard/table.bash b/terminal-dashboard/table.bash index c3a232f2e..3950be29c 100755 --- a/terminal-dashboard/table.bash +++ b/terminal-dashboard/table.bash @@ -1,5 +1,9 @@ #!/bin/bash +# Knowing Script dir beware of symlink +[[ -L ${0} ]] && SCRIPT_DIR=$(readlink ${0}) || SCRIPT_DIR=${0} +SCRIPT_DIR="${SCRIPT_DIR%/*}" + # Arguments # # 1. : Required: A supported collection, i.e. "citadel", "fortesss", etc. @@ -13,95 +17,12 @@ # # bash table.bash edifice prerelease -COLLECTION=$1 +# shellcheck source=./_dashboard_lib.sh +. ${SCRIPT_DIR}/_dashboard_lib.sh +COLLECTION=$1 PACKAGE_REPO=${2:-stable} - -if [ "$COLLECTION" = "citadel" ]; then - LIBS=( - "ignition-cmake2" - "ignition-math6" - "ignition-tools" - "ignition-common3" - "ignition-msgs5" - "ignition-transport8" - "ignition-fuel-tools4" - "ignition-plugin" - "ignition-rendering3" - "sdformat9" - "ignition-physics2" - "ignition-sensors3" - "ignition-gui3" - "ignition-gazebo3" - "ignition-launch2" - "ignition-citadel" - ) -elif [ "$COLLECTION" = "dome" ]; then - LIBS=( - "ignition-cmake2" - "ignition-math6" - "ignition-tools" - "ignition-common3" - "ignition-msgs6" - "ignition-transport9" - "ignition-fuel-tools5" - "ignition-plugin" - "ignition-rendering4" - "sdformat10" - "ignition-physics3" - "ignition-sensors4" - "ignition-gui4" - "ignition-gazebo4" - "ignition-launch3" - "ignition-dome" - ) -elif [ "$COLLECTION" = "edifice" ]; then - LIBS=( - "ignition-cmake2" - "ignition-math6" - "ignition-utils1" - "ignition-tools" - "ignition-common4" - "ignition-msgs7" - "ignition-transport10" - "ignition-fuel-tools6" - "ignition-plugin" - "ignition-rendering5" - "sdformat11" - "ignition-physics4" - "ignition-sensors5" - "ignition-gui5" - "ignition-gazebo5" - "ignition-launch4" - "ignition-edifice" - ) -elif [ "$COLLECTION" = "fortress" ]; then - LIBS=( - "ignition-cmake2" - "ignition-math6" - "ignition-utils1" - "ignition-tools" - "ignition-common4" - "ignition-msgs8" - "ignition-transport11" - "ignition-fuel-tools7" - "ignition-plugin" - "ignition-rendering6" - "sdformat12" - "ignition-physics5" - "ignition-sensors6" - "ignition-gui6" - "ignition-gazebo6" - "ignition-launch5" - "ignition-fortress" - ) -else - echo "Missing collection" - exit -fi - ARCHS=( "amd64" "i386" "arm64" "armhf") - DISTROS=( "ubuntu" "debian" ) COLUMN=" " @@ -110,8 +31,7 @@ GREEN="\e[42m" YELLOW="\e[43m" RED="\e[101m" -for LIB in "${LIBS[@]}" -do +for LIB in $(get_libraries_by_collection "${COLLECTION}" ); do echo -e "\e[107m\e[90m${LIB}\e[49m\e[39m" LIB_VER="" diff --git a/terminal-dashboard/unreleased.bash b/terminal-dashboard/unreleased.bash new file mode 100755 index 000000000..0e6e4488f --- /dev/null +++ b/terminal-dashboard/unreleased.bash @@ -0,0 +1,45 @@ +#!/bin/bash + +# Knowing Script dir beware of symlink +[[ -L ${0} ]] && SCRIPT_DIR=$(readlink ${0}) || SCRIPT_DIR=${0} +SCRIPT_DIR="${SCRIPT_DIR%/*}" + +# Helper script to check if there are unreleased changes for supported versions. +# The script prints a list of links that can be clicked to see if there are new +# changes since the latest release. +# +# Arguments +# +# 1. : Required: A supported collection, i.e. "citadel", "fortesss", etc. +# +# Usage +# +# bash unreleased.bash +# +# For example +# +# bash unreleased.bash edifice + +# shellcheck source=./_dashboard_lib.sh +. "${SCRIPT_DIR}"/_dashboard_lib.sh + +COLLECTION=$1 +PACKAGE_REPO=${2:-stable} + +for LIB in $(get_libraries_by_collection "${COLLECTION}" ); do + LIB_SHORT="${LIB/ignition-/ign-}" + LIB_NAME=${LIB_SHORT//[[:digit:]]/} + + PKG_VERSION=$(wget -qO- http://packages.osrfoundation.org/gazebo/ubuntu-${PACKAGE_REPO}/dists/focal/main/binary-amd64/Packages | \ + grep -1 -m 1 -e "Source: ${LIB}" -e "Package: ${LIB}" | \ + sed -n 's/^Version: \(.*\)/\1/p' | uniq) + + PKG_VERSION=${PKG_VERSION%%~*} + + COMPACT_VERSION=${PKG_VERSION#* } + COMPACT_VERSION=${COMPACT_VERSION%%-*} + + echo https://github.com/ignitionrobotics/${LIB_NAME}/compare/${LIB}_${COMPACT_VERSION}...${LIB_SHORT} + +done + From 116747bb6ec052bedb117930676aab2eedf8902f Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Thu, 11 Nov 2021 10:44:45 -0800 Subject: [PATCH 016/540] ign-physics CI/debbuilds use large-memory label (#526) Signed-off-by: Steve Peters Co-authored-by: Jose Luis Rivero --- jenkins-scripts/dsl/ignition.dsl | 11 +++++++++++ release.py | 2 ++ 2 files changed, 13 insertions(+) diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index 617b8061c..ddc45bf25 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -307,7 +307,12 @@ ignition_software.each { ign_sw -> abi_job.with { if (ign_sw == 'physics') + { label "huge-memory" + // on ARM native nodes in buildfarm we need to restrict to 2 the + // compilation threads to avoid OOM killer + GLOBAL_SHELL_CMD = GLOBAL_SHELL_CMD + '\nif [[ $(uname -m) == "aarch64" ]]; then export MAKE_JOBS=2; fi' + } steps { shell("""\ @@ -344,6 +349,9 @@ ignition_software.each { ign_sw -> include_gpu_label_if_needed(ignition_ci_any_job, ign_sw) ignition_ci_any_job.with { + if (ign_sw == 'physics') + label "huge-memory" + steps { shell("""\ @@ -434,6 +442,9 @@ ignition_software.each { ign_sw -> include_gpu_label_if_needed(ignition_ci_job, ign_sw) ignition_ci_job.with { + if (ign_sw == 'physics') + label "huge-memory" + triggers { scm('@daily') } diff --git a/release.py b/release.py index d02023611..a76f60d0d 100755 --- a/release.py +++ b/release.py @@ -617,6 +617,8 @@ def go(argv): # Need to use JENKINS_NODE_TAG parameter for large memory nodes # since it runs qemu emulation linux_platform_params['JENKINS_NODE_TAG'] = 'linux-' + a + '|| large-memory' + elif ('ignition-physics' in args.package_alias): + linux_platform_params['JENKINS_NODE_TAG'] = 'large-memory' if (NIGHTLY and a == 'i386'): continue From 5ac4cef5a07c99eb1edb666caef57aa9733b23e3 Mon Sep 17 00:00:00 2001 From: Louise Poubel Date: Sun, 14 Nov 2021 21:06:36 -0800 Subject: [PATCH 017/540] Re-add collection to dashboard lib (#553) Signed-off-by: Louise Poubel --- terminal-dashboard/_dashboard_lib.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/terminal-dashboard/_dashboard_lib.sh b/terminal-dashboard/_dashboard_lib.sh index 87a5ed80a..373d80f2e 100644 --- a/terminal-dashboard/_dashboard_lib.sh +++ b/terminal-dashboard/_dashboard_lib.sh @@ -22,7 +22,8 @@ get_libraries_by_collection() ignition-sensors3 ignition-gui3 ignition-gazebo3 - ignition-launch2" + ignition-launch2 + ignition-citadel" elif [ "$COLLECTION" = "dome" ]; then LIBS=" ignition-cmake2 @@ -39,7 +40,8 @@ get_libraries_by_collection() ignition-sensors4 ignition-gui4 ignition-gazebo4 - ignition-launch3" + ignition-launch3 + ignition-dome" elif [ "$COLLECTION" = "edifice" ]; then LIBS=" ignition-cmake2 @@ -57,7 +59,8 @@ get_libraries_by_collection() ignition-sensors5 ignition-gui5 ignition-gazebo5 - ignition-launch4" + ignition-launch4 + ignition-edifice" elif [ "$COLLECTION" = "fortress" ]; then LIBS=" ignition-cmake2 @@ -75,7 +78,8 @@ get_libraries_by_collection() ignition-sensors6 ignition-gui6 ignition-gazebo6 - ignition-launch5" + ignition-launch5 + ignition-fortress" else return 1 fi From 9eb0ab77166f39bad92be401b509ae753c49c79e Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Tue, 16 Nov 2021 18:34:35 -0800 Subject: [PATCH 018/540] Bumps in garden : ign-rendering7 (#558) Signed-off-by: Steve Peters --- jenkins-scripts/dsl/ignition_collection.dsl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index d7525aa57..055171cb8 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -31,7 +31,7 @@ ignition_collections = [ 'utils' : [ debbuild: 'ign-utils1' , branch: 'ign-utils1' ], 'common' : [ debbuild: 'ign-common4' , branch: 'ign-common4' ], 'msgs' : [ debbuild: 'ign-msgs8' , branch: 'ign-msgs8' ], - 'rendering' : [ debbuild: 'ign-rendering6' , branch: 'ign-rendering6' ], + 'rendering' : [ debbuild: 'ign-rendering7' , branch: 'main' ], 'sdformat' : [ debbuild: 'sdformat12' , branch: 'sdf12' ], 'fuel-tools': [ debbuild: 'ign-fuel-tools7' , branch: 'ign-fuel-tools7' ], 'transport' : [ debbuild: 'ign-transport11' , branch: 'ign-transport11' ], From 66921e95ecfc5294fb6039ddb54b4aee904e1a84 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 18 Nov 2021 20:50:05 +0100 Subject: [PATCH 019/540] Support for OGRE extra includes in abi checker (#560) --- jenkins-scripts/docker/lib/generic-abi-base.bash | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/jenkins-scripts/docker/lib/generic-abi-base.bash b/jenkins-scripts/docker/lib/generic-abi-base.bash index 3aa096fd6..317ff917a 100644 --- a/jenkins-scripts/docker/lib/generic-abi-base.bash +++ b/jenkins-scripts/docker/lib/generic-abi-base.bash @@ -37,6 +37,13 @@ if [[ "${NEED_C17_COMPILER}" == "true" ]]; then ABI_CXX_STANDARD=c++17 fi +# Needed to compile software using OGRE-2.2. Being used on every build, should not hurt +EXTRA_INCLUDES=""" + + /usr/include/OGRE-2.2/Hlms/Common + +""" + cat > build.sh << DELIM #!/bin/bash @@ -109,6 +116,8 @@ cat > pkg.xml << CURRENT_DELIM /usr/local/destination_branch/include/\$DEST_DIR + ${EXTRA_INCLUDES} + CURRENT_DELIM @@ -136,6 +145,8 @@ cat > devel.xml << DEVEL_DELIM /usr/local/source_branch/include/\$SRC_DIR + ${EXTRA_INCLUDES} + DEVEL_DELIM From f9bab5e0b602500a3da03f5b293f2b27ee994c8b Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Fri, 19 Nov 2021 08:45:12 -0800 Subject: [PATCH 020/540] ABI builds: only set ogre2.2 paths if needed (#562) Fix for builds not using ogre2.2 after #560. Set the EXTRA_INCLUDES variable in ignition-abichecker.bash to add ogre2 include paths only if a package using ignition-rendering6+ is detected. Signed-off-by: Steve Peters --- jenkins-scripts/docker/ignition-abichecker.bash | 13 +++++++++++++ jenkins-scripts/docker/lib/generic-abi-base.bash | 7 ------- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/jenkins-scripts/docker/ignition-abichecker.bash b/jenkins-scripts/docker/ignition-abichecker.bash index 2852ab5bf..f6bcbe667 100644 --- a/jenkins-scripts/docker/ignition-abichecker.bash +++ b/jenkins-scripts/docker/ignition-abichecker.bash @@ -47,6 +47,19 @@ then export NEED_C17_COMPILER=true fi +# check if OGRE-2.2 include paths are needed +if [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-gazebo" && ${IGN_NAME_PREFIX_MAJOR_VERSION} -ge 6 ]] || \ + [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-gui" && ${IGN_NAME_PREFIX_MAJOR_VERSION} -ge 6 ]] || \ + [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-sensors" && ${IGN_NAME_PREFIX_MAJOR_VERSION} -ge 6 ]] || \ + [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-rendering" && ${IGN_NAME_PREFIX_MAJOR_VERSION} -ge 6 ]] +then + export EXTRA_INCLUDES=""" + + /usr/include/OGRE-2.2/Hlms/Common + +""" +fi + # default to use stable repos export ABI_JOB_REPOS="stable" diff --git a/jenkins-scripts/docker/lib/generic-abi-base.bash b/jenkins-scripts/docker/lib/generic-abi-base.bash index 317ff917a..cef131181 100644 --- a/jenkins-scripts/docker/lib/generic-abi-base.bash +++ b/jenkins-scripts/docker/lib/generic-abi-base.bash @@ -37,13 +37,6 @@ if [[ "${NEED_C17_COMPILER}" == "true" ]]; then ABI_CXX_STANDARD=c++17 fi -# Needed to compile software using OGRE-2.2. Being used on every build, should not hurt -EXTRA_INCLUDES=""" - - /usr/include/OGRE-2.2/Hlms/Common - -""" - cat > build.sh << DELIM #!/bin/bash From 0a08a1a11e3d03894ab6721d3905f41a151fca54 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 19 Nov 2021 18:29:37 +0100 Subject: [PATCH 021/540] Make debbuilds unstable when a missing component is triggering a warning (#559) * Unstable debbuild is there are missing dependencies * Fix regexp Signed-off-by: Jose Luis Rivero * Use java regexp no perl Signed-off-by: Jose Luis Rivero --- jenkins-scripts/parser_rules/debbuild_missing.parser | 1 + 1 file changed, 1 insertion(+) diff --git a/jenkins-scripts/parser_rules/debbuild_missing.parser b/jenkins-scripts/parser_rules/debbuild_missing.parser index 4d975e705..6a0179bad 100644 --- a/jenkins-scripts/parser_rules/debbuild_missing.parser +++ b/jenkins-scripts/parser_rules/debbuild_missing.parser @@ -1,2 +1,3 @@ warning /^dh_missing: / warning /.*binaries-have-file-conflict.*/ +warning /^\s*-- Skipping component \[.*\]: Missing dependency.*\./ From 28e1c2c1947cf679403edd90a17a9e7f3a1f6904 Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Fri, 19 Nov 2021 12:27:48 -0800 Subject: [PATCH 022/540] Fix ABI builds with ogre 2.2 for launch5+ (#563) Signed-off-by: Steve Peters --- jenkins-scripts/docker/ignition-abichecker.bash | 1 + 1 file changed, 1 insertion(+) diff --git a/jenkins-scripts/docker/ignition-abichecker.bash b/jenkins-scripts/docker/ignition-abichecker.bash index f6bcbe667..bc9dc7dcd 100644 --- a/jenkins-scripts/docker/ignition-abichecker.bash +++ b/jenkins-scripts/docker/ignition-abichecker.bash @@ -50,6 +50,7 @@ fi # check if OGRE-2.2 include paths are needed if [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-gazebo" && ${IGN_NAME_PREFIX_MAJOR_VERSION} -ge 6 ]] || \ [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-gui" && ${IGN_NAME_PREFIX_MAJOR_VERSION} -ge 6 ]] || \ + [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-launch" && ${IGN_NAME_PREFIX_MAJOR_VERSION} -ge 5 ]] || \ [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-sensors" && ${IGN_NAME_PREFIX_MAJOR_VERSION} -ge 6 ]] || \ [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-rendering" && ${IGN_NAME_PREFIX_MAJOR_VERSION} -ge 6 ]] then From 3702deeb968a3d15fd0b9e34dbfa77ee4dfce54c Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Fri, 19 Nov 2021 18:51:24 -0800 Subject: [PATCH 023/540] bump_dependency.bash: use main in collection.yaml (#568) Signed-off-by: Steve Peters --- release-repo-scripts/bump_dependency.bash | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/release-repo-scripts/bump_dependency.bash b/release-repo-scripts/bump_dependency.bash index 8e86f46e3..a94a3b47a 100755 --- a/release-repo-scripts/bump_dependency.bash +++ b/release-repo-scripts/bump_dependency.bash @@ -334,7 +334,6 @@ for ((i = 0; i < "${#LIBRARIES[@]}"; i++)); do fi echo -e "${GREEN}${LIB}: Updating source code${DEFAULT}" - # TODO: handle yaml file in collection source like gazebodistro (main instead of dep+1) for ((j = 0; j < "${#LIBRARIES[@]}"; j++)); do DEP_LIB=${LIBRARIES[$j]#"ign-"} @@ -346,6 +345,11 @@ for ((i = 0; i < "${#LIBRARIES[@]}"; i++)); do # Second run with _ instead of -, to support multiple variations of fuel-tools DEP_LIB=${DEP_LIB//-/_} find . -type f ! -name 'Changelog.md' ! -name 'Migration.md' -print0 | xargs -0 sed -i "s ${DEP_LIB}${DEP_PREV_VER} ${DEP_LIB}${DEP_VER} g" + + # Replace collection yaml branch names with main + if [[ "${LIB}" == "ign-${COLLECTION}" ]]; then + find . -type f -name "collection-${COLLECTION}.yaml" -print0 | xargs -0 sed -i "s ign-${DEP_LIB}${DEP_VER} main g" + fi done commitAndPR ${ORG} main From e798e29817770e42fedeaaf3e4ab69663853636d Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Tue, 23 Nov 2021 14:51:39 -0800 Subject: [PATCH 024/540] Nightly builds: Use ign-msgs9, ign-common5 (#570) Preparation for #554, #567. Signed-off-by: Steve Peters --- jenkins-scripts/dsl/ignition_collection.dsl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index 055171cb8..6a1999d07 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -29,8 +29,8 @@ ignition_collections = [ 'math' : [ debbuild: 'ign-math6' , branch: 'ign-math6' ], 'plugin' : [ debbuild: 'ign-plugin' , branch: 'ign-plugin1' ], 'utils' : [ debbuild: 'ign-utils1' , branch: 'ign-utils1' ], - 'common' : [ debbuild: 'ign-common4' , branch: 'ign-common4' ], - 'msgs' : [ debbuild: 'ign-msgs8' , branch: 'ign-msgs8' ], + 'common' : [ debbuild: 'ign-common5' , branch: 'main' ], + 'msgs' : [ debbuild: 'ign-msgs9' , branch: 'main' ], 'rendering' : [ debbuild: 'ign-rendering7' , branch: 'main' ], 'sdformat' : [ debbuild: 'sdformat12' , branch: 'sdf12' ], 'fuel-tools': [ debbuild: 'ign-fuel-tools7' , branch: 'ign-fuel-tools7' ], From 9bfe8c23e9801a60da815132395626a508cb4860 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 24 Nov 2021 17:27:33 +0100 Subject: [PATCH 025/540] Update vcpkg snapshot to 2021.05.12 (#571) --- jenkins-scripts/lib/windows_env_vars.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/lib/windows_env_vars.bat b/jenkins-scripts/lib/windows_env_vars.bat index 5ab072d73..011542765 100644 --- a/jenkins-scripts/lib/windows_env_vars.bat +++ b/jenkins-scripts/lib/windows_env_vars.bat @@ -15,6 +15,6 @@ set VCPKG_CMD=%VCPKG_DIR%\vcpkg.exe set VCPKG_CMAKE_TOOLCHAIN_FILE=%VCPKG_DIR%/scripts/buildsystems/vcpkg.cmake if NOT DEFINED VCPKG_SNAPSHOT ( :: see https://github.com/microsoft/vcpkg/releases - set VCPKG_SNAPSHOT=2020.11 + set VCPKG_SNAPSHOT=2021.05.12 ) goto :EOF From 25fb8bccc30d45b62064c3ebbd1f6658711ba221 Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Mon, 29 Nov 2021 11:50:56 -0800 Subject: [PATCH 026/540] bump_dependency.bash: improve homebrew support (#557) * Use master if DOCS_BRANCH is not set * Add FORMULA_BASE variable to deduplicate logic * Fix handling of sdformat * Use LIB_ in an extra place * sed: improve url replacement, remove revision line * Build from main branch instead of SOURCE_COMMIT Also remove the sha256 line and clean up version string. Signed-off-by: Steve Peters --- release-repo-scripts/bump_dependency.bash | 42 +++++++++++++---------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/release-repo-scripts/bump_dependency.bash b/release-repo-scripts/bump_dependency.bash index a94a3b47a..7a6c859b6 100755 --- a/release-repo-scripts/bump_dependency.bash +++ b/release-repo-scripts/bump_dependency.bash @@ -19,13 +19,15 @@ # Requires the 'gh' CLI to be installed. # # Usage: -# $ ./bump_dependency.bash ; ; +# $ ./bump_dependency.bash ; ; [] # # For example, to bump to `ign-rendering6` and all its dependencies, as well as -# `sdf12` and its dependencies on fortress: +# `sdf12` and its dependencies on fortress using the `chapulina/fortress` branch for `docs`: # # ./bump_dependency.bash fortress "ign-rendering;sdformat" "6;12" 428 edifice "chapulina/fortress" # +# The `docs_branch` parameter is optional and defaults to `master` if not specified. +# # The script clones all the necessary repositories under /tmp/bump_dependency. # # Before committing to each repository, the script asks "Commit ?". @@ -57,6 +59,9 @@ VERSION_INPUT=${3} ISSUE_NUMBER=${4} PREV_COLLECTION=${5} DOCS_BRANCH=${6} +if [[ -z "${DOCS_BRANCH}" ]]; then + DOCS_BRANCH=master +fi COMMIT_MSG="Bumps in ${COLLECTION}" PR_TEXT="See https://github.com/${TOOLING_ORG}/release-tools/issues/${ISSUE_NUMBER}" @@ -308,9 +313,6 @@ for ((i = 0; i < "${#LIBRARIES[@]}"; i++)); do PREV_VER="$((${VER}-1))" LIB_UPPER=`echo ${LIB#"ign-"} | tr a-z A-Z` ORG=${IGN_ORG} - if [ "$LIB" = "sdformat" ]; then - ORG=${OSRF_ORG} - fi BUMP_BRANCH="bump_${COLLECTION}_${LIB}${VER}" echo -e "${BLUE_BG}Processing [${LIB}]${DEFAULT_BG}" @@ -325,7 +327,7 @@ for ((i = 0; i < "${#LIBRARIES[@]}"; i++)); do startFromCleanBranch ${BUMP_BRANCH} main # Check if main branch of that library is the correct version - PROJECT_NAME="${LIB//-/_}${VER}" + PROJECT_NAME="${LIB_}${VER}" PROJECT_NAME="${PROJECT_NAME/ign_/ignition-}" PROJECT="project.*(${PROJECT_NAME}" if ! grep -q ${PROJECT} "CMakeLists.txt"; then @@ -354,8 +356,6 @@ for ((i = 0; i < "${#LIBRARIES[@]}"; i++)); do commitAndPR ${ORG} main - SOURCE_COMMIT=`git rev-parse HEAD` - ################## # release repo ################## @@ -386,27 +386,27 @@ for ((i = 0; i < "${#LIBRARIES[@]}"; i++)); do cd ${TEMP_DIR}/homebrew-simulation startFromCleanBranch bump_${COLLECTION}_${LIB} master - FORMULA="Formula/${LIB/ign/ignition}${VER}.rb" + # expand ign-* to ignition-* + FORMULA_BASE=${LIB/ign/ignition} + # construct path with major version suffix + FORMULA="Formula/${FORMULA_BASE}${VER}.rb" if [ ! -f "$FORMULA" ]; then echo -e "${GREEN}${LIB}: Creating ${FORMULA}${DEFAULT}" - git rm Aliases/${LIB/ign/ignition}${VER} + git rm Aliases/${FORMULA_BASE}${VER} # Collection if ! [[ $VER == ?(-)+([0-9]) ]] ; then cp Formula/ignition-${PREV_COLLECTION}.rb $FORMULA else - cp Formula/${LIB/ign/ignition}${PREV_VER}.rb $FORMULA + cp Formula/${FORMULA_BASE}${PREV_VER}.rb $FORMULA fi git add $FORMULA fi echo -e "${GREEN}${LIB}: Updating ${FORMULA}${DEFAULT}" - URL="https://github.com/${ORG}/${LIB}/archive/${SOURCE_COMMIT}.tar.gz" - wget $URL - SHA=`sha256sum ${SOURCE_COMMIT}.tar.gz | cut -d " " -f 1` - rm ${SOURCE_COMMIT}.tar.gz + URL="https://github.com/${ORG}/${LIB}.git" # libN sed -i -E "s ((${LIB#"ign-"}))${PREV_VER} \1${VER} g" $FORMULA @@ -415,15 +415,19 @@ for ((i = 0; i < "${#LIBRARIES[@]}"; i++)); do sed -i "s ${LIB}${PREV_VER} main g" $FORMULA # class IgnitionLibN sed -i -E "s/((class Ignition.*))${PREV_VER}/\1${VER}/g" $FORMULA + sed -i -E "s/((class Sdformat))${PREV_VER}/\1${VER}/g" $FORMULA # remove bottle - TODO: this is only needed for new formulae sed -i -e "/bottle do/,/end/d" $FORMULA # URL from release to commit - TODO: remove manual step - sed -i "s@url.*@url \"$URL\"@g" $FORMULA - # SHA - TODO: remove manual step - sed -i "s/sha256.*/sha256 \"$SHA\"/g" $FORMULA + sed -i "s@^ url.*@ url \"$URL\", branch: \"main\"@g" $FORMULA + # SHA - remove in favor of building from `main` branch + sed -i "/^ sha256.*/d" $FORMULA + # revision - remove if present + sed -i "/^ revision.*/d" $FORMULA # version + PREV_VER_NONNEGATIVE=$([[ "${PREV_VER}" -lt 0 ]] && echo "0" || echo "${PREV_VER}") sed -i "/ version /d" $FORMULA - sed -i "/url.*/a \ \ version\ \"${PREV_VER}.999.999~0~`date +"%Y%m%d"`~${SOURCE_COMMIT:0:6}\"" $FORMULA + sed -i "/^ url.*/a\ version \"${PREV_VER_NONNEGATIVE}.999.999~0~`date +"%Y%m%d"`\"" $FORMULA # Remove extra blank lines cat -s $FORMULA | tee $FORMULA From 12a6ddf9dca5afe17d037137a403ad6cb1300ac5 Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Tue, 30 Nov 2021 09:18:18 -0800 Subject: [PATCH 027/540] Brew CI: set PATH if cmake@3.21.4 is used (#577) gazebo11 needs an older version of cmake on homebrew, but it is keg-only so prepend its bin folder to the PATH. Workaround for #573. Signed-off-by: Steve Peters --- jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash b/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash index 0a64789da..dfc9fa6a4 100644 --- a/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash +++ b/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash @@ -119,6 +119,10 @@ fi if brew ruby -e "exit ! '${PROJECT_FORMULA}'.f.recursive_dependencies.map(&:name).keep_if { |d| d == 'qwt-qt5' }.empty?"; then CMAKE_ARGS='-DQWT_WIN_INCLUDE_DIR=/usr/local/opt/qwt-qt5/lib/qwt.framework/Headers -DQWT_WIN_LIBRARY_DIR=/usr/local/opt/qwt-qt5/lib' fi +# Workaround for cmake@3.21.4: set PATH +if brew ruby -e "exit ! '${PROJECT_FORMULA}'.f.recursive_dependencies.map(&:name).keep_if { |d| d == 'cmake@3.21.4' }.empty?"; then + export PATH=/usr/local/opt/cmake@3.21.4/bin:${PATH} +fi # Workaround for tbb@2020_u3: set CPATH, LIBRARY_PATH, and CMAKE_PREFIX_PATH if brew ruby -e "exit ! '${PROJECT_FORMULA}'.f.recursive_dependencies.map(&:name).keep_if { |d| d == 'tbb@2020_u3' }.empty?"; then export CMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH}:/usr/local/opt/tbb@2020_u3 From 205fce5a96b65507a94f13dd9a18a2d3b6cca807 Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Tue, 30 Nov 2021 14:33:29 -0800 Subject: [PATCH 028/540] Parse comments to build bottles for new distros (#556) Comment "brew-bot-tag: build-for-new-distro-{distro}" along with "build bottle" in a homebrew-simulation pull request to build bottles for new macOS versions. This only works if the pull request doesn't change the package version. Signed-off-by: Steve Peters --- jenkins-scripts/dsl/brew_release.dsl | 4 ++++ .../lib/homebrew_bottle_creation.bash | 19 +++++++++++++++++++ .../lib/homebrew_bottle_pullrequest.bash | 11 ++++++++++- 3 files changed, 33 insertions(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/brew_release.dsl b/jenkins-scripts/dsl/brew_release.dsl index d44932810..240056530 100644 --- a/jenkins-scripts/dsl/brew_release.dsl +++ b/jenkins-scripts/dsl/brew_release.dsl @@ -230,6 +230,7 @@ bottle_job_builder.with parameters { currentBuild() predefinedProp("PULL_REQUEST_URL", "https://github.com/osrf/homebrew-simulation/pull/\${ghprbPullId}") + predefinedProp("ghprbCommentBody", "\${ghprbCommentBody}") } } } @@ -254,6 +255,9 @@ bottle_job_hash_updater.with parameters { + // copy the github trigger comment for extra parameter parsing + stringParam("ghprbCommentBody", '', + 'GitHub trigger comment, which can be parsed for extra parameters.') // reuse the pull request created by homebrew_pull_request_updater in step 1 stringParam("PULL_REQUEST_URL", '', 'Pull request URL (osrf/homebrew-simulation) pointing to a pull request.') diff --git a/jenkins-scripts/lib/homebrew_bottle_creation.bash b/jenkins-scripts/lib/homebrew_bottle_creation.bash index 03b40ec22..e9358e5f0 100644 --- a/jenkins-scripts/lib/homebrew_bottle_creation.bash +++ b/jenkins-scripts/lib/homebrew_bottle_creation.bash @@ -13,6 +13,10 @@ if [ -z "${ghprbActualCommit}" ]; then echo ghprbActualCommit not specified exit -1 fi +if [ -z "${ghprbCommentBody}" ]; then + echo ghprbCommentBody not specified + exit -1 +fi if [ -z "${ghprbGhRepository}" ]; then echo ghprbGhRepository not specified exit -1 @@ -33,6 +37,20 @@ export GITHUB_BASE_REF=${ghprbTargetBranch} export GITHUB_REPOSITORY=${ghprbGhRepository} export GITHUB_REF=${sha1} export GITHUB_SHA=${ghprbActualCommit} +MACOS_VERSION_TO_SYM=$(brew ruby -e 'puts "#{MacOS.version.to_sym}"') +if [[ "${ghprbCommentBody}" =~ 'brew-bot-tag:' ]]; then + if [[ "${ghprbCommentBody}" =~ 'build-for-new-distro-' ]]; then + echo Found a build-for-new-distro- option in the comment. Limiting to matching macOS versions. + export KEEP_OLD=--keep-old + if [[ "${ghprbCommentBody}" =~ build-for-new-distro-${MACOS_VERSION_TO_SYM} ]]; then + echo Found a match for build-for-new-distro-${MACOS_VERSION_TO_SYM} in comment. + echo Proceeding with bottle build. + else + echo Did not find --only-${MACOS_VERSION_TO_SYM} in comment string \"${ghprbCommentBody}\" + exit 0 + fi + fi +fi echo '# END SECTION' echo '# BEGIN SECTION: clean up environment' @@ -75,6 +93,7 @@ brew test-bot --tap=osrf/simulation \ brew test-bot --tap=osrf/simulation \ --fail-fast \ + ${KEEP_OLD} \ --only-formulae \ --root-url=https://osrf-distributions.s3.amazonaws.com/bottles-simulation echo '# END SECTION' diff --git a/jenkins-scripts/lib/homebrew_bottle_pullrequest.bash b/jenkins-scripts/lib/homebrew_bottle_pullrequest.bash index dbb1e4610..100699af8 100644 --- a/jenkins-scripts/lib/homebrew_bottle_pullrequest.bash +++ b/jenkins-scripts/lib/homebrew_bottle_pullrequest.bash @@ -11,6 +11,10 @@ set -e BOTTLE_JSON_DIR=${WORKSPACE}/pkgs echo '# BEGIN SECTION: check variables' +if [ -z "${ghprbCommentBody}" ]; then + echo ghprbCommentBody not specified + exit -1 +fi if [ -z "${PULL_REQUEST_URL}" ]; then echo PULL_REQUEST_URL not specified exit -1 @@ -22,6 +26,11 @@ PULL_REQUEST_HEAD_REPO=$(curl ${PULL_REQUEST_API_URL} \ | python3 -c 'import json, sys; print(json.loads(sys.stdin.read())["head"]["repo"]["ssh_url"])') PULL_REQUEST_BRANCH=$(curl ${PULL_REQUEST_API_URL} \ | python3 -c 'import json, sys; print(json.loads(sys.stdin.read())["head"]["ref"])') +if [[ "${ghprbCommentBody}" =~ 'brew-bot-tag:' ]]; then + if [[ "${ghprbCommentBody}" =~ 'build-for-new-distro-' ]]; then + export KEEP_OLD=--keep-old + fi +fi echo '# END SECTION' # note that matrix projects use subdirectories on pkgs/ with the label of different configurations @@ -37,7 +46,7 @@ fi echo "# BEGIN SECTION: update bottle hashes" -${BREW} bottle --merge --write --no-commit ${FILES_WITH_NEW_HASH} +${BREW} bottle --merge --write --no-commit ${KEEP_OLD} ${FILES_WITH_NEW_HASH} # ensure that all modified files are committed export FORMULA_PATH='-a' From 8f2664ca7d1369e56b772ce3e9f8802153206fdf Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 1 Dec 2021 17:52:34 +0100 Subject: [PATCH 029/540] Force vcs import to make possible change between forks (#576) --- jenkins-scripts/lib/windows_library.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/lib/windows_library.bat b/jenkins-scripts/lib/windows_library.bat index 8a446b1a5..df931df9c 100644 --- a/jenkins-scripts/lib/windows_library.bat +++ b/jenkins-scripts/lib/windows_library.bat @@ -145,7 +145,7 @@ if "%GAZEBODISTRO_BRANCH%" == "" (set GAZEBODISTRO_BRANCH=master) if exist %gzdistro_dir% (rmdir /s /q %gzdistro_dir%) git clone https://github.com/ignition-tooling/gazebodistro %gzdistro_dir% -b %GAZEBODISTRO_BRANCH% -vcs import --retry 5 < "%gzdistro_dir%\%1" "%2" || goto :error +vcs import --retry 5 --force < "%gzdistro_dir%\%1" "%2" || goto :error vcs pull || goto :error goto :EOF From 56bc22ca1dc7a267d9fe5aed724eed0257f37754 Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Wed, 1 Dec 2021 09:35:06 -0800 Subject: [PATCH 030/540] Try to checkout matching gzdev branch for PR jobs (#572) If GitHub branch contains `ci_matching_branch/`, then try to checkout that branch from gzdev. Clone without --depth 1 to support this. Signed-off-by: Steve Peters --- .../docker/lib/docker_generate_dockerfile.bash | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash b/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash index c38b8ed54..6c5b1d9b1 100644 --- a/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash +++ b/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash @@ -33,24 +33,35 @@ export APT_PARAMS= GZDEV_DIR=${WORKSPACE}/gzdev GZDEV_BRANCH=${GZDEV_BRANCH:-master} +if [[ "$ghprbSourceBranch" =~ ci_matching_branch\/ ]]; then + GZDEV_TRY_BRANCH=$ghprbSourceBranch +fi dockerfile_install_gzdev_repos() { cat >> Dockerfile << DELIM_OSRF_REPO_GIT RUN rm -fr ${GZDEV_DIR} -RUN git clone --depth 1 https://github.com/ignition-tooling/gzdev -b ${GZDEV_BRANCH} ${GZDEV_DIR} +RUN git clone https://github.com/ignition-tooling/gzdev -b ${GZDEV_BRANCH} ${GZDEV_DIR} +RUN if [ -n $GZDEV_TRY_BRANCH ]; then \ + git -C ${GZDEV_DIR} fetch origin $GZDEV_TRY_BRANCH || true; \ + git -C ${GZDEV_DIR} checkout $GZDEV_TRY_BRANCH || true; \ + fi || true" +# print branch for informational purposes +RUN git -C ${GZDEV_DIR} branch DELIM_OSRF_REPO_GIT if [[ -n ${GZDEV_PROJECT_NAME} ]]; then # debian sid docker images does not return correct name so we need to use # force-linux-distro cat >> Dockerfile << DELIM_OSRF_REPO_GZDEV RUN ${GZDEV_DIR}/gzdev.py repository enable --project=${GZDEV_PROJECT_NAME} --force-linux-distro=${DISTRO} || ( git -C ${GZDEV_DIR} pull origin ${GZDEV_BRANCH} && \ + if [ -n $GZDEV_TRY_BRANCH ]; then git -C ${GZDEV_DIR} checkout $GZDEV_TRY_BRANCH; fi || true && \ ${GZDEV_DIR}/gzdev.py repository enable --project=${GZDEV_PROJECT_NAME} --force-linux-distro=${DISTRO} ) DELIM_OSRF_REPO_GZDEV else for repo in ${OSRF_REPOS_TO_USE}; do cat >> Dockerfile << DELIM_OSRF_REPO RUN ${GZDEV_DIR}/gzdev.py repository enable osrf ${repo} --force-linux-distro=${DISTRO} || ( git -C ${GZDEV_DIR} pull origin ${GZDEV_BRANCH} && \ + if [ -n $GZDEV_TRY_BRANCH ]; then git -C ${GZDEV_DIR} checkout $GZDEV_TRY_BRANCH; fi || true && \ ${GZDEV_DIR}/gzdev.py repository enable osrf ${repo} --force-linux-distro=${DISTRO} ) DELIM_OSRF_REPO done From 99524e1347c272920f4b384f3372037d3d38bf36 Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Wed, 1 Dec 2021 13:43:33 -0800 Subject: [PATCH 031/540] Fix syntax error, erroneous " Signed-off-by: Steve Peters --- jenkins-scripts/docker/lib/docker_generate_dockerfile.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash b/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash index 6c5b1d9b1..22e3a2720 100644 --- a/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash +++ b/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash @@ -45,7 +45,7 @@ RUN git clone https://github.com/ignition-tooling/gzdev -b ${GZDEV_BRANCH} ${GZD RUN if [ -n $GZDEV_TRY_BRANCH ]; then \ git -C ${GZDEV_DIR} fetch origin $GZDEV_TRY_BRANCH || true; \ git -C ${GZDEV_DIR} checkout $GZDEV_TRY_BRANCH || true; \ - fi || true" + fi || true # print branch for informational purposes RUN git -C ${GZDEV_DIR} branch DELIM_OSRF_REPO_GIT From 99e54bb50eb9d7e43794638e92cc0db91d70c608 Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Thu, 2 Dec 2021 17:16:23 -0800 Subject: [PATCH 032/540] Brew CI: try checking out matching branches (#575) Part of #564. Signed-off-by: Steve Peters --- .../lib/project-default-devel-homebrew-amd64.bash | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash b/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash index dfc9fa6a4..8f57bf355 100644 --- a/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash +++ b/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash @@ -61,9 +61,14 @@ echo '# BEGIN SECTION: setup the osrf/simulation tap' brew tap osrf/simulation echo '# END SECTION' -if [[ -n "${PULL_REQUEST_URL}" ]]; then - echo "# BEGIN SECTION: pulling ${PULL_REQUEST_URL}" - brew pull ${PULL_REQUEST_URL} +if [[ -n "${ghprbSourceBranch}" ]] && \ + [[ "${ghprbSourceBranch}" =~ ci_matching_branch\/ ]] +then + echo "# BEGIN SECTION: trying to checkout branch ${ghprbSourceBranch} from osrf/simulation" + pushd $(brew --repo osrf/simulation) + git fetch origin ${ghprbSourceBranch} || true + git checkout ${ghprbSourceBranch} || true + popd echo '# END SECTION' fi From 670d76f951652daa12729fb1287d22b5bd910b1d Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Fri, 3 Dec 2021 11:56:36 -0800 Subject: [PATCH 033/540] Use ci_matching_branch/ in bump_dependency.bash (#580) * Part of #564. * Fix replacing ign-fuel-tools branch name with main Replace branch name before swapping - for _ in DEP_LIB. * Make source code pull request after metadata The source code PRs will fail without the metadata, so commit the metadata first. Signed-off-by: Steve Peters --- release-repo-scripts/bump_dependency.bash | 84 +++++++++++------------ 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/release-repo-scripts/bump_dependency.bash b/release-repo-scripts/bump_dependency.bash index 7a6c859b6..5135dbf5f 100755 --- a/release-repo-scripts/bump_dependency.bash +++ b/release-repo-scripts/bump_dependency.bash @@ -181,7 +181,7 @@ commitAndPR() { # Sanity check that we're on a bump branch already local CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD) - if [[ ! $CURRENT_BRANCH = bump_* ]] + if [[ ! $CURRENT_BRANCH =~ bump_* ]] then echo -e "${RED}${REPO}: Something's wrong, trying to commit to branch ${CURRENT_BRANCH}.${DEFAULT}" return @@ -313,49 +313,10 @@ for ((i = 0; i < "${#LIBRARIES[@]}"; i++)); do PREV_VER="$((${VER}-1))" LIB_UPPER=`echo ${LIB#"ign-"} | tr a-z A-Z` ORG=${IGN_ORG} - BUMP_BRANCH="bump_${COLLECTION}_${LIB}${VER}" + BUMP_BRANCH="ci_matching_branch/bump_${COLLECTION}_${LIB}${VER}" echo -e "${BLUE_BG}Processing [${LIB}]${DEFAULT_BG}" - ################## - # source code - ################## - - echo -e "${GREEN}${LIB}: source code${DEFAULT}" - - cloneIfNeeded ${ORG} ${LIB} - startFromCleanBranch ${BUMP_BRANCH} main - - # Check if main branch of that library is the correct version - PROJECT_NAME="${LIB_}${VER}" - PROJECT_NAME="${PROJECT_NAME/ign_/ignition-}" - PROJECT="project.*(${PROJECT_NAME}" - if ! grep -q ${PROJECT} "CMakeLists.txt"; then - echo -e "${RED}Wrong project name on [CMakeLists.txt], looking for [$PROJECT_NAME].${DEFAULT}" - exit - fi - - echo -e "${GREEN}${LIB}: Updating source code${DEFAULT}" - for ((j = 0; j < "${#LIBRARIES[@]}"; j++)); do - - DEP_LIB=${LIBRARIES[$j]#"ign-"} - DEP_VER=${VERSIONS[$j]} - DEP_PREV_VER="$((${DEP_VER}-1))" - - find . -type f ! -name 'Changelog.md' ! -name 'Migration.md' -print0 | xargs -0 sed -i "s ${DEP_LIB}${DEP_PREV_VER} ${DEP_LIB}${DEP_VER} g" - - # Second run with _ instead of -, to support multiple variations of fuel-tools - DEP_LIB=${DEP_LIB//-/_} - find . -type f ! -name 'Changelog.md' ! -name 'Migration.md' -print0 | xargs -0 sed -i "s ${DEP_LIB}${DEP_PREV_VER} ${DEP_LIB}${DEP_VER} g" - - # Replace collection yaml branch names with main - if [[ "${LIB}" == "ign-${COLLECTION}" ]]; then - find . -type f -name "collection-${COLLECTION}.yaml" -print0 | xargs -0 sed -i "s ign-${DEP_LIB}${DEP_VER} main g" - fi - done - - commitAndPR ${ORG} main - ################## # release repo ################## @@ -384,7 +345,7 @@ for ((i = 0; i < "${#LIBRARIES[@]}"; i++)); do echo -e "${GREEN}${LIB}: homebrew${DEFAULT}" cd ${TEMP_DIR}/homebrew-simulation - startFromCleanBranch bump_${COLLECTION}_${LIB} master + startFromCleanBranch ${BUMP_BRANCH} master # expand ign-* to ignition-* FORMULA_BASE=${LIB/ign/ignition} @@ -469,6 +430,45 @@ for ((i = 0; i < "${#LIBRARIES[@]}"; i++)); do commitAndPR ${TOOLING_ORG} master + ################## + # source code + ################## + + echo -e "${GREEN}${LIB}: source code${DEFAULT}" + + cloneIfNeeded ${ORG} ${LIB} + startFromCleanBranch ${BUMP_BRANCH} main + + # Check if main branch of that library is the correct version + PROJECT_NAME="${LIB_}${VER}" + PROJECT_NAME="${PROJECT_NAME/ign_/ignition-}" + PROJECT="project.*(${PROJECT_NAME}" + if ! grep -q ${PROJECT} "CMakeLists.txt"; then + echo -e "${RED}Wrong project name on [CMakeLists.txt], looking for [$PROJECT_NAME].${DEFAULT}" + exit + fi + + echo -e "${GREEN}${LIB}: Updating source code${DEFAULT}" + for ((j = 0; j < "${#LIBRARIES[@]}"; j++)); do + + DEP_LIB=${LIBRARIES[$j]#"ign-"} + DEP_VER=${VERSIONS[$j]} + DEP_PREV_VER="$((${DEP_VER}-1))" + + find . -type f ! -name 'Changelog.md' ! -name 'Migration.md' -print0 | xargs -0 sed -i "s ${DEP_LIB}${DEP_PREV_VER} ${DEP_LIB}${DEP_VER} g" + + # Replace collection yaml branch names with main + if [[ "${LIB}" == "ign-${COLLECTION}" ]]; then + find . -type f -name "collection-${COLLECTION}.yaml" -print0 | xargs -0 sed -i "s ign-${DEP_LIB}${DEP_VER} main g" + fi + + # Second run with _ instead of -, to support multiple variations of fuel-tools + DEP_LIB=${DEP_LIB//-/_} + find . -type f ! -name 'Changelog.md' ! -name 'Migration.md' -print0 | xargs -0 sed -i "s ${DEP_LIB}${DEP_PREV_VER} ${DEP_LIB}${DEP_VER} g" + done + + commitAndPR ${ORG} main + # Collection ends here if ! [[ $VER == ?(-)+([0-9]) ]] ; then continue From c6f34ded833082937adedebd8983f95eeedce5dc Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Fri, 3 Dec 2021 18:29:17 -0800 Subject: [PATCH 034/540] Bumps in garden : ci_matching_branch/bump_garden_ign-transport12 (#582) Signed-off-by: Steve Peters --- jenkins-scripts/dsl/ignition_collection.dsl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index 6a1999d07..d62ed91ad 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -34,7 +34,7 @@ ignition_collections = [ 'rendering' : [ debbuild: 'ign-rendering7' , branch: 'main' ], 'sdformat' : [ debbuild: 'sdformat12' , branch: 'sdf12' ], 'fuel-tools': [ debbuild: 'ign-fuel-tools7' , branch: 'ign-fuel-tools7' ], - 'transport' : [ debbuild: 'ign-transport11' , branch: 'ign-transport11' ], + 'transport' : [ debbuild: 'ign-transport12' , branch: 'main' ], 'gui' : [ debbuild: 'ign-gui7' , branch: 'main' ], 'sensors' : [ debbuild: 'ign-sensors7' , branch: 'main' ], 'physics' : [ debbuild: 'ign-physics5' , branch: 'ign-physics5' ], From be8a55acb93016ba7b339278fc3d02328baa2f89 Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Fri, 3 Dec 2021 18:34:33 -0800 Subject: [PATCH 035/540] Add python tool to check for matching CI branches (#583) Use in windows CI to try matching gazebodistro branch. Fixes #579. Part of #564. Signed-off-by: Steve Peters --- .../docker/lib/docker_generate_dockerfile.bash | 2 +- .../project-default-devel-homebrew-amd64.bash | 2 +- jenkins-scripts/lib/windows_library.bat | 14 ++++++++++++++ .../tools/detect_ci_matching_branch.py | 16 ++++++++++++++++ 4 files changed, 32 insertions(+), 2 deletions(-) create mode 100644 jenkins-scripts/tools/detect_ci_matching_branch.py diff --git a/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash b/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash index 22e3a2720..a28c1c32d 100644 --- a/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash +++ b/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash @@ -33,7 +33,7 @@ export APT_PARAMS= GZDEV_DIR=${WORKSPACE}/gzdev GZDEV_BRANCH=${GZDEV_BRANCH:-master} -if [[ "$ghprbSourceBranch" =~ ci_matching_branch\/ ]]; then +if python3 ${SCRIPT_DIR}/../tools/detect_ci_matching_branch.py "${ghprbSourceBranch}"; then GZDEV_TRY_BRANCH=$ghprbSourceBranch fi diff --git a/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash b/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash index 8f57bf355..aea36cdc9 100644 --- a/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash +++ b/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash @@ -62,7 +62,7 @@ brew tap osrf/simulation echo '# END SECTION' if [[ -n "${ghprbSourceBranch}" ]] && \ - [[ "${ghprbSourceBranch}" =~ ci_matching_branch\/ ]] + python3 ${SCRIPT_DIR}/tools/detect_ci_matching_branch.py "${ghprbSourceBranch}" then echo "# BEGIN SECTION: trying to checkout branch ${ghprbSourceBranch} from osrf/simulation" pushd $(brew --repo osrf/simulation) diff --git a/jenkins-scripts/lib/windows_library.bat b/jenkins-scripts/lib/windows_library.bat index df931df9c..61605d1e1 100644 --- a/jenkins-scripts/lib/windows_library.bat +++ b/jenkins-scripts/lib/windows_library.bat @@ -139,12 +139,26 @@ goto :EOF :: :: arg1: Name of the yaml file in the gazebodistro repro :: arg2: directory destination (default .) +setlocal EnableDelayedExpansion set gzdistro_dir=gazebodistro if "%GAZEBODISTRO_BRANCH%" == "" (set GAZEBODISTRO_BRANCH=master) if exist %gzdistro_dir% (rmdir /s /q %gzdistro_dir%) git clone https://github.com/ignition-tooling/gazebodistro %gzdistro_dir% -b %GAZEBODISTRO_BRANCH% +:: Check if ci_matching_branch name is used +if "%ghprbSourceBranch%" == "" (echo ghprbSourceBranch is unset) else ( + python "%SCRIPT_DIR%\tools\detect_ci_matching_branch.py" "%ghprbSourceBranch%" + if "%ERRORLEVEL%" == "0" ( + echo trying to checkout branch %ghprbSourceBranch% from gazebodistro + git -C %gzdistro_dir% fetch origin %ghprbSourceBranch% || rem + git -C %gzdistro_dir% checkout %ghprbSourceBranch% || rem + ) else ( + echo branch name %ghprbSourceBranch% is not a match + ) + :: print branch for informational purposes + git -C %gzdistro_dir% branch +) vcs import --retry 5 --force < "%gzdistro_dir%\%1" "%2" || goto :error vcs pull || goto :error goto :EOF diff --git a/jenkins-scripts/tools/detect_ci_matching_branch.py b/jenkins-scripts/tools/detect_ci_matching_branch.py new file mode 100644 index 000000000..95e9d06a0 --- /dev/null +++ b/jenkins-scripts/tools/detect_ci_matching_branch.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python +from __future__ import print_function +import re +import sys + +if len(sys.argv) != 2: + print('need to branch name', file=sys.stderr) + exit() +branchName = sys.argv[1] + +pattern = 'ci_matching_branch/' +match = re.search(pattern, branchName) +if match: + print(f"{branchName} matches {pattern}") +else: + sys.exit(f"{branchName} does not match {pattern}") From a83b6d129b5a3c01367e5e89bcd6159a6aa3ef5d Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Fri, 3 Dec 2021 21:16:06 -0800 Subject: [PATCH 036/540] Fix _ handling in bump_dependency for formula (#584) There were some errors in osrf/homebrew-simulation#1704 with fuel_tools not properly recognized by the regex. Signed-off-by: Steve Peters --- release-repo-scripts/bump_dependency.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-repo-scripts/bump_dependency.bash b/release-repo-scripts/bump_dependency.bash index 5135dbf5f..c7b624924 100755 --- a/release-repo-scripts/bump_dependency.bash +++ b/release-repo-scripts/bump_dependency.bash @@ -371,7 +371,7 @@ for ((i = 0; i < "${#LIBRARIES[@]}"; i++)); do # libN sed -i -E "s ((${LIB#"ign-"}))${PREV_VER} \1${VER} g" $FORMULA - sed -i -E "s ((${LIB_#"ign-"}))${PREV_VER} \1${VER} g" $FORMULA + sed -i -E "s ((${LIB_#"ign_"}))${PREV_VER} \1${VER} g" $FORMULA # ign-libN -> main sed -i "s ${LIB}${PREV_VER} main g" $FORMULA # class IgnitionLibN From 988ca251d9c0c25e6b58a8b1695f3dc59cee95f4 Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Fri, 3 Dec 2021 21:24:43 -0800 Subject: [PATCH 037/540] Bumps in garden : ci_matching_branch/bump_garden_ign-fuel-tools8 (#581) Signed-off-by: Steve Peters --- jenkins-scripts/dsl/ignition_collection.dsl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index d62ed91ad..db290ef3f 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -33,7 +33,7 @@ ignition_collections = [ 'msgs' : [ debbuild: 'ign-msgs9' , branch: 'main' ], 'rendering' : [ debbuild: 'ign-rendering7' , branch: 'main' ], 'sdformat' : [ debbuild: 'sdformat12' , branch: 'sdf12' ], - 'fuel-tools': [ debbuild: 'ign-fuel-tools7' , branch: 'ign-fuel-tools7' ], + 'fuel-tools': [ debbuild: 'ign-fuel-tools8' , branch: 'main' ], 'transport' : [ debbuild: 'ign-transport12' , branch: 'main' ], 'gui' : [ debbuild: 'ign-gui7' , branch: 'main' ], 'sensors' : [ debbuild: 'ign-sensors7' , branch: 'main' ], From 53ff52c0aed7d1fb1074f9f631933ee8f8bce26d Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Mon, 6 Dec 2021 09:19:16 -0800 Subject: [PATCH 038/540] garden view: show main branch CI jobs (#586) Signed-off-by: Steve Peters --- jenkins-scripts/dsl/ignition_collection.dsl | 48 ++++++++++----------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index db290ef3f..4d135d51c 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -372,17 +372,17 @@ ignition_collection_jobs = ], 'garden' : [ 'ign_common-ign-4-win', - 'ign_fuel-tools-ign-7-win', - 'ign_gazebo-ign-6-win', - 'ign_gui-ign-6-win', - 'ign_launch-ign-5-win', - 'ign_msgs-ign-8-win', + 'ign_fuel-tools-ci-win', + 'ign_gazebo-ci-win', + 'ign_gui-ci-win', + 'ign_launch-ci-win', + 'ign_msgs-ci-win', 'ign_physics-ign-5-win', 'ign_plugin-ign-1-win', - 'ign_rendering-ign-6-win', - 'ign_sensors-ign-6-win', + 'ign_rendering-ci-win', + 'ign_sensors-ci-win', 'ign_tools-ign-1-win', - 'ign_transport-ign-11-win', + 'ign_transport-ci-win', 'ign_utils-ign-1-win', 'ignition_cmake-ci-ign-cmake2-bionic-amd64', 'ignition_cmake-ci-ign-cmake2-homebrew-amd64', @@ -396,20 +396,20 @@ ignition_collection_jobs = 'ignition_garden-ci-main-homebrew-amd64', 'ignition_garden-install-pkg-bionic-amd64', 'ignition_garden-install_bottle-homebrew-amd64', - 'ignition_fuel-tools-ci-ign-fuel-tools7-bionic-amd64', - 'ignition_fuel-tools-ci-ign-fuel-tools7-homebrew-amd64', + 'ignition_fuel-tools-ci-main-bionic-amd64', + 'ignition_fuel-tools-ci-main-homebrew-amd64', 'ignition_fuel-tools7-install-pkg-bionic-amd64', 'ignition_fuel-tools7-install_bottle-homebrew-amd64', - 'ignition_gazebo-ci-ign-gazebo6-bionic-amd64', - 'ignition_gazebo-ci-ign-gazebo6-homebrew-amd64', + 'ignition_gazebo-ci-main-bionic-amd64', + 'ignition_gazebo-ci-main-homebrew-amd64', 'ignition_gazebo6-install-pkg-bionic-amd64', 'ignition_gazebo6-install_bottle-homebrew-amd64', - 'ignition_gui-ci-ign-gui6-bionic-amd64', - 'ignition_gui-ci-ign-gui6-homebrew-amd64', + 'ignition_gui-ci-main-bionic-amd64', + 'ignition_gui-ci-main-homebrew-amd64', 'ignition_gui6-install-pkg-bionic-amd64', 'ignition_gui6-install_bottle-homebrew-amd64', - 'ignition_launch-ci-ign-launch5-bionic-amd64', - 'ignition_launch-ci-ign-launch5-homebrew-amd64', + 'ignition_launch-ci-main-bionic-amd64', + 'ignition_launch-ci-main-homebrew-amd64', 'ignition_launch5-install-pkg-bionic-amd64', 'ignition_launch5-install_bottle-homebrew-amd64', 'ignition_math-ci-ign-math6-bionic-amd64', @@ -417,8 +417,8 @@ ignition_collection_jobs = 'ignition_math-ci-ign-math6-windows7-amd64', 'ignition_math6-install-pkg-bionic-amd64', 'ignition_math6-install_bottle-homebrew-amd64', - 'ignition_msgs-ci-ign-msgs8-bionic-amd64', - 'ignition_msgs-ci-ign-msgs8-homebrew-amd64', + 'ignition_msgs-ci-main-bionic-amd64', + 'ignition_msgs-ci-main-homebrew-amd64', 'ignition_msgs8-install-pkg-bionic-amd64', 'ignition_msgs8-install_bottle-homebrew-amd64', 'ignition_physics-ci-ign-physics5-bionic-amd64', @@ -429,20 +429,20 @@ ignition_collection_jobs = 'ignition_plugin-ci-ign-plugin1-homebrew-amd64', 'ignition_plugin-install-pkg-bionic-amd64', 'ignition_plugin1-install_bottle-homebrew-amd64', - 'ignition_rendering-ci-ign-rendering6-bionic-amd64', - 'ignition_rendering-ci-ign-rendering6-homebrew-amd64', + 'ignition_rendering-ci-main-bionic-amd64', + 'ignition_rendering-ci-main-homebrew-amd64', 'ignition_rendering6-install-pkg-bionic-amd64', 'ignition_rendering6-install_bottle-homebrew-amd64', - 'ignition_sensors-ci-ign-sensors6-bionic-amd64', - 'ignition_sensors-ci-ign-sensors6-homebrew-amd64', + 'ignition_sensors-ci-main-bionic-amd64', + 'ignition_sensors-ci-main-homebrew-amd64', 'ignition_sensors6-install-pkg-bionic-amd64', 'ignition_sensors6-install_bottle-homebrew-amd64', 'ignition_tools-ci-ign-tools1-bionic-amd64', 'ignition_tools-ci-ign-tools1-homebrew-amd64', 'ignition_tools-install-pkg-bionic-amd64', 'ignition_tools1-install_bottle-homebrew-amd64', - 'ignition_transport-ci-ign-transport11-bionic-amd64', - 'ignition_transport-ci-ign-transport11-homebrew-amd64', + 'ignition_transport-ci-main-bionic-amd64', + 'ignition_transport-ci-main-homebrew-amd64', 'ignition_transport11-install-pkg-bionic-amd64', 'ignition_transport11-install_bottle-homebrew-amd64', 'ignition_utils-ci-ign-utils1-bionic-amd64', From 2e53e7569f1a950931af3b2c3e3cb009f8b5f4dd Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 7 Dec 2021 18:52:16 +0100 Subject: [PATCH 039/540] Fail hard of Windows when a command fails (#569) Do not use /b on error routines to let Windows kill the whole terminal so the build scripts don't keep running. --- jenkins-scripts/lib/colcon-default-devel-windows.bat | 2 +- jenkins-scripts/lib/windows_library.bat | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/jenkins-scripts/lib/colcon-default-devel-windows.bat b/jenkins-scripts/lib/colcon-default-devel-windows.bat index 32de2caad..ef0a76b58 100644 --- a/jenkins-scripts/lib/colcon-default-devel-windows.bat +++ b/jenkins-scripts/lib/colcon-default-devel-windows.bat @@ -130,4 +130,4 @@ goto :EOF :error - error routine echo Failed with error #%errorlevel%. -exit /b %errorlevel% +exit %errorlevel% diff --git a/jenkins-scripts/lib/windows_library.bat b/jenkins-scripts/lib/windows_library.bat index 61605d1e1..97fdaa928 100644 --- a/jenkins-scripts/lib/windows_library.bat +++ b/jenkins-scripts/lib/windows_library.bat @@ -295,4 +295,4 @@ goto :EOF :error - error routine :: echo Failed in windows_library with error #%errorlevel%. -exit /B %errorlevel% +exit %errorlevel% From f83b2db60f643289f5f67bdfb9e79df04b533d22 Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Fri, 10 Dec 2021 07:55:52 -0800 Subject: [PATCH 040/540] sdformat: don't compile with USE_INTERNAL_URDF (#590) Signed-off-by: Steve Peters --- jenkins-scripts/docker/sdformat-abichecker.bash | 4 ---- 1 file changed, 4 deletions(-) diff --git a/jenkins-scripts/docker/sdformat-abichecker.bash b/jenkins-scripts/docker/sdformat-abichecker.bash index be9c1687e..7f56ed7b2 100644 --- a/jenkins-scripts/docker/sdformat-abichecker.bash +++ b/jenkins-scripts/docker/sdformat-abichecker.bash @@ -19,10 +19,6 @@ fi export ABI_JOB_SOFTWARE_NAME="sdformat" export ABI_JOB_PKG_DEPENDENCIES_VAR_NAME="SDFORMAT_BASE_DEPENDENCIES" -if [[ ${SDFORMAT_MAJOR_VERSION} -ge 6 ]]; then - export BUILDING_EXTRA_CMAKE_PARAMS="-DUSE_INTERNAL_URDF:BOOL=True" -fi - if [[ ${SDFORMAT_MAJOR_VERSION} -ge 8 ]]; then export NEED_C17_COMPILER=true fi From 47fb17b9ba9bf0eb4e505dfeb9dc3cc392a2a09a Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Fri, 10 Dec 2021 07:57:14 -0800 Subject: [PATCH 041/540] sdformat-abichecker: set GZDEV_PROJECT_NAME (#589) * sdformat-abichecker: set GZDEV_PROJECT_NAME Needed by ignitionrobotics/sdformat#780 Signed-off-by: Steve Peters * detect_ci_matching_branch: use format() This is compatible with more verisons of python3. Signed-off-by: Steve Peters --- jenkins-scripts/docker/sdformat-abichecker.bash | 6 +++++- jenkins-scripts/tools/detect_ci_matching_branch.py | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/jenkins-scripts/docker/sdformat-abichecker.bash b/jenkins-scripts/docker/sdformat-abichecker.bash index 7f56ed7b2..1bcd61f83 100644 --- a/jenkins-scripts/docker/sdformat-abichecker.bash +++ b/jenkins-scripts/docker/sdformat-abichecker.bash @@ -22,7 +22,11 @@ export ABI_JOB_PKG_DEPENDENCIES_VAR_NAME="SDFORMAT_BASE_DEPENDENCIES" if [[ ${SDFORMAT_MAJOR_VERSION} -ge 8 ]]; then export NEED_C17_COMPILER=true fi - + +# default to use stable repos export ABI_JOB_REPOS="stable" +# set GZDEV_PROJECT_NAME so it can override repos if necessary +export GZDEV_PROJECT_NAME=${ABI_JOB_SOFTWARE_NAME}${SDFORMAT_MAJOR_VERSION} + . ${SCRIPT_DIR}/lib/generic-abi-base.bash diff --git a/jenkins-scripts/tools/detect_ci_matching_branch.py b/jenkins-scripts/tools/detect_ci_matching_branch.py index 95e9d06a0..3ee892f1e 100644 --- a/jenkins-scripts/tools/detect_ci_matching_branch.py +++ b/jenkins-scripts/tools/detect_ci_matching_branch.py @@ -4,7 +4,7 @@ import sys if len(sys.argv) != 2: - print('need to branch name', file=sys.stderr) + print('need to specify branch name', file=sys.stderr) exit() branchName = sys.argv[1] @@ -13,4 +13,4 @@ if match: print(f"{branchName} matches {pattern}") else: - sys.exit(f"{branchName} does not match {pattern}") + sys.exit("{} does not match {}".format(branchName, pattern)) From f0c87c095e44e92435dd2cd22d782d2ec02c228d Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Fri, 10 Dec 2021 09:29:48 -0800 Subject: [PATCH 042/540] sdformat: again stop -DUSE_INTERNAL_URDF=ON (#592) I missed one spot in #590. Signed-off-by: Steve Peters --- jenkins-scripts/docker/sdformat-compilation.bash | 4 ---- 1 file changed, 4 deletions(-) diff --git a/jenkins-scripts/docker/sdformat-compilation.bash b/jenkins-scripts/docker/sdformat-compilation.bash index f0ca4b85f..8cfb50a1f 100644 --- a/jenkins-scripts/docker/sdformat-compilation.bash +++ b/jenkins-scripts/docker/sdformat-compilation.bash @@ -18,10 +18,6 @@ fi export BUILDING_SOFTWARE_DIRECTORY="sdformat" -if [[ ${SDFORMAT_MAJOR_VERSION} -ge 6 ]]; then - export BUILDING_EXTRA_CMAKE_PARAMS="-DUSE_INTERNAL_URDF:BOOL=True" -fi - if [[ ${SDFORMAT_MAJOR_VERSION} -ge 8 ]]; then export NEED_C17_COMPILER=true fi From a899e66d7eae05eb8d5fd481bc419a0279758a80 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 21 Dec 2021 01:23:43 +0100 Subject: [PATCH 043/540] Clean up any possible lock on gitconfig from aborted builds (#591) Signed-off-by: Jose Luis Rivero --- jenkins-scripts/lib/_homebrew_cleanup.bash | 3 +++ 1 file changed, 3 insertions(+) diff --git a/jenkins-scripts/lib/_homebrew_cleanup.bash b/jenkins-scripts/lib/_homebrew_cleanup.bash index 95a0f01d7..824b44533 100644 --- a/jenkins-scripts/lib/_homebrew_cleanup.bash +++ b/jenkins-scripts/lib/_homebrew_cleanup.bash @@ -45,5 +45,8 @@ export HOMEBREW_GIT_NAME=${GIT_AUTHOR_NAME} export GIT_AUTHOR_EMAIL="osrfbuild@osrfoundation.org" export GIT_COMMITTER_EMAIL=${GIT_AUTHOR_EMAIL} export HOMEBREW_GIT_EMAIL=${GIT_AUTHOR_EMAIL} +# Cleanup any existing lock left behind from aborted builds +# see https://github.com/osrf/buildfarmer/issues/257 +rm -f ${HOME}/.gitconfig.lock git config --global user.name "${GIT_AUTHOR_NAME}" git config --global user.email "${GIT_AUTHOR_EMAIL}" From efbdacdd65b32558bf535e388b6b3488185b96c5 Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Mon, 20 Dec 2021 23:53:26 -0800 Subject: [PATCH 044/540] release.py: only use arm* nodes for arm* builds (#599) We currently always have an arm node in our buildfarm, and sometimes arm builds fail in qemu, so update the docker node label tag to only use arm agents for arm builds. Signed-off-by: Steve Peters --- release.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release.py b/release.py index a76f60d0d..dce19ee40 100755 --- a/release.py +++ b/release.py @@ -616,7 +616,7 @@ def go(argv): continue # Need to use JENKINS_NODE_TAG parameter for large memory nodes # since it runs qemu emulation - linux_platform_params['JENKINS_NODE_TAG'] = 'linux-' + a + '|| large-memory' + linux_platform_params['JENKINS_NODE_TAG'] = 'linux-' + a elif ('ignition-physics' in args.package_alias): linux_platform_params['JENKINS_NODE_TAG'] = 'large-memory' From b9bfc6b11622951e9544049a21cfdff23d6b7c56 Mon Sep 17 00:00:00 2001 From: Michael Carroll Date: Tue, 21 Dec 2021 15:48:52 -0600 Subject: [PATCH 045/540] ignition.dsl: colcon for ign-math on windows (#602) Signed-off-by: Michael Carroll --- jenkins-scripts/dsl/ignition.dsl | 1 + jenkins-scripts/dsl/ignition_collection.dsl | 8 ++++---- jenkins-scripts/ign_math-default-devel-windows-amd64.bat | 4 +++- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index ddc45bf25..b15d95673 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -95,6 +95,7 @@ ignition_colcon_win = [ 'common', 'gazebo', 'gui', 'launch', + 'math', 'msgs', 'physics', 'plugin', diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index 4d135d51c..70b74f15d 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -52,6 +52,7 @@ ignition_collection_jobs = 'ign_fuel-tools-ign-4-win', 'ign_gazebo-ign-3-win', 'ign_gui-ign-3-win', + 'ign_math-ign-6-win', 'ign_msgs-ign-5-win', 'ign_physics-ign-2-win', 'ign_plugin-ign-1-win', @@ -89,7 +90,6 @@ ignition_collection_jobs = 'ignition_launch2-install_bottle-homebrew-amd64', 'ignition_math-ci-ign-math6-bionic-amd64', 'ignition_math-ci-ign-math6-homebrew-amd64', - 'ignition_math-ci-ign-math6-windows7-amd64', 'ignition_math6-install-pkg-bionic-amd64', 'ignition_math6-install_bottle-homebrew-amd64', 'ignition_msgs-ci-ign-msgs5-bionic-amd64', @@ -130,6 +130,7 @@ ignition_collection_jobs = 'ign_fuel-tools-ign-5-win', 'ign_gazebo-ign-4-win', 'ign_gui-ign-4-win', + 'ign_math-ign-6-win', 'ign_msgs-ign-6-win', 'ign_physics-ign-3-win', 'ign_plugin-ign-1-win', @@ -167,7 +168,6 @@ ignition_collection_jobs = 'ignition_launch3-install_bottle-homebrew-amd64', 'ignition_math-ci-ign-math6-bionic-amd64', 'ignition_math-ci-ign-math6-homebrew-amd64', - 'ignition_math-ci-ign-math6-windows7-amd64', 'ignition_math6-install-pkg-bionic-amd64', 'ignition_math6-install_bottle-homebrew-amd64', 'ignition_msgs-ci-ign-msgs6-bionic-amd64', @@ -208,6 +208,7 @@ ignition_collection_jobs = 'ign_fuel-tools-ign-6-win', 'ign_gazebo-ign-5-win', 'ign_gui-ign-5-win', + 'ign_math-ign-6-win', 'ign_msgs-ign-7-win', 'ign_physics-ign-4-win', 'ign_plugin-ign-1-win', @@ -246,7 +247,6 @@ ignition_collection_jobs = 'ignition_launch4-install_bottle-homebrew-amd64', 'ignition_math-ci-ign-math6-bionic-amd64', 'ignition_math-ci-ign-math6-homebrew-amd64', - 'ignition_math-ci-ign-math6-windows7-amd64', 'ignition_math6-install-pkg-bionic-amd64', 'ignition_math6-install_bottle-homebrew-amd64', 'ignition_msgs-ci-ign-msgs7-bionic-amd64', @@ -376,6 +376,7 @@ ignition_collection_jobs = 'ign_gazebo-ci-win', 'ign_gui-ci-win', 'ign_launch-ci-win', + 'ign_math-ci-win', 'ign_msgs-ci-win', 'ign_physics-ign-5-win', 'ign_plugin-ign-1-win', @@ -414,7 +415,6 @@ ignition_collection_jobs = 'ignition_launch5-install_bottle-homebrew-amd64', 'ignition_math-ci-ign-math6-bionic-amd64', 'ignition_math-ci-ign-math6-homebrew-amd64', - 'ignition_math-ci-ign-math6-windows7-amd64', 'ignition_math6-install-pkg-bionic-amd64', 'ignition_math6-install_bottle-homebrew-amd64', 'ignition_msgs-ci-main-bionic-amd64', diff --git a/jenkins-scripts/ign_math-default-devel-windows-amd64.bat b/jenkins-scripts/ign_math-default-devel-windows-amd64.bat index 9beefc9db..e5c6acf92 100644 --- a/jenkins-scripts/ign_math-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_math-default-devel-windows-amd64.bat @@ -3,5 +3,7 @@ set SCRIPT_DIR=%~dp0 set VCS_DIRECTORY=ign-math set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true +set COLCON_PACKAGE=ignition-math +set COLCON_AUTO_MAJOR_VERSION=true -call "%SCRIPT_DIR%/lib/generic-default-devel-windows.bat" +call "%SCRIPT_DIR%\lib\colcon-default-devel-windows.bat" From a5501274fdb8aec471bf4e1701842788eada02ec Mon Sep 17 00:00:00 2001 From: Louise Poubel Date: Wed, 22 Dec 2021 11:09:03 -0800 Subject: [PATCH 046/540] =?UTF-8?q?=F0=9F=8F=81=20Dome=20EOL=20(#601)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🏁 Dome EOL Signed-off-by: Louise Poubel --- jenkins-scripts/dsl/ignition.dsl | 18 ++--- jenkins-scripts/dsl/ignition_collection.dsl | 81 --------------------- jenkins-scripts/dsl/sdformat.dsl | 2 +- jenkins-scripts/tools/buildcop_stats.bash | 2 +- terminal-dashboard/_dashboard_lib.sh | 18 ----- 5 files changed, 11 insertions(+), 110 deletions(-) diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index b15d95673..8f43b23fd 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -45,18 +45,18 @@ ignition_no_test = [ ] // main, ABI check, install pkg) ignition_branches = [ 'cmake' : [ '2' ], 'common' : [ '1', '3', '4' ], - 'fuel-tools' : [ '1', '4', '5', '6', '7' ], - 'gazebo' : [ '3', '4', '5', '6' ], - 'gui' : [ '0', '3', '4', '5', '6' ], - 'launch' : [ '2', '3', '4', '5' ], + 'fuel-tools' : [ '1', '4', '6', '7' ], + 'gazebo' : [ '3', '5', '6' ], + 'gui' : [ '0', '3', '5', '6' ], + 'launch' : [ '2', '4', '5' ], 'math' : [ '4', '6' ], - 'msgs' : [ '1', '5', '6', '7', '8' ], - 'physics' : [ '2', '3', '4', '5' ], + 'msgs' : [ '1', '5', '7', '8' ], + 'physics' : [ '2', '4', '5' ], 'plugin' : [ '1' ], - 'rendering' : [ '3', '4', '5', '6' ], - 'sensors' : [ '3', '4', '5', '6' ], + 'rendering' : [ '3', '5', '6' ], + 'sensors' : [ '3', '5', '6' ], 'tools' : [ '1' ], - 'transport' : [ '4', '8', '9', '10', '11' ], + 'transport' : [ '4', '8', '10', '11' ], 'utils' : [ '1' ]] // DESC: prerelease branches are managed as any other supported branches for // special cases different to major branches: get compilation CI on the branch diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index 70b74f15d..b7b8a7a7e 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -10,9 +10,6 @@ ignition_collections = [ [ name : 'citadel', distros : [ 'bionic' ], ], - [ name : 'dome', - distros : [ 'focal' ], - ], [ name : 'edifice', distros : [ 'focal' ], ], @@ -125,84 +122,6 @@ ignition_collection_jobs = 'sdformat-ci-sdformat9-windows7-amd64', 'sdformat-install-sdformat9_pkg-bionic-amd64' ], - 'dome' : [ - 'ign_common-ign-3-win', - 'ign_fuel-tools-ign-5-win', - 'ign_gazebo-ign-4-win', - 'ign_gui-ign-4-win', - 'ign_math-ign-6-win', - 'ign_msgs-ign-6-win', - 'ign_physics-ign-3-win', - 'ign_plugin-ign-1-win', - 'ign_rendering-ign-4-win', - 'ign_sensors-ign-4-win', - 'ign_tools-ign-1-win', - 'ign_transport-ign-9-win', - 'ignition_cmake-ci-ign-cmake2-bionic-amd64', - 'ignition_cmake-ci-ign-cmake2-homebrew-amd64', - 'ignition_cmake-ci-ign-cmake2-windows7-amd64', - 'ignition_cmake2-install-pkg-bionic-amd64', - 'ignition_cmake2-install_bottle-homebrew-amd64', - 'ignition_common-ci-ign-common3-bionic-amd64', - 'ignition_common-ci-ign-common3-homebrew-amd64', - 'ignition_common3-install-pkg-bionic-amd64', - 'ignition_common3-install_bottle-homebrew-amd64', - 'ignition_dome-ci-main-homebrew-amd64', - 'ignition_dome-install-pkg-bionic-amd64', - 'ignition_dome-install_bottle-homebrew-amd64', - 'ignition_fuel-tools-ci-ign-fuel-tools5-bionic-amd64', - 'ignition_fuel-tools-ci-ign-fuel-tools5-homebrew-amd64', - 'ignition_fuel-tools5-install-pkg-bionic-amd64', - 'ignition_fuel-tools5-install_bottle-homebrew-amd64', - 'ignition_gazebo-ci-ign-gazebo4-bionic-amd64', - 'ignition_gazebo-ci-ign-gazebo4-homebrew-amd64', - 'ignition_gazebo4-install-pkg-bionic-amd64', - 'ignition_gazebo4-install_bottle-homebrew-amd64', - 'ignition_gui-ci-ign-gui4-bionic-amd64', - 'ignition_gui-ci-ign-gui4-homebrew-amd64', - 'ignition_gui4-install-pkg-bionic-amd64', - 'ignition_gui4-install_bottle-homebrew-amd64', - 'ignition_launch-ci-ign-launch3-bionic-amd64', - 'ignition_launch-ci-ign-launch3-homebrew-amd64', - 'ignition_launch3-install-pkg-bionic-amd64', - 'ignition_launch3-install_bottle-homebrew-amd64', - 'ignition_math-ci-ign-math6-bionic-amd64', - 'ignition_math-ci-ign-math6-homebrew-amd64', - 'ignition_math6-install-pkg-bionic-amd64', - 'ignition_math6-install_bottle-homebrew-amd64', - 'ignition_msgs-ci-ign-msgs6-bionic-amd64', - 'ignition_msgs-ci-ign-msgs6-homebrew-amd64', - 'ignition_msgs6-install-pkg-bionic-amd64', - 'ignition_msgs6-install_bottle-homebrew-amd64', - 'ignition_physics-ci-ign-physics3-bionic-amd64', - 'ignition_physics-ci-ign-physics3-homebrew-amd64', - 'ignition_physics3-install-pkg-bionic-amd64', - 'ignition_physics3-install_bottle-homebrew-amd64', - 'ignition_plugin-ci-ign-plugin1-bionic-amd64', - 'ignition_plugin-ci-ign-plugin1-homebrew-amd64', - 'ignition_plugin-install-pkg-bionic-amd64', - 'ignition_plugin1-install_bottle-homebrew-amd64', - 'ignition_rendering-ci-ign-rendering4-bionic-amd64', - 'ignition_rendering-ci-ign-rendering4-homebrew-amd64', - 'ignition_rendering4-install-pkg-bionic-amd64', - 'ignition_rendering4-install_bottle-homebrew-amd64', - 'ignition_sensors-ci-ign-sensors4-bionic-amd64', - 'ignition_sensors-ci-ign-sensors4-homebrew-amd64', - 'ignition_sensors4-install-pkg-bionic-amd64', - 'ignition_sensors4-install_bottle-homebrew-amd64', - 'ignition_tools-ci-ign-tools1-bionic-amd64', - 'ignition_tools-ci-ign-tools1-homebrew-amd64', - 'ignition_tools-install-pkg-bionic-amd64', - 'ignition_tools1-install_bottle-homebrew-amd64', - 'ignition_transport-ci-ign-transport9-bionic-amd64', - 'ignition_transport-ci-ign-transport9-homebrew-amd64', - 'ignition_transport9-install-pkg-bionic-amd64', - 'ignition_transport9-install_bottle-homebrew-amd64', - 'sdformat-ci-sdformat10-bionic-amd64', - 'sdformat-ci-sdformat10-homebrew-amd64', - 'sdformat-ci-sdformat10-windows7-amd64', - 'sdformat-install-sdformat10_pkg-bionic-amd64' - ], 'edifice' : [ 'ign_common-ign-4-win', 'ign_fuel-tools-ign-6-win', diff --git a/jenkins-scripts/dsl/sdformat.dsl b/jenkins-scripts/dsl/sdformat.dsl index 3da399eea..a1439261b 100644 --- a/jenkins-scripts/dsl/sdformat.dsl +++ b/jenkins-scripts/dsl/sdformat.dsl @@ -1,7 +1,7 @@ import _configs_.* import javaposse.jobdsl.dsl.Job -def sdformat_supported_branches = [ 'sdformat6' , 'sdformat9', 'sdformat10', 'sdformat11', 'sdformat12' ] +def sdformat_supported_branches = [ 'sdformat6' , 'sdformat9', 'sdformat11', 'sdformat12' ] def sdformat_gz11_branches = [ 'sdformat9', 'sdformat10', 'sdformat11', 'sdformat12', 'main' ] // nightly and prereleases def extra_sdformat_debbuilder = [ ] diff --git a/jenkins-scripts/tools/buildcop_stats.bash b/jenkins-scripts/tools/buildcop_stats.bash index 5c7d75926..c2c03bbcc 100755 --- a/jenkins-scripts/tools/buildcop_stats.bash +++ b/jenkins-scripts/tools/buildcop_stats.bash @@ -1,6 +1,6 @@ #!/bin/bash -views='ign-citadel ign-dome' +views='ign-citadel ign-edifice ign-fortress ign-garden' echo "# Build Cop Report $(date +%Y-%m-%d)" echo "## Aggregate Results as of $(date '+%Y-%m-%d %H:%M:%S')" diff --git a/terminal-dashboard/_dashboard_lib.sh b/terminal-dashboard/_dashboard_lib.sh index 373d80f2e..1e0bb5536 100644 --- a/terminal-dashboard/_dashboard_lib.sh +++ b/terminal-dashboard/_dashboard_lib.sh @@ -24,24 +24,6 @@ get_libraries_by_collection() ignition-gazebo3 ignition-launch2 ignition-citadel" - elif [ "$COLLECTION" = "dome" ]; then - LIBS=" - ignition-cmake2 - ignition-math6 - ignition-tools - ignition-common3 - ignition-msgs6 - ignition-transport9 - ignition-fuel-tools5 - ignition-plugin - ignition-rendering4 - sdformat10 - ignition-physics3 - ignition-sensors4 - ignition-gui4 - ignition-gazebo4 - ignition-launch3 - ignition-dome" elif [ "$COLLECTION" = "edifice" ]; then LIBS=" ignition-cmake2 From 81e5794f67850a89c44e4f1e117a5cc58be58ddd Mon Sep 17 00:00:00 2001 From: Martin Pecka Date: Thu, 23 Dec 2021 03:45:13 +0100 Subject: [PATCH 047/540] EOL template: add a task about notifying the community (#604) Signed-off-by: Martin Pecka --- .github/ISSUE_TEMPLATE/eol_collection.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/ISSUE_TEMPLATE/eol_collection.md b/.github/ISSUE_TEMPLATE/eol_collection.md index a201cedcd..8f56649a7 100644 --- a/.github/ISSUE_TEMPLATE/eol_collection.md +++ b/.github/ISSUE_TEMPLATE/eol_collection.md @@ -13,6 +13,7 @@ When opening PRs, add a link back to this issue for easier tracking. --> +- [ ] Write a post to https://community.gazebosim.org/ announcing the community it has last X weeks for finishing their PRs. - [ ] Check library versions that can be EOL’d on the [versions table](https://github.com/ignitionrobotics/docs/blob/master/tools/versions.md) * - [ ] Make releases with outstanding changes in each library about to be EOl'ed From b482cab0c49fa60f1f658d4d15b0ceb0af78f791 Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Mon, 27 Dec 2021 09:21:05 -0800 Subject: [PATCH 048/540] sdformat.dsl: re-enable sdf10 until release (#605) We have not yet made the final release of libsdformat10, so re-enable it as a supported branch to allow ABI-checker CI jobs. Signed-off-by: Steve Peters --- jenkins-scripts/dsl/sdformat.dsl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/sdformat.dsl b/jenkins-scripts/dsl/sdformat.dsl index a1439261b..3da399eea 100644 --- a/jenkins-scripts/dsl/sdformat.dsl +++ b/jenkins-scripts/dsl/sdformat.dsl @@ -1,7 +1,7 @@ import _configs_.* import javaposse.jobdsl.dsl.Job -def sdformat_supported_branches = [ 'sdformat6' , 'sdformat9', 'sdformat11', 'sdformat12' ] +def sdformat_supported_branches = [ 'sdformat6' , 'sdformat9', 'sdformat10', 'sdformat11', 'sdformat12' ] def sdformat_gz11_branches = [ 'sdformat9', 'sdformat10', 'sdformat11', 'sdformat12', 'main' ] // nightly and prereleases def extra_sdformat_debbuilder = [ ] From cdf4f649c89c30cc54714524ddfd8b3faf52e96f Mon Sep 17 00:00:00 2001 From: Louise Poubel Date: Mon, 27 Dec 2021 10:35:55 -0800 Subject: [PATCH 049/540] Bumps in garden : bump_garden_ign-math7 (#607) Signed-off-by: Louise Poubel --- jenkins-scripts/dsl/ignition_collection.dsl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index b7b8a7a7e..f061bf348 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -23,7 +23,7 @@ ignition_collections = [ nightly_jobs: [ 'tools' : [ debbuild: 'ign-tools' , branch: 'ign-tools1' ], 'cmake' : [ debbuild: 'ign-cmake2' , branch: 'ign-cmake2' ], - 'math' : [ debbuild: 'ign-math6' , branch: 'ign-math6' ], + 'math' : [ debbuild: 'ign-math7' , branch: 'main' ], 'plugin' : [ debbuild: 'ign-plugin' , branch: 'ign-plugin1' ], 'utils' : [ debbuild: 'ign-utils1' , branch: 'ign-utils1' ], 'common' : [ debbuild: 'ign-common5' , branch: 'main' ], From cccba7434d8834726c2d16ec4097dbe82e69414e Mon Sep 17 00:00:00 2001 From: Louise Poubel Date: Mon, 27 Dec 2021 11:44:44 -0800 Subject: [PATCH 050/540] Bumps in garden : ci_matching_branch/bump_garden_ign-physics6 (#609) Signed-off-by: Louise Poubel --- jenkins-scripts/dsl/ignition_collection.dsl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index f061bf348..d35484c4a 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -34,7 +34,7 @@ ignition_collections = [ 'transport' : [ debbuild: 'ign-transport12' , branch: 'main' ], 'gui' : [ debbuild: 'ign-gui7' , branch: 'main' ], 'sensors' : [ debbuild: 'ign-sensors7' , branch: 'main' ], - 'physics' : [ debbuild: 'ign-physics5' , branch: 'ign-physics5' ], + 'physics' : [ debbuild: 'ign-physics6' , branch: 'main' ], 'gazebo' : [ debbuild: 'ign-gazebo7' , branch: 'main' ], 'launch' : [ debbuild: 'ign-launch6' , branch: 'main' ], 'garden' : [ debbuild: 'ign-garden' , branch: 'main' ], From 6707a6630495d76f9f92987815951ad3b81ea8c6 Mon Sep 17 00:00:00 2001 From: Louise Poubel Date: Mon, 27 Dec 2021 11:44:50 -0800 Subject: [PATCH 051/540] Bumps in garden : ci_matching_branch/bump_garden_sdformat13 (#608) Signed-off-by: Louise Poubel --- jenkins-scripts/dsl/ignition_collection.dsl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index d35484c4a..4e5a84b57 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -29,7 +29,7 @@ ignition_collections = [ 'common' : [ debbuild: 'ign-common5' , branch: 'main' ], 'msgs' : [ debbuild: 'ign-msgs9' , branch: 'main' ], 'rendering' : [ debbuild: 'ign-rendering7' , branch: 'main' ], - 'sdformat' : [ debbuild: 'sdformat12' , branch: 'sdf12' ], + 'sdformat' : [ debbuild: 'sdformat13' , branch: 'main' ], 'fuel-tools': [ debbuild: 'ign-fuel-tools8' , branch: 'main' ], 'transport' : [ debbuild: 'ign-transport12' , branch: 'main' ], 'gui' : [ debbuild: 'ign-gui7' , branch: 'main' ], From 60b460d073a2754fe6862cec9a92e4a9d4029ce7 Mon Sep 17 00:00:00 2001 From: Louise Poubel Date: Mon, 27 Dec 2021 18:07:12 -0800 Subject: [PATCH 052/540] detect_ci_matching_branch: support python < 3.6 (#612) Signed-off-by: Louise Poubel --- jenkins-scripts/tools/detect_ci_matching_branch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/tools/detect_ci_matching_branch.py b/jenkins-scripts/tools/detect_ci_matching_branch.py index 3ee892f1e..54ad8506b 100644 --- a/jenkins-scripts/tools/detect_ci_matching_branch.py +++ b/jenkins-scripts/tools/detect_ci_matching_branch.py @@ -11,6 +11,6 @@ pattern = 'ci_matching_branch/' match = re.search(pattern, branchName) if match: - print(f"{branchName} matches {pattern}") + print(branchName, "matches", pattern) else: sys.exit("{} does not match {}".format(branchName, pattern)) From cf450823be3d7846d3d09be445fb57f44386673f Mon Sep 17 00:00:00 2001 From: Louise Poubel Date: Tue, 28 Dec 2021 18:02:00 -0800 Subject: [PATCH 053/540] sdformat.dsl: re-disable sdf10 (#613) Signed-off-by: Louise Poubel --- jenkins-scripts/dsl/sdformat.dsl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/sdformat.dsl b/jenkins-scripts/dsl/sdformat.dsl index 3da399eea..a1439261b 100644 --- a/jenkins-scripts/dsl/sdformat.dsl +++ b/jenkins-scripts/dsl/sdformat.dsl @@ -1,7 +1,7 @@ import _configs_.* import javaposse.jobdsl.dsl.Job -def sdformat_supported_branches = [ 'sdformat6' , 'sdformat9', 'sdformat10', 'sdformat11', 'sdformat12' ] +def sdformat_supported_branches = [ 'sdformat6' , 'sdformat9', 'sdformat11', 'sdformat12' ] def sdformat_gz11_branches = [ 'sdformat9', 'sdformat10', 'sdformat11', 'sdformat12', 'main' ] // nightly and prereleases def extra_sdformat_debbuilder = [ ] From 6def3098dc16e555b5b63dfdcf3eeebca5c7d0b9 Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Wed, 29 Dec 2021 08:58:34 -0800 Subject: [PATCH 054/540] Enable sdformat13-debbuilder job (#614) Signed-off-by: Steve Peters --- jenkins-scripts/dsl/sdformat.dsl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/sdformat.dsl b/jenkins-scripts/dsl/sdformat.dsl index a1439261b..d5fe1890a 100644 --- a/jenkins-scripts/dsl/sdformat.dsl +++ b/jenkins-scripts/dsl/sdformat.dsl @@ -4,7 +4,7 @@ import javaposse.jobdsl.dsl.Job def sdformat_supported_branches = [ 'sdformat6' , 'sdformat9', 'sdformat11', 'sdformat12' ] def sdformat_gz11_branches = [ 'sdformat9', 'sdformat10', 'sdformat11', 'sdformat12', 'main' ] // nightly and prereleases -def extra_sdformat_debbuilder = [ ] +def extra_sdformat_debbuilder = [ 'sdformat13' ] // Main platform using for quick CI def ci_distro = Globals.get_ci_distro() From 8533d079866cfb314d7477b8964148978ca3f4cc Mon Sep 17 00:00:00 2001 From: Louise Poubel Date: Wed, 29 Dec 2021 11:30:24 -0800 Subject: [PATCH 055/540] Move "enable nightlies" task from bump libraries to post-release (#615) Signed-off-by: Louise Poubel --- .github/ISSUE_TEMPLATE/bump_libraries.md | 1 - .github/ISSUE_TEMPLATE/release_collection.md | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bump_libraries.md b/.github/ISSUE_TEMPLATE/bump_libraries.md index 06b633b29..ab41324fb 100644 --- a/.github/ISSUE_TEMPLATE/bump_libraries.md +++ b/.github/ISSUE_TEMPLATE/bump_libraries.md @@ -30,7 +30,6 @@ Libraries being bumped: - [ ] Release repositories of all downstream libraries ([example](https://github.com/ignition-release/ign-fuel-tools7-release/pull/1)) - [ ] Use `main` branch on `gazebodistro` ([example](https://github.com/ignition-tooling/gazebodistro/pull/42)) - [ ] Build nightlies from the `main` branch ([example](https://github.com/ignition-tooling/release-tools/pull/437)) - - [ ] Update gzdev to use nightlies on CI ([example](https://github.com/ignition-tooling/gzdev/pull/31)) - [ ] homebrew-simulation: create formula and update dependencies ([example](https://github.com/osrf/homebrew-simulation/pull/14230)) - [ ] docs (the collection’s page) ([example](https://github.com/ignitionrobotics/docs/pull/175)) * diff --git a/.github/ISSUE_TEMPLATE/release_collection.md b/.github/ISSUE_TEMPLATE/release_collection.md index 1145fced9..368df7703 100644 --- a/.github/ISSUE_TEMPLATE/release_collection.md +++ b/.github/ISSUE_TEMPLATE/release_collection.md @@ -56,7 +56,7 @@ When opening PRs, add a link back to this issue for easier tracking. - [ ] Add files to `gazebodistro` ([example](https://github.com/ignition-tooling/gazebodistro/pull/12)) - [ ] Add aliases to `homebrew-simulation` ([example](https://github.com/osrf/homebrew-simulation/commit/1f8602af6f52e06e0542eebfbdbe97f5f6cf950c)) - [ ] Create new `-release` repositories (use [this script](https://github.com/ignition-tooling/release-tools/blob/master/release-repo-scripts/new_ignition_release_repos.bash)) - + - [ ] Enable nightlies for all `main` branches on `gzdev` ([example](https://github.com/ignition-tooling/gzdev/pull/50)) If the collection will be officially paired with a ROS 2 distro: From 1d8ce08bd9d813d0b6f664acd7021c765dd73ecc Mon Sep 17 00:00:00 2001 From: Louise Poubel Date: Thu, 30 Dec 2021 15:30:23 -0800 Subject: [PATCH 056/540] Update release template after using it for Fortress (#617) Signed-off-by: Louise Poubel --- .github/ISSUE_TEMPLATE/release_collection.md | 49 ++++++++++++++++++-- 1 file changed, 44 insertions(+), 5 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/release_collection.md b/.github/ISSUE_TEMPLATE/release_collection.md index 368df7703..420e95b6d 100644 --- a/.github/ISSUE_TEMPLATE/release_collection.md +++ b/.github/ISSUE_TEMPLATE/release_collection.md @@ -22,41 +22,60 @@ When opening PRs, add a link back to this issue for easier tracking. - [ ] Choose name for next release. - [ ] Merge each library forward from previous versions + * - [ ] Label pull requests being considered to enter release with [beta](https://github.com/search?q=org%3Aignitionrobotics+label%3Abeta&state=open) Pull requests opened against the new collection after then: - Bug fixes: will be considered for the initial release and labeled `beta` - New features: will not be considered for the initial release. - [ ] Pre-release libraries as all `beta` labels are merged. + * - [ ] Make collection pre-release after all libraries are pre-released. -- [ ] Update public documentation to install from pre-releases instead of nightlies. + * +- [ ] Update public documentation to install from pre-releases instead of nightlies ([example](https://github.com/ignitionrobotics/docs/pull/196/files#diff-ebeee4adce7cb444f663b59020fb9f43f6f9adf36f63657b8afdeeea7a8530d1)). + * +- [ ] Change nightlies to pre-release for release branches on `gzdev` ([example](https://github.com/ignition-tooling/gzdev/pull/35)) + * ## Code freeze - [ ] Start T-Shirt campaign + * - [ ] Hold tutorial party ([example](https://community.gazebosim.org/t/ignition-edifice-tutorial-party-support-open-source-and-get-t-shirts/866)) + * - [ ] PRs fixing documentation and critical bugs can be merged and pre-released - As libraries have all their documentation reviewed: - [ ] Create stable branches off of main - [ ] Make stable releases + * ## Release day - [ ] Make collection stable release after all libraries are released. + * - [ ] Update `gazebodistro` ([example](https://github.com/ignition-tooling/gazebodistro/pull/31)) + * - [ ] Make GitHub releases for all the X.0.0 versions ([example](https://github.com/ignitionrobotics/ign-transport/releases/tag/ignition-transport9_9.0.0)) - [ ] Update docs to say release is stable ([example](https://github.com/ignitionrobotics/docs/pull/171)) + * - [ ] Make announcement + * ## Post-release - [ ] Enable Jenkins CI for new stable branches ([example](https://github.com/ignition-tooling/release-tools/pull/299)) -- [ ] Remove nightlies for release branches on `gzdev` ([example](https://github.com/ignition-tooling/gzdev/pull/19)) + * +- [ ] Remove pre-release for release branches on `gzdev` ([example](https://github.com/ignition-tooling/gzdev/pull/36)) + * - [ ] Bump all `main` branches to the next major versioni (`X.0.0~pre1`) - [ ] Source code ([example](https://github.com/ignitionrobotics/ign-common/pull/193)) + * - [ ] Add files to `gazebodistro` ([example](https://github.com/ignition-tooling/gazebodistro/pull/12)) + * - [ ] Add aliases to `homebrew-simulation` ([example](https://github.com/osrf/homebrew-simulation/commit/1f8602af6f52e06e0542eebfbdbe97f5f6cf950c)) + * - [ ] Create new `-release` repositories (use [this script](https://github.com/ignition-tooling/release-tools/blob/master/release-repo-scripts/new_ignition_release_repos.bash)) - [ ] Enable nightlies for all `main` branches on `gzdev` ([example](https://github.com/ignition-tooling/gzdev/pull/50)) +- [ ] Update all repositories to default to the new stable branches. If the collection will be officially paired with a ROS 2 distro: @@ -67,6 +86,27 @@ If the collection will be officially paired with a ROS 2 distro: # Status tracker +Repo | Version | Release Branch | Feature frozen | Synced with previous versions? | Prerelease? | Code Frozen | Stable release? | Open PRs | Pending dep release +-- | -- | -- | -- | -- | -- | -- | -- | -- | -- +[ign-tools](https://github.com/ignitionrobotics/ign-tools) | | ign-toolsN | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | [PRs](https://github.com/ignitionrobotics/ign-tools/pulls/?q=is%3Apr+is%3Aopen+label%3A%22beta%22) | +[ign-cmake](https://github.com/ignitionrobotics/ign-cmake) | | ign-cmakeN | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | [PRs](https://github.com/ignitionrobotics/ign-cmake/pulls/?q=is%3Apr+is%3Aopen+label%3A%22beta%22) | +[ign-math](https://github.com/ignitionrobotics/ign-math) | | ign-mathN | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | [PRs](https://github.com/ignitionrobotics/ign-math/pulls/?q=is%3Apr+is%3Aopen+label%3A%22beta%22) | +[ign-plugin](https://github.com/ignitionrobotics/ign-plugin) | | ign-pluginN | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | [PRs](https://github.com/ignitionrobotics/ign-plugin/pulls/?q=is%3Apr+is%3Aopen+label%3A%22beta%22) | +[ign-utils](https://github.com/ignitionrobotics/ign-utils) | | ign-utilsN | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | [PRs](https://github.com/ignitionrobotics/ign-utils/pulls/?q=is%3Apr+is%3Aopen+label%3A%22beta%22) | +[ign-common](https://github.com/ignitionrobotics/ign-common) | | ign-commonN | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | [PRs](https://github.com/ignitionrobotics/ign-common/pulls/?q=is%3Apr+is%3Aopen+label%3A%22beta%22) | +[ign-msgs](https://github.com/ignitionrobotics/ign-msgs) | | ign-msgsN | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | [PRs](https://github.com/ignitionrobotics/ign-msgs/pulls/?q=is%3Apr+is%3Aopen+label%3A%22beta%22) | +[ign-rendering](https://github.com/ignitionrobotics/ign-rendering) | | ign-renderingN | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | [PRs](https://github.com/ignitionrobotics/ign-rendering/pulls/?q=is%3Apr+is%3Aopen+label%3A%22beta%22) | +[sdformat](https://github.com/ignitionrobotics/sdformat) | | sdfN | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | [PRs](https://github.com/ignitionrobotics/sdformat/pulls/?q=is%3Apr+is%3Aopen+label%3A%22beta%22) | +[ign-fuel-tools](https://github.com/ignitionrobotics/ign-fuel-tools) | | ign-fuel-toolsN | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | [PRs](https://github.com/ignitionrobotics/ign-fuel-tools/pulls/?q=is%3Apr+is%3Aopen+label%3A%22beta%22) | +[ign-transport](https://github.com/ignitionrobotics/ign-transport) | | ign-transportN | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | [PRs](https://github.com/ignitionrobotics/ign-transport/pulls/?q=is%3Apr+is%3Aopen+label%3A%22beta%22) | +[ign-gui](https://github.com/ignitionrobotics/ign-gui) | | ign-guiN | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | [PRs](https://github.com/ignitionrobotics/ign-gui/pulls/?q=is%3Apr+is%3Aopen+label%3A%22beta%22) | +[ign-sensors](https://github.com/ignitionrobotics/ign-sensors) | | ign-sensorsN | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | [PRs](https://github.com/ignitionrobotics/ign-sensors/pulls/?q=is%3Apr+is%3Aopen+label%3A%22beta%22) | +[ign-physics](https://github.com/ignitionrobotics/ign-physics) | | ign-physicsN | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | [PRs](https://github.com/ignitionrobotics/ign-physics/pulls/?q=is%3Apr+is%3Aopen+label%3A%22beta%22) | +[ign-gazebo](https://github.com/ignitionrobotics/ign-gazebo) | | ign-gazeboN | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | [PRs](https://github.com/ignitionrobotics/ign-gazebo/pulls/?q=is%3Apr+is%3Aopen+label%3A%22beta%22) | +[ign-launch](https://github.com/ignitionrobotics/ign-launch) | | ign-launchN | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | [PRs](https://github.com/ignitionrobotics/ign-launch/pulls/?q=is%3Apr+is%3Aopen+label%3A%22beta%22) | +[ign-](https://github.com/ignitionrobotics/ign-) | | main | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | [PRs](https://github.com/ignitionrobotics/ign-/pulls/?q=is%3Apr+is%3Aopen+label%3A%22beta%22) | +[ros_ign](https://github.com/ignitionrobotics/ros_ign) | | | ✔️ | ✔️ | ✔️ | | | [PRs](https://github.com/ignitionrobotics/ros_ign/pulls/?q=is%3Apr+is%3Aopen+label%3A%22beta%22) | + Track stable release of metapackage `ignition-`. * :hourglass: TODO @@ -79,10 +119,9 @@ Track stable release of metapackage `ignition-`. * **Ubuntu:** - [ ] :hourglass: Focal/amd64 - - [ ] :hourglass: Bionic/amd64 - - [ ] :hourglass: Bionic/i386 + - [ ] :hourglass: Jammy/amd64 - [ ] :hourglass: Focal/arm64 - - [ ] :hourglass: Bionic/arm64 + - [ ] :hourglass: Jammy/arm64 * **Debian:** - [ ] :hourglass: Buster/i386 From 90dcab11625031c2c30be9ffe79c023d853cec49 Mon Sep 17 00:00:00 2001 From: Jorge Perez Date: Tue, 4 Jan 2022 18:41:24 -0300 Subject: [PATCH 057/540] Pin activesupport to version 6.1.4.4 (#603) Signed-off-by: Jorge Perez --- jenkins-scripts/docker/ign_msgs-compilation.bash | 2 -- 1 file changed, 2 deletions(-) diff --git a/jenkins-scripts/docker/ign_msgs-compilation.bash b/jenkins-scripts/docker/ign_msgs-compilation.bash index 14199c0f8..7a47506aa 100644 --- a/jenkins-scripts/docker/ign_msgs-compilation.bash +++ b/jenkins-scripts/docker/ign_msgs-compilation.bash @@ -16,8 +16,6 @@ fi export BUILDING_SOFTWARE_DIRECTORY="ign-msgs" export BUILDING_PKG_DEPENDENCIES_VAR_NAME="IGN_MSGS_DEPENDENCIES" -# skip ruby on Xenial, because protobuf requires Ruby version >= 2.4.4 -export DOCKER_POSTINSTALL_HOOK="gem install protobuf || if [ `lsb_release -sr` = "16.04" ]; then true; else false; fi" # Identify IGN_MSGS_MAJOR_VERSION to help with dependency resolution IGN_MSGS_MAJOR_VERSION=$(\ From 3752a160085769793311de805cbf9225571dc1ed Mon Sep 17 00:00:00 2001 From: Louise Poubel Date: Tue, 4 Jan 2022 16:26:05 -0800 Subject: [PATCH 058/540] Default to Focal for CI (#565) * Default to Focal for CI Signed-off-by: Louise Poubel * Remove Globals.get_gz11_ubuntu_distro Signed-off-by: Steve Peters * Update jenkins-scripts/dsl/_configs_/Globals.groovy Co-authored-by: Jose Luis Rivero * Add testing jobs for Focal and Jammy Ubuntu packages * Remove nightly support * Remove ci_distro_default hack * Move master jobs to gazebo11 * Use libdart Ubuntu's packages on Focal and newer * Use right fuel-tools-dev file * Support dart packages in Gazebo11 * Missing dart packages * Fix package name * Add ikfast to dart * Fix syntax * Back to use bionic for Gazebo Co-authored-by: Steve Peters Co-authored-by: Jose Luis Rivero --- jenkins-scripts/dsl/_configs_/Globals.groovy | 16 +--- jenkins-scripts/dsl/gazebo.dsl | 77 +++++--------------- jenkins-scripts/dsl/sdformat.dsl | 16 +--- jenkins-scripts/lib/dependencies_archive.sh | 17 ++++- 4 files changed, 38 insertions(+), 88 deletions(-) diff --git a/jenkins-scripts/dsl/_configs_/Globals.groovy b/jenkins-scripts/dsl/_configs_/Globals.groovy index f6b121d61..8fd517ae9 100644 --- a/jenkins-scripts/dsl/_configs_/Globals.groovy +++ b/jenkins-scripts/dsl/_configs_/Globals.groovy @@ -57,12 +57,12 @@ class Globals // Main CI platform static ArrayList get_ci_distro() { - return [ 'bionic' ] + return [ 'focal' ] } static ArrayList get_abi_distro() { - return [ 'bionic' ] + return [ 'focal' ] } static ArrayList get_ci_gpu() @@ -72,7 +72,7 @@ class Globals static ArrayList get_other_supported_distros() { - return [ 'bionic' ] + return [ ] } static ArrayList get_supported_arches() @@ -108,14 +108,4 @@ class Globals static String get_ros2_development_distro() { return 'rolling' } - - static String get_gz11_ubuntu_distro() - { - return 'bionic' - } - - static String get_gz11_mac_distro() - { - return 'mojave' - } } diff --git a/jenkins-scripts/dsl/gazebo.dsl b/jenkins-scripts/dsl/gazebo.dsl index 201b1c3a9..bbb02cb5a 100644 --- a/jenkins-scripts/dsl/gazebo.dsl +++ b/jenkins-scripts/dsl/gazebo.dsl @@ -3,15 +3,13 @@ import javaposse.jobdsl.dsl.Job def gazebo_supported_branches = [ 'gazebo9', 'gazebo11' ] def gazebo_supported_build_types = [ 'Release', 'Debug', 'Coverage' ] -// nightly_gazebo_branch is not the branch used to get the code from but -// the one used to generate the corresponding debbuild job. -def nightly_gazebo_branch = [ 'gazebo11' ] // testing official packages without osrf repo -def ubuntu_official_packages_distros = [ 'bionic' : 'gazebo9' ] - +def ubuntu_official_packages_distros = [ 'bionic' : 'gazebo9', + 'focal' : 'gazebo9', + 'jammy' : 'gazebo11'] // Main platform using for quick CI -def ci_distro_default = [ 'bionic' ] -def ci_distro = Globals.get_ci_distro() +// Using bionic until https://github.com/osrf/gazebo/issues/3151 is fixed +def ci_distro = [ 'bionic' ] def ci_gpu = Globals.get_ci_gpu() def abi_distro = Globals.get_abi_distro() // Other supported platform to be checked but no for quick @@ -25,7 +23,6 @@ def all_supported_gpus = Globals.get_all_supported_gpus() def DISABLE_TESTS = false def DISABLE_GITHUB_INTEGRATION = false -String ci_distro_default_str = ci_distro_default[0] String ci_distro_str = ci_distro[0] String ci_gpu_str = ci_gpu[0] String ci_build_any_job_name_linux = "gazebo-ci-pr_any-ubuntu_auto-amd64-gpu-${ci_gpu_str}" @@ -139,43 +136,13 @@ ci_distro.each { distro -> } } - // -------------------------------------------------------------- - // 2. Create the master ci jobs - def gazebo_ci_job = job("gazebo-ci-master-${ci_distro_default_str}-${arch}-gpu-${gpu}") - if (is_watched_by_buildcop('master', ci_distro_default_str, gpu)) - { - Globals.extra_emails = Globals.build_cop_email - } - OSRFLinuxCompilation.create(gazebo_ci_job) - OSRFGitHub.create(gazebo_ci_job, "osrf/gazebo") - - gazebo_ci_job.with - { - label "gpu-reliable && large-memory" - - triggers { - scm('*/5 * * * *') - } - - steps { - shell("""\ - #!/bin/bash -xe - - export DISTRO=${ci_distro_default_str} - export ARCH=${arch} - export GPU_SUPPORT_NEEDED=true - /bin/bash -xe ./scripts/jenkins-scripts/docker/gazebo-compilation.bash - """.stripIndent()) - } - } - // reset build cop email in global list of mails Globals.extra_emails = '' } // end of gpu } // end of arch } // end of distro -// OTHER CI SUPPORTED JOBS (master branch) @ SCM/DAILY +// OTHER CI SUPPORTED JOBS (gazebo11 branch) @ SCM/DAILY other_supported_distros.each { distro -> supported_arches.each { arch -> @@ -185,8 +152,8 @@ other_supported_distros.each { distro -> gpus = [ 'none' ] gpus.each { gpu -> - // ci_master job for the rest of arches / scm@daily - def gazebo_ci_job = job("gazebo-ci-master-${distro}-${arch}-gpu-${gpu}") + // gazebo11 job for the rest of arches / scm@daily + def gazebo_ci_job = job("gazebo-ci-gazebo11-${distro}-${arch}-gpu-${gpu}") OSRFLinuxCompilation.create(gazebo_ci_job) OSRFGitHub.create(gazebo_ci_job, "osrf/gazebo") @@ -259,10 +226,6 @@ ci_distro.each { distro -> // BRANCHES CI JOB @ SCM/DAILY gazebo_supported_branches.each { branch -> distros = ci_distro - if (branch == 'gazebo11') - { - distros = ci_distro_default - } distros.each { distro -> supported_arches.each { arch -> ci_gpu.each { gpu -> @@ -303,11 +266,11 @@ gazebo_supported_branches.each { branch -> } // end of branch // EXPERIMENTAL ARCHES @ SCM/WEEKLY -ci_distro_default.each { distro -> +ci_distro.each { distro -> experimental_arches.each { arch -> - def gazebo_ci_job = job("gazebo-ci-master-${distro}-${arch}-gpu-none") + def gazebo_ci_job = job("gazebo-ci-gazebo11-${distro}-${arch}-gpu-none") OSRFLinuxCompilation.create(gazebo_ci_job) - OSRFGitHub.create(gazebo_ci_job, "osrf/gazebo") + OSRFGitHub.create(gazebo_ci_job, "osrf/gazebo", "gazebo11") gazebo_ci_job.with { @@ -332,12 +295,12 @@ ci_distro_default.each { distro -> } // COVERAGE TYPE @ SCM/DAILY -ci_distro_default.each { distro -> +ci_distro.each { distro -> supported_arches.each { arch -> ci_gpu.each { gpu -> def gazebo_ci_job = job("gazebo-ci-coverage-${distro}-${arch}-gpu-${gpu}") OSRFLinuxCompilation.create(gazebo_ci_job) - OSRFGitHub.create(gazebo_ci_job, "osrf/gazebo", "master") + OSRFGitHub.create(gazebo_ci_job, "osrf/gazebo", "gazebo11") gazebo_ci_job.with { triggers { @@ -367,12 +330,12 @@ ci_distro_default.each { distro -> } // BUILD TYPES CI JOBS @ SCM/DAILY -ci_distro_default.each { distro -> +ci_distro.each { distro -> supported_arches.each { arch -> gazebo_supported_build_types.each { build_type -> - def gazebo_ci_job = job("gazebo-ci_BT${build_type}-master-${distro}-${arch}-gpu-none") + def gazebo_ci_job = job("gazebo-ci_BT${build_type}-gazebo11-${distro}-${arch}-gpu-none") OSRFLinuxCompilation.create(gazebo_ci_job) - OSRFGitHub.create(gazebo_ci_job, "osrf/gazebo") + OSRFGitHub.create(gazebo_ci_job, "osrf/gazebo", "gazebo11") gazebo_ci_job.with { @@ -429,10 +392,6 @@ all_supported_distros.each { distro -> // INSTALL LINUX -DEV PACKAGES ALL PLATFORMS @ CRON/DAILY gazebo_supported_branches.each { branch -> distros = ci_distro - if (branch == 'gazebo11') - { - distros = ci_distro_default - } distros.each { distro -> supported_arches.each { arch -> // -------------------------------------------------------------- @@ -459,7 +418,7 @@ ubuntu_official_packages_distros.each { distro, branch -> // -------------------------------------------------------------- // DEBBUILD: linux package builder -all_debbuild_branches = gazebo_supported_branches + nightly_gazebo_branch +all_debbuild_branches = gazebo_supported_branches all_debbuild_branches.each { branch -> def build_pkg_job = job("${branch}-debbuilder") OSRFLinuxBuildPkg.create(build_pkg_job) @@ -516,7 +475,7 @@ install_brew_job.with // 2. default in all branches @SCM/daily // No gazebo2 for brew -all_branches = gazebo_supported_branches + 'master' +all_branches = gazebo_supported_branches all_branches.each { branch -> if (is_watched_by_buildcop(branch)) Globals.extra_emails = Globals.build_cop_email diff --git a/jenkins-scripts/dsl/sdformat.dsl b/jenkins-scripts/dsl/sdformat.dsl index d5fe1890a..c87f04801 100644 --- a/jenkins-scripts/dsl/sdformat.dsl +++ b/jenkins-scripts/dsl/sdformat.dsl @@ -61,7 +61,7 @@ abi_distro.each { distro -> // MAIN CI job // CI JOBS @ SCM/5 min -[ Globals.get_gz11_ubuntu_distro() ].each { distro -> +ci_distro.each { distro -> supported_arches.each { arch -> // -------------------------------------------------------------- // 1. Create the main ci jobs @@ -139,10 +139,6 @@ other_supported_distros.each { distro -> // BRANCHES CI JOB @ SCM/DAILY sdformat_supported_branches.each { branch -> ci_distro.each { distro -> - // special check to modify ci_distro if the branch is part of gz11 - if (branch in sdformat_gz11_branches) - distro = Globals.get_gz11_ubuntu_distro() - supported_arches.each { arch -> // ci_main job for the rest of arches / scm@daily def sdformat_ci_job = job("sdformat-ci-${branch}-${distro}-${arch}") @@ -171,7 +167,7 @@ sdformat_supported_branches.each { branch -> // // EXPERIMENTAL ARCHES @ SCM/WEEKLY -[ Globals.get_gz11_ubuntu_distro() ].each { distro -> +ci_distro.each { distro -> experimental_arches.each { arch -> def sdformat_ci_job = job("sdformat-ci-main-${distro}-${arch}") OSRFLinuxCompilation.create(sdformat_ci_job) @@ -198,13 +194,7 @@ sdformat_supported_branches.each { branch -> // INSTALL LINUX -DEV PACKAGES ALL PLATFORMS @ CRON/DAILY sdformat_supported_branches.each { branch -> - // special check to modify ci_distro if the branch is part of gz11 - if (branch in sdformat_gz11_branches) - ref_distro = [ Globals.get_gz11_ubuntu_distro() ] - else - ref_distro = ci_distro - - ref_distro.each { distro -> + ci_distro.each { distro -> supported_arches.each { arch -> // -------------------------------------------------------------- def install_default_job = job("sdformat-install-${branch}_pkg-${distro}-${arch}") diff --git a/jenkins-scripts/lib/dependencies_archive.sh b/jenkins-scripts/lib/dependencies_archive.sh index 6ff1d4621..920afe371 100644 --- a/jenkins-scripts/lib/dependencies_archive.sh +++ b/jenkins-scripts/lib/dependencies_archive.sh @@ -128,7 +128,18 @@ bullet_pkg="libbullet-dev" # choose dart version if $DART_FROM_PKGS; then - dart_pkg="libdart6-utils-urdf-dev" + dart_pkgs="libdart-utils-urdf-dev \ + libdart-external-odelcpsolver-dev \ + libdart-external-ikfast-dev" + if [[ ${GAZEBO_MAJOR_VERSION} -ge 11 ]]; then + dart_pkgs="${dart_pkgs} libdart-collision-bullet-dev \ + libdart-collision-ode-dev" + fi + # If on Bionic/Xenial use custom dart packages, these have a different name + # dart -> dart6 + if [[ ${DISTRO} == 'bionic' ]] || [[ ${DISTRO} == 'xenial' ]]; then + dart_pkgs="${dart_pkgs//dart/dart6}" + fi fi # By default, the stable version of gazebo @@ -163,7 +174,7 @@ if ! ${GAZEBO_EXPERIMENTAL_BUILD}; then libboost-iostreams-dev \\ ${bullet_pkg} \\ libsimbody-dev \\ - ${dart_pkg}" + ${dart_pkgs}" if [[ ${GAZEBO_MAJOR_VERSION} -ge 11 ]]; then GAZEBO_BASE_DEPENDENCIES_NO_SDFORMAT="${GAZEBO_BASE_DEPENDENCIES_NO_SDFORMAT} \\ @@ -175,7 +186,7 @@ if ! ${GAZEBO_EXPERIMENTAL_BUILD}; then elif [[ ${GAZEBO_MAJOR_VERSION} -ge 9 ]]; then GAZEBO_BASE_DEPENDENCIES_NO_SDFORMAT="${GAZEBO_BASE_DEPENDENCIES_NO_SDFORMAT} \\ libignition-common-dev \\ - libignition-fuel-tools-dev \\ + libignition-fuel-tools1-dev \\ libignition-transport4-dev \\ libignition-math4-dev \\ libignition-msgs-dev" From ea4f011e091e0f1f6f6fd18395f694a422ce4802 Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Tue, 4 Jan 2022 20:37:03 -0800 Subject: [PATCH 059/540] Fix checking for ci_matching_branch on windows (#618) Explicitly set error codes in detect_ci_matching_branch.py and fix the syntax for checking ERRORLEVEL in windows_library.bat Signed-off-by: Steve Peters --- jenkins-scripts/lib/windows_library.bat | 4 +++- jenkins-scripts/tools/detect_ci_matching_branch.py | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/jenkins-scripts/lib/windows_library.bat b/jenkins-scripts/lib/windows_library.bat index 97fdaa928..f6ccbf395 100644 --- a/jenkins-scripts/lib/windows_library.bat +++ b/jenkins-scripts/lib/windows_library.bat @@ -149,7 +149,9 @@ git clone https://github.com/ignition-tooling/gazebodistro %gzdistro_dir% -b %GA :: Check if ci_matching_branch name is used if "%ghprbSourceBranch%" == "" (echo ghprbSourceBranch is unset) else ( python "%SCRIPT_DIR%\tools\detect_ci_matching_branch.py" "%ghprbSourceBranch%" - if "%ERRORLEVEL%" == "0" ( + :: "if ERRORLEVEL N" tests for ">= N" + :: To test for error code == 0, use !(error >= 1) + if NOT ERRORLEVEL 1 ( echo trying to checkout branch %ghprbSourceBranch% from gazebodistro git -C %gzdistro_dir% fetch origin %ghprbSourceBranch% || rem git -C %gzdistro_dir% checkout %ghprbSourceBranch% || rem diff --git a/jenkins-scripts/tools/detect_ci_matching_branch.py b/jenkins-scripts/tools/detect_ci_matching_branch.py index 54ad8506b..02c19788c 100644 --- a/jenkins-scripts/tools/detect_ci_matching_branch.py +++ b/jenkins-scripts/tools/detect_ci_matching_branch.py @@ -5,7 +5,7 @@ if len(sys.argv) != 2: print('need to specify branch name', file=sys.stderr) - exit() + sys.exit(1) branchName = sys.argv[1] pattern = 'ci_matching_branch/' @@ -13,4 +13,5 @@ if match: print(branchName, "matches", pattern) else: - sys.exit("{} does not match {}".format(branchName, pattern)) + print("{} does not match {}".format(branchName, pattern)) + sys.exit(1) From a9f83a255ef747782b8041321ce0a412ee976fd2 Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Wed, 5 Jan 2022 12:43:08 -0800 Subject: [PATCH 060/540] ignition_collection.dsl: fix distro in views (#620) * ignition_collection.dsl: fix distro in views Update bionic -> focal for most jobs in ignition collection views. Signed-off-by: Steve Peters * Fix math6 windows job name in fortress view Signed-off-by: Steve Peters --- jenkins-scripts/dsl/ignition_collection.dsl | 260 ++++++++++---------- 1 file changed, 130 insertions(+), 130 deletions(-) diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index 4e5a84b57..210b58aa9 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -60,67 +60,67 @@ ignition_collection_jobs = 'ignition_citadel-ci-main-homebrew-amd64', 'ignition_citadel-install-pkg-bionic-amd64', 'ignition_citadel-install_bottle-homebrew-amd64', - 'ignition_cmake-ci-ign-cmake2-bionic-amd64', + 'ignition_cmake-ci-ign-cmake2-focal-amd64', 'ignition_cmake-ci-ign-cmake2-homebrew-amd64', 'ignition_cmake-ci-ign-cmake2-windows7-amd64', - 'ignition_cmake2-install-pkg-bionic-amd64', + 'ignition_cmake2-install-pkg-focal-amd64', 'ignition_cmake2-install_bottle-homebrew-amd64', - 'ignition_common-ci-ign-common3-bionic-amd64', + 'ignition_common-ci-ign-common3-focal-amd64', 'ignition_common-ci-ign-common3-homebrew-amd64', - 'ignition_common3-install-pkg-bionic-amd64', + 'ignition_common3-install-pkg-focal-amd64', 'ignition_common3-install_bottle-homebrew-amd64', - 'ignition_fuel-tools-ci-ign-fuel-tools4-bionic-amd64', + 'ignition_fuel-tools-ci-ign-fuel-tools4-focal-amd64', 'ignition_fuel-tools-ci-ign-fuel-tools4-homebrew-amd64', - 'ignition_fuel-tools4-install-pkg-bionic-amd64', + 'ignition_fuel-tools4-install-pkg-focal-amd64', 'ignition_fuel-tools4-install_bottle-homebrew-amd64', - 'ignition_gazebo-ci-ign-gazebo3-bionic-amd64', + 'ignition_gazebo-ci-ign-gazebo3-focal-amd64', 'ignition_gazebo-ci-ign-gazebo3-homebrew-amd64', - 'ignition_gazebo3-install-pkg-bionic-amd64', + 'ignition_gazebo3-install-pkg-focal-amd64', 'ignition_gazebo3-install_bottle-homebrew-amd64', - 'ignition_gui-ci-ign-gui3-bionic-amd64', + 'ignition_gui-ci-ign-gui3-focal-amd64', 'ignition_gui-ci-ign-gui3-homebrew-amd64', - 'ignition_gui3-install-pkg-bionic-amd64', + 'ignition_gui3-install-pkg-focal-amd64', 'ignition_gui3-install_bottle-homebrew-amd64', - 'ignition_launch-ci-ign-launch2-bionic-amd64', + 'ignition_launch-ci-ign-launch2-focal-amd64', 'ignition_launch-ci-ign-launch2-homebrew-amd64', - 'ignition_launch2-install-pkg-bionic-amd64', + 'ignition_launch2-install-pkg-focal-amd64', 'ignition_launch2-install_bottle-homebrew-amd64', - 'ignition_math-ci-ign-math6-bionic-amd64', + 'ignition_math-ci-ign-math6-focal-amd64', 'ignition_math-ci-ign-math6-homebrew-amd64', - 'ignition_math6-install-pkg-bionic-amd64', + 'ignition_math6-install-pkg-focal-amd64', 'ignition_math6-install_bottle-homebrew-amd64', - 'ignition_msgs-ci-ign-msgs5-bionic-amd64', + 'ignition_msgs-ci-ign-msgs5-focal-amd64', 'ignition_msgs-ci-ign-msgs5-homebrew-amd64', - 'ignition_msgs5-install-pkg-bionic-amd64', + 'ignition_msgs5-install-pkg-focal-amd64', 'ignition_msgs5-install_bottle-homebrew-amd64', - 'ignition_physics-ci-ign-physics2-bionic-amd64', + 'ignition_physics-ci-ign-physics2-focal-amd64', 'ignition_physics-ci-ign-physics2-homebrew-amd64', - 'ignition_physics2-install-pkg-bionic-amd64', + 'ignition_physics2-install-pkg-focal-amd64', 'ignition_physics2-install_bottle-homebrew-amd64', - 'ignition_plugin-ci-ign-plugin1-bionic-amd64', + 'ignition_plugin-ci-ign-plugin1-focal-amd64', 'ignition_plugin-ci-ign-plugin1-homebrew-amd64', - 'ignition_plugin-install-pkg-bionic-amd64', + 'ignition_plugin-install-pkg-focal-amd64', 'ignition_plugin1-install_bottle-homebrew-amd64', - 'ignition_rendering-ci-ign-rendering3-bionic-amd64', + 'ignition_rendering-ci-ign-rendering3-focal-amd64', 'ignition_rendering-ci-ign-rendering3-homebrew-amd64', - 'ignition_rendering3-install-pkg-bionic-amd64', + 'ignition_rendering3-install-pkg-focal-amd64', 'ignition_rendering3-install_bottle-homebrew-amd64', - 'ignition_sensors-ci-ign-sensors3-bionic-amd64', + 'ignition_sensors-ci-ign-sensors3-focal-amd64', 'ignition_sensors-ci-ign-sensors3-homebrew-amd64', - 'ignition_sensors3-install-pkg-bionic-amd64', + 'ignition_sensors3-install-pkg-focal-amd64', 'ignition_sensors3-install_bottle-homebrew-amd64', - 'ignition_tools-ci-ign-tools1-bionic-amd64', + 'ignition_tools-ci-ign-tools1-focal-amd64', 'ignition_tools-ci-ign-tools1-homebrew-amd64', - 'ignition_tools-install-pkg-bionic-amd64', + 'ignition_tools-install-pkg-focal-amd64', 'ignition_tools1-install_bottle-homebrew-amd64', - 'ignition_transport-ci-ign-transport8-bionic-amd64', + 'ignition_transport-ci-ign-transport8-focal-amd64', 'ignition_transport-ci-ign-transport8-homebrew-amd64', - 'ignition_transport8-install-pkg-bionic-amd64', + 'ignition_transport8-install-pkg-focal-amd64', 'ignition_transport8-install_bottle-homebrew-amd64', - 'sdformat-ci-sdformat9-bionic-amd64', + 'sdformat-ci-sdformat9-focal-amd64', 'sdformat-ci-sdformat9-homebrew-amd64', 'sdformat-ci-sdformat9-windows7-amd64', - 'sdformat-install-sdformat9_pkg-bionic-amd64' + 'sdformat-install-sdformat9_pkg-focal-amd64' ], 'edifice' : [ 'ign_common-ign-4-win', @@ -136,74 +136,74 @@ ignition_collection_jobs = 'ign_tools-ign-1-win', 'ign_transport-ign-10-win', 'ign_utils-ign-1-win', - 'ignition_cmake-ci-ign-cmake2-bionic-amd64', + 'ignition_cmake-ci-ign-cmake2-focal-amd64', 'ignition_cmake-ci-ign-cmake2-homebrew-amd64', 'ignition_cmake-ci-ign-cmake2-windows7-amd64', - 'ignition_cmake2-install-pkg-bionic-amd64', + 'ignition_cmake2-install-pkg-focal-amd64', 'ignition_cmake2-install_bottle-homebrew-amd64', - 'ignition_common-ci-ign-common4-bionic-amd64', + 'ignition_common-ci-ign-common4-focal-amd64', 'ignition_common-ci-ign-common4-homebrew-amd64', - 'ignition_common4-install-pkg-bionic-amd64', + 'ignition_common4-install-pkg-focal-amd64', 'ignition_common4-install_bottle-homebrew-amd64', 'ignition_edifice-ci-main-homebrew-amd64', - 'ignition_edifice-install-pkg-bionic-amd64', + 'ignition_edifice-install-pkg-focal-amd64', 'ignition_edifice-install_bottle-homebrew-amd64', - 'ignition_fuel-tools-ci-ign-fuel-tools6-bionic-amd64', + 'ignition_fuel-tools-ci-ign-fuel-tools6-focal-amd64', 'ignition_fuel-tools-ci-ign-fuel-tools6-homebrew-amd64', - 'ignition_fuel-tools6-install-pkg-bionic-amd64', + 'ignition_fuel-tools6-install-pkg-focal-amd64', 'ignition_fuel-tools6-install_bottle-homebrew-amd64', - 'ignition_gazebo-ci-ign-gazebo5-bionic-amd64', + 'ignition_gazebo-ci-ign-gazebo5-focal-amd64', 'ignition_gazebo-ci-ign-gazebo5-homebrew-amd64', - 'ignition_gazebo5-install-pkg-bionic-amd64', + 'ignition_gazebo5-install-pkg-focal-amd64', 'ignition_gazebo5-install_bottle-homebrew-amd64', - 'ignition_gui-ci-ign-gui5-bionic-amd64', + 'ignition_gui-ci-ign-gui5-focal-amd64', 'ignition_gui-ci-ign-gui5-homebrew-amd64', - 'ignition_gui5-install-pkg-bionic-amd64', + 'ignition_gui5-install-pkg-focal-amd64', 'ignition_gui5-install_bottle-homebrew-amd64', - 'ignition_launch-ci-ign-launch4-bionic-amd64', + 'ignition_launch-ci-ign-launch4-focal-amd64', 'ignition_launch-ci-ign-launch4-homebrew-amd64', - 'ignition_launch4-install-pkg-bionic-amd64', + 'ignition_launch4-install-pkg-focal-amd64', 'ignition_launch4-install_bottle-homebrew-amd64', - 'ignition_math-ci-ign-math6-bionic-amd64', + 'ignition_math-ci-ign-math6-focal-amd64', 'ignition_math-ci-ign-math6-homebrew-amd64', - 'ignition_math6-install-pkg-bionic-amd64', + 'ignition_math6-install-pkg-focal-amd64', 'ignition_math6-install_bottle-homebrew-amd64', - 'ignition_msgs-ci-ign-msgs7-bionic-amd64', + 'ignition_msgs-ci-ign-msgs7-focal-amd64', 'ignition_msgs-ci-ign-msgs7-homebrew-amd64', - 'ignition_msgs7-install-pkg-bionic-amd64', + 'ignition_msgs7-install-pkg-focal-amd64', 'ignition_msgs7-install_bottle-homebrew-amd64', - 'ignition_physics-ci-ign-physics4-bionic-amd64', + 'ignition_physics-ci-ign-physics4-focal-amd64', 'ignition_physics-ci-ign-physics4-homebrew-amd64', - 'ignition_physics4-install-pkg-bionic-amd64', + 'ignition_physics4-install-pkg-focal-amd64', 'ignition_physics4-install_bottle-homebrew-amd64', - 'ignition_plugin-ci-ign-plugin1-bionic-amd64', + 'ignition_plugin-ci-ign-plugin1-focal-amd64', 'ignition_plugin-ci-ign-plugin1-homebrew-amd64', - 'ignition_plugin-install-pkg-bionic-amd64', + 'ignition_plugin-install-pkg-focal-amd64', 'ignition_plugin1-install_bottle-homebrew-amd64', - 'ignition_rendering-ci-ign-rendering5-bionic-amd64', + 'ignition_rendering-ci-ign-rendering5-focal-amd64', 'ignition_rendering-ci-ign-rendering5-homebrew-amd64', - 'ignition_rendering5-install-pkg-bionic-amd64', + 'ignition_rendering5-install-pkg-focal-amd64', 'ignition_rendering5-install_bottle-homebrew-amd64', - 'ignition_sensors-ci-ign-sensors5-bionic-amd64', + 'ignition_sensors-ci-ign-sensors5-focal-amd64', 'ignition_sensors-ci-ign-sensors5-homebrew-amd64', - 'ignition_sensors5-install-pkg-bionic-amd64', + 'ignition_sensors5-install-pkg-focal-amd64', 'ignition_sensors5-install_bottle-homebrew-amd64', - 'ignition_tools-ci-ign-tools1-bionic-amd64', + 'ignition_tools-ci-ign-tools1-focal-amd64', 'ignition_tools-ci-ign-tools1-homebrew-amd64', - 'ignition_tools-install-pkg-bionic-amd64', + 'ignition_tools-install-pkg-focal-amd64', 'ignition_tools1-install_bottle-homebrew-amd64', - 'ignition_transport-ci-ign-transport10-bionic-amd64', + 'ignition_transport-ci-ign-transport10-focal-amd64', 'ignition_transport-ci-ign-transport10-homebrew-amd64', - 'ignition_transport10-install-pkg-bionic-amd64', + 'ignition_transport10-install-pkg-focal-amd64', 'ignition_transport10-install_bottle-homebrew-amd64', - 'ignition_utils-ci-ign-utils1-bionic-amd64', + 'ignition_utils-ci-ign-utils1-focal-amd64', 'ignition_utils-ci-ign-utils1-homebrew-amd64', - 'ignition_utils-install-pkg-bionic-amd64', + 'ignition_utils-install-pkg-focal-amd64', 'ignition_utils1-install_bottle-homebrew-amd64', - 'sdformat-ci-sdformat11-bionic-amd64', + 'sdformat-ci-sdformat11-focal-amd64', 'sdformat-ci-sdformat11-homebrew-amd64', 'sdformat-ci-sdformat11-windows7-amd64', - 'sdformat-install-sdformat11_pkg-bionic-amd64' + 'sdformat-install-sdformat11_pkg-focal-amd64' ], 'fortress' : [ 'ign_common-ign-4-win', @@ -211,6 +211,7 @@ ignition_collection_jobs = 'ign_gazebo-ign-6-win', 'ign_gui-ign-6-win', 'ign_launch-ign-5-win', + 'ign_math-ign-6-win', 'ign_msgs-ign-8-win', 'ign_physics-ign-5-win', 'ign_plugin-ign-1-win', @@ -219,75 +220,74 @@ ignition_collection_jobs = 'ign_tools-ign-1-win', 'ign_transport-ign-11-win', 'ign_utils-ign-1-win', - 'ignition_cmake-ci-ign-cmake2-bionic-amd64', + 'ignition_cmake-ci-ign-cmake2-focal-amd64', 'ignition_cmake-ci-ign-cmake2-homebrew-amd64', 'ignition_cmake-ci-ign-cmake2-windows7-amd64', - 'ignition_cmake2-install-pkg-bionic-amd64', + 'ignition_cmake2-install-pkg-focal-amd64', 'ignition_cmake2-install_bottle-homebrew-amd64', - 'ignition_common-ci-ign-common4-bionic-amd64', + 'ignition_common-ci-ign-common4-focal-amd64', 'ignition_common-ci-ign-common4-homebrew-amd64', - 'ignition_common4-install-pkg-bionic-amd64', + 'ignition_common4-install-pkg-focal-amd64', 'ignition_common4-install_bottle-homebrew-amd64', 'ignition_fortress-ci-main-homebrew-amd64', - 'ignition_fortress-install-pkg-bionic-amd64', + 'ignition_fortress-install-pkg-focal-amd64', 'ignition_fortress-install_bottle-homebrew-amd64', - 'ignition_fuel-tools-ci-ign-fuel-tools7-bionic-amd64', + 'ignition_fuel-tools-ci-ign-fuel-tools7-focal-amd64', 'ignition_fuel-tools-ci-ign-fuel-tools7-homebrew-amd64', - 'ignition_fuel-tools7-install-pkg-bionic-amd64', + 'ignition_fuel-tools7-install-pkg-focal-amd64', 'ignition_fuel-tools7-install_bottle-homebrew-amd64', - 'ignition_gazebo-ci-ign-gazebo6-bionic-amd64', + 'ignition_gazebo-ci-ign-gazebo6-focal-amd64', 'ignition_gazebo-ci-ign-gazebo6-homebrew-amd64', - 'ignition_gazebo6-install-pkg-bionic-amd64', + 'ignition_gazebo6-install-pkg-focal-amd64', 'ignition_gazebo6-install_bottle-homebrew-amd64', - 'ignition_gui-ci-ign-gui6-bionic-amd64', + 'ignition_gui-ci-ign-gui6-focal-amd64', 'ignition_gui-ci-ign-gui6-homebrew-amd64', - 'ignition_gui6-install-pkg-bionic-amd64', + 'ignition_gui6-install-pkg-focal-amd64', 'ignition_gui6-install_bottle-homebrew-amd64', - 'ignition_launch-ci-ign-launch5-bionic-amd64', + 'ignition_launch-ci-ign-launch5-focal-amd64', 'ignition_launch-ci-ign-launch5-homebrew-amd64', - 'ignition_launch5-install-pkg-bionic-amd64', + 'ignition_launch5-install-pkg-focal-amd64', 'ignition_launch5-install_bottle-homebrew-amd64', - 'ignition_math-ci-ign-math6-bionic-amd64', + 'ignition_math-ci-ign-math6-focal-amd64', 'ignition_math-ci-ign-math6-homebrew-amd64', - 'ignition_math-ci-ign-math6-windows7-amd64', - 'ignition_math6-install-pkg-bionic-amd64', + 'ignition_math6-install-pkg-focal-amd64', 'ignition_math6-install_bottle-homebrew-amd64', - 'ignition_msgs-ci-ign-msgs8-bionic-amd64', + 'ignition_msgs-ci-ign-msgs8-focal-amd64', 'ignition_msgs-ci-ign-msgs8-homebrew-amd64', - 'ignition_msgs8-install-pkg-bionic-amd64', + 'ignition_msgs8-install-pkg-focal-amd64', 'ignition_msgs8-install_bottle-homebrew-amd64', - 'ignition_physics-ci-ign-physics5-bionic-amd64', + 'ignition_physics-ci-ign-physics5-focal-amd64', 'ignition_physics-ci-ign-physics5-homebrew-amd64', - 'ignition_physics5-install-pkg-bionic-amd64', + 'ignition_physics5-install-pkg-focal-amd64', 'ignition_physics5-install_bottle-homebrew-amd64', - 'ignition_plugin-ci-ign-plugin1-bionic-amd64', + 'ignition_plugin-ci-ign-plugin1-focal-amd64', 'ignition_plugin-ci-ign-plugin1-homebrew-amd64', - 'ignition_plugin-install-pkg-bionic-amd64', + 'ignition_plugin-install-pkg-focal-amd64', 'ignition_plugin1-install_bottle-homebrew-amd64', - 'ignition_rendering-ci-ign-rendering6-bionic-amd64', + 'ignition_rendering-ci-ign-rendering6-focal-amd64', 'ignition_rendering-ci-ign-rendering6-homebrew-amd64', - 'ignition_rendering6-install-pkg-bionic-amd64', + 'ignition_rendering6-install-pkg-focal-amd64', 'ignition_rendering6-install_bottle-homebrew-amd64', - 'ignition_sensors-ci-ign-sensors6-bionic-amd64', + 'ignition_sensors-ci-ign-sensors6-focal-amd64', 'ignition_sensors-ci-ign-sensors6-homebrew-amd64', - 'ignition_sensors6-install-pkg-bionic-amd64', + 'ignition_sensors6-install-pkg-focal-amd64', 'ignition_sensors6-install_bottle-homebrew-amd64', - 'ignition_tools-ci-ign-tools1-bionic-amd64', + 'ignition_tools-ci-ign-tools1-focal-amd64', 'ignition_tools-ci-ign-tools1-homebrew-amd64', - 'ignition_tools-install-pkg-bionic-amd64', + 'ignition_tools-install-pkg-focal-amd64', 'ignition_tools1-install_bottle-homebrew-amd64', - 'ignition_transport-ci-ign-transport11-bionic-amd64', + 'ignition_transport-ci-ign-transport11-focal-amd64', 'ignition_transport-ci-ign-transport11-homebrew-amd64', - 'ignition_transport11-install-pkg-bionic-amd64', + 'ignition_transport11-install-pkg-focal-amd64', 'ignition_transport11-install_bottle-homebrew-amd64', - 'ignition_utils-ci-ign-utils1-bionic-amd64', + 'ignition_utils-ci-ign-utils1-focal-amd64', 'ignition_utils-ci-ign-utils1-homebrew-amd64', - 'ignition_utils-install-pkg-bionic-amd64', + 'ignition_utils-install-pkg-focal-amd64', 'ignition_utils1-install_bottle-homebrew-amd64', - 'sdformat-ci-sdformat12-bionic-amd64', + 'sdformat-ci-sdformat12-focal-amd64', 'sdformat-ci-sdformat12-homebrew-amd64', 'sdformat-ci-sdformat12-windows7-amd64', - 'sdformat-install-sdformat12_pkg-bionic-amd64' + 'sdformat-install-sdformat12_pkg-focal-amd64' ], 'garden' : [ 'ign_common-ign-4-win', @@ -304,74 +304,74 @@ ignition_collection_jobs = 'ign_tools-ign-1-win', 'ign_transport-ci-win', 'ign_utils-ign-1-win', - 'ignition_cmake-ci-ign-cmake2-bionic-amd64', + 'ignition_cmake-ci-ign-cmake2-focal-amd64', 'ignition_cmake-ci-ign-cmake2-homebrew-amd64', 'ignition_cmake-ci-ign-cmake2-windows7-amd64', - 'ignition_cmake2-install-pkg-bionic-amd64', + 'ignition_cmake2-install-pkg-focal-amd64', 'ignition_cmake2-install_bottle-homebrew-amd64', - 'ignition_common-ci-ign-common4-bionic-amd64', + 'ignition_common-ci-ign-common4-focal-amd64', 'ignition_common-ci-ign-common4-homebrew-amd64', - 'ignition_common4-install-pkg-bionic-amd64', + 'ignition_common4-install-pkg-focal-amd64', 'ignition_common4-install_bottle-homebrew-amd64', 'ignition_garden-ci-main-homebrew-amd64', - 'ignition_garden-install-pkg-bionic-amd64', + 'ignition_garden-install-pkg-focal-amd64', 'ignition_garden-install_bottle-homebrew-amd64', - 'ignition_fuel-tools-ci-main-bionic-amd64', + 'ignition_fuel-tools-ci-main-focal-amd64', 'ignition_fuel-tools-ci-main-homebrew-amd64', - 'ignition_fuel-tools7-install-pkg-bionic-amd64', + 'ignition_fuel-tools7-install-pkg-focal-amd64', 'ignition_fuel-tools7-install_bottle-homebrew-amd64', - 'ignition_gazebo-ci-main-bionic-amd64', + 'ignition_gazebo-ci-main-focal-amd64', 'ignition_gazebo-ci-main-homebrew-amd64', - 'ignition_gazebo6-install-pkg-bionic-amd64', + 'ignition_gazebo6-install-pkg-focal-amd64', 'ignition_gazebo6-install_bottle-homebrew-amd64', - 'ignition_gui-ci-main-bionic-amd64', + 'ignition_gui-ci-main-focal-amd64', 'ignition_gui-ci-main-homebrew-amd64', - 'ignition_gui6-install-pkg-bionic-amd64', + 'ignition_gui6-install-pkg-focal-amd64', 'ignition_gui6-install_bottle-homebrew-amd64', - 'ignition_launch-ci-main-bionic-amd64', + 'ignition_launch-ci-main-focal-amd64', 'ignition_launch-ci-main-homebrew-amd64', - 'ignition_launch5-install-pkg-bionic-amd64', + 'ignition_launch5-install-pkg-focal-amd64', 'ignition_launch5-install_bottle-homebrew-amd64', - 'ignition_math-ci-ign-math6-bionic-amd64', + 'ignition_math-ci-ign-math6-focal-amd64', 'ignition_math-ci-ign-math6-homebrew-amd64', - 'ignition_math6-install-pkg-bionic-amd64', + 'ignition_math6-install-pkg-focal-amd64', 'ignition_math6-install_bottle-homebrew-amd64', - 'ignition_msgs-ci-main-bionic-amd64', + 'ignition_msgs-ci-main-focal-amd64', 'ignition_msgs-ci-main-homebrew-amd64', - 'ignition_msgs8-install-pkg-bionic-amd64', + 'ignition_msgs8-install-pkg-focal-amd64', 'ignition_msgs8-install_bottle-homebrew-amd64', - 'ignition_physics-ci-ign-physics5-bionic-amd64', + 'ignition_physics-ci-ign-physics5-focal-amd64', 'ignition_physics-ci-ign-physics5-homebrew-amd64', - 'ignition_physics5-install-pkg-bionic-amd64', + 'ignition_physics5-install-pkg-focal-amd64', 'ignition_physics5-install_bottle-homebrew-amd64', - 'ignition_plugin-ci-ign-plugin1-bionic-amd64', + 'ignition_plugin-ci-ign-plugin1-focal-amd64', 'ignition_plugin-ci-ign-plugin1-homebrew-amd64', - 'ignition_plugin-install-pkg-bionic-amd64', + 'ignition_plugin-install-pkg-focal-amd64', 'ignition_plugin1-install_bottle-homebrew-amd64', - 'ignition_rendering-ci-main-bionic-amd64', + 'ignition_rendering-ci-main-focal-amd64', 'ignition_rendering-ci-main-homebrew-amd64', - 'ignition_rendering6-install-pkg-bionic-amd64', + 'ignition_rendering6-install-pkg-focal-amd64', 'ignition_rendering6-install_bottle-homebrew-amd64', - 'ignition_sensors-ci-main-bionic-amd64', + 'ignition_sensors-ci-main-focal-amd64', 'ignition_sensors-ci-main-homebrew-amd64', - 'ignition_sensors6-install-pkg-bionic-amd64', + 'ignition_sensors6-install-pkg-focal-amd64', 'ignition_sensors6-install_bottle-homebrew-amd64', - 'ignition_tools-ci-ign-tools1-bionic-amd64', + 'ignition_tools-ci-ign-tools1-focal-amd64', 'ignition_tools-ci-ign-tools1-homebrew-amd64', - 'ignition_tools-install-pkg-bionic-amd64', + 'ignition_tools-install-pkg-focal-amd64', 'ignition_tools1-install_bottle-homebrew-amd64', - 'ignition_transport-ci-main-bionic-amd64', + 'ignition_transport-ci-main-focal-amd64', 'ignition_transport-ci-main-homebrew-amd64', - 'ignition_transport11-install-pkg-bionic-amd64', + 'ignition_transport11-install-pkg-focal-amd64', 'ignition_transport11-install_bottle-homebrew-amd64', - 'ignition_utils-ci-ign-utils1-bionic-amd64', + 'ignition_utils-ci-ign-utils1-focal-amd64', 'ignition_utils-ci-ign-utils1-homebrew-amd64', - 'ignition_utils-install-pkg-bionic-amd64', + 'ignition_utils-install-pkg-focal-amd64', 'ignition_utils1-install_bottle-homebrew-amd64', - 'sdformat-ci-sdformat12-bionic-amd64', + 'sdformat-ci-sdformat12-focal-amd64', 'sdformat-ci-sdformat12-homebrew-amd64', 'sdformat-ci-sdformat12-windows7-amd64', - 'sdformat-install-sdformat12_pkg-bionic-amd64' + 'sdformat-install-sdformat12_pkg-focal-amd64' ], ] From c7e99233bea062706b56289f52bb13c99d573256 Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Wed, 5 Jan 2022 12:55:01 -0800 Subject: [PATCH 061/540] Remove IGN_PHYSICS_DEPENDENCIES variable (#619) The dependencies are already properly defined in the packages.apt files in the ign-physics repository. The IGN_PHYSICS_DEPENDENCIES variable in dependencies_archive.sh is incorrect, and we'd be better off without it. Signed-off-by: Steve Peters --- jenkins-scripts/lib/dependencies_archive.sh | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/jenkins-scripts/lib/dependencies_archive.sh b/jenkins-scripts/lib/dependencies_archive.sh index 920afe371..5f2cf2095 100644 --- a/jenkins-scripts/lib/dependencies_archive.sh +++ b/jenkins-scripts/lib/dependencies_archive.sh @@ -521,23 +521,6 @@ elif [[ -n "${IGN_GUI_MAJOR_VERSION}" && ${IGN_GUI_MAJOR_VERSION} -eq 3 ]]; then libignition-transport8-dev" fi -IGN_PHYSICS_DEPENDENCIES="libbenchmark-dev \\ - dart6-data \\ - libdart6-collision-ode-dev \\ - libdart6-dev \\ - libdart6-utils-urdf-dev \\ - libignition-cmake2-dev \\ - libignition-common3-dev \\ - libignition-math6-dev \\ - libignition-math6-eigen3-dev \\ - libignition-plugin-dev" -if [[ -n "${IGN_PHYSICS_MAJOR_VERSION}" && ${IGN_PHYSICS_MAJOR_VERSION} -ge 3 ]]; then - IGN_PHYSICS_DEPENDENCIES="${IGN_PHYSICS_DEPENDENCIES} \\ - libsdformat10-dev" -elif [[ -n "${IGN_PHYSICS_MAJOR_VERSION}" && ${IGN_PHYSICS_MAJOR_VERSION} -eq 2 ]]; then - IGN_PHYSICS_DEPENDENCIES="${IGN_PHYSICS_DEPENDENCIES} \\ - libsdformat9-dev" -fi IGN_PHYSICS_DART_FROM_PKGS="true" IGN_PLUGIN_DEPENDENCIES="libignition-cmake2-dev" From 5bd8bdaf606ca3c74d6f41623371cd76ec49b429 Mon Sep 17 00:00:00 2001 From: Louise Poubel Date: Tue, 11 Jan 2022 10:20:24 -0800 Subject: [PATCH 062/540] Fix failure to clone abi-compliance-checker (#623) Signed-off-by: Louise Poubel --- jenkins-scripts/docker/lib/generic-abi-base.bash | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jenkins-scripts/docker/lib/generic-abi-base.bash b/jenkins-scripts/docker/lib/generic-abi-base.bash index cef131181..fabf03b2f 100644 --- a/jenkins-scripts/docker/lib/generic-abi-base.bash +++ b/jenkins-scripts/docker/lib/generic-abi-base.bash @@ -54,7 +54,7 @@ fi echo '# BEGIN SECTION: compile and install branch: ${DEST_BRANCH}' cp -a $WORKSPACE/${ABI_JOB_SOFTWARE_NAME} /tmp/${ABI_JOB_SOFTWARE_NAME} cd /tmp/${ABI_JOB_SOFTWARE_NAME} -git fetch origin +git fetch origin git checkout origin/${DEST_BRANCH} # Normal cmake routine for ${ABI_JOB_SOFTWARE_NAME} rm -rf $WORKSPACE/build @@ -94,7 +94,7 @@ echo '# BEGIN SECTION: install the ABI checker' # Install abi-compliance-checker.git cd $WORKSPACE rm -fr $WORKSPACE/abi-compliance-checker -git clone git://github.com/lvc/abi-compliance-checker.git +git clone https://github.com/lvc/abi-compliance-checker cd abi-compliance-checker sudo perl Makefile.pl -install --prefix=/usr From 494f63a90ba74de7b809da2ebe24082bc1ec4f83 Mon Sep 17 00:00:00 2001 From: Louise Poubel Date: Tue, 11 Jan 2022 11:21:26 -0800 Subject: [PATCH 063/540] Add Garden to terminal dashboard (#622) Signed-off-by: Louise Poubel --- terminal-dashboard/_dashboard_lib.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/terminal-dashboard/_dashboard_lib.sh b/terminal-dashboard/_dashboard_lib.sh index 1e0bb5536..2712e4c7f 100644 --- a/terminal-dashboard/_dashboard_lib.sh +++ b/terminal-dashboard/_dashboard_lib.sh @@ -62,6 +62,25 @@ get_libraries_by_collection() ignition-gazebo6 ignition-launch5 ignition-fortress" + elif [ "$COLLECTION" = "garden" ]; then + LIBS=" + ignition-cmake2 + ignition-math7 + ignition-utils1 + ignition-tools + ignition-common5 + ignition-msgs9 + ignition-transport12 + ignition-fuel-tools8 + ignition-plugin + ignition-rendering7 + sdformat13 + ignition-physics6 + ignition-sensors7 + ignition-gui7 + ignition-gazebo7 + ignition-launch6 + ignition-garden" else return 1 fi From 528a840cbb26e19abe4540925596625408b2afcc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Hern=C3=A1ndez=20Cordero?= Date: Wed, 19 Jan 2022 23:27:02 +0100 Subject: [PATCH 064/540] Skip pxr for sdformat (#625) Signed-off-by: ahcorde --- jenkins-scripts/docker/sdformat-compilation.bash | 2 ++ 1 file changed, 2 insertions(+) diff --git a/jenkins-scripts/docker/sdformat-compilation.bash b/jenkins-scripts/docker/sdformat-compilation.bash index 8cfb50a1f..c7b0be5f2 100644 --- a/jenkins-scripts/docker/sdformat-compilation.bash +++ b/jenkins-scripts/docker/sdformat-compilation.bash @@ -24,6 +24,8 @@ fi export GZDEV_PROJECT_NAME="sdformat${SDFORMAT_MAJOR_VERSION}" +export BUILDING_EXTRA_CMAKE_PARAMS="-DSKIP_usd=true" + # master and major branches compilations export BUILDING_PKG_DEPENDENCIES_VAR_NAME="SDFORMAT_BASE_DEPENDENCIES" From 5271d4633544ffd3fec05feb40dc9c846c6f714c Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Wed, 19 Jan 2022 16:40:24 -0800 Subject: [PATCH 065/540] Ignore brew doctor warnings for proj@7 (#626) Needed by osrf/homebrew-simulation#1790 Signed-off-by: Steve Peters --- .../lib/project-default-devel-homebrew-amd64.bash | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash b/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash index aea36cdc9..18ed2e689 100644 --- a/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash +++ b/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash @@ -168,10 +168,10 @@ echo '# END SECTION' echo "#BEGIN SECTION: brew doctor analysis" brew missing || brew install $(brew missing | awk '{print $2}') && brew missing -# if sdl installed, skip brew doctor -# remove this line when open-scene-graph stops depending on sdl -# https://github.com/Homebrew/homebrew-core/pull/81101 -brew list | grep '^sdl$' || brew doctor +# if proj@7 installed, skip brew doctor +# remove this line when gdal stops depending on a deprecated version of proj +# https://github.com/Homebrew/homebrew-core/issues/82441 +brew list | grep '^proj@7$' || brew doctor echo '# END SECTION' # CHECK PRE_TESTS_EXECUTION_HOOK AND RUN From 07673741149a0dced75439c50e8f37a8d0cc5144 Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Fri, 21 Jan 2022 04:15:54 -0800 Subject: [PATCH 066/540] ign_common windows: use gdal (#624) * ign_rendering windows: use gdal * ign_common windows: use gdal * Use --recurse with vcpkg install * Add gdal dependency to other windows CI scripts Signed-off-by: Steve Peters --- jenkins-scripts/ign_common-default-devel-windows-amd64.bat | 5 +++++ jenkins-scripts/ign_gazebo-default-devel-windows-amd64.bat | 4 ++++ jenkins-scripts/ign_gui-default-devel-windows-amd64.bat | 4 ++++ jenkins-scripts/ign_launch-default-devel-windows-amd64.bat | 4 ++++ jenkins-scripts/ign_physics-default-devel-windows-amd64.bat | 4 ++++ .../ign_rendering-default-devel-windows-amd64.bat | 5 ++++- jenkins-scripts/ign_sensors-default-devel-windows-amd64.bat | 4 ++++ jenkins-scripts/lib/windows_library.bat | 2 +- 8 files changed, 30 insertions(+), 2 deletions(-) diff --git a/jenkins-scripts/ign_common-default-devel-windows-amd64.bat b/jenkins-scripts/ign_common-default-devel-windows-amd64.bat index 7f5a5341b..8f42e7dd5 100644 --- a/jenkins-scripts/ign_common-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_common-default-devel-windows-amd64.bat @@ -5,6 +5,11 @@ set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true set DEPEN_PKGS=ffmpeg freeimage gts tinyxml2 +for /f %%i in ('python "%SCRIPT_DIR%\tools\detect_cmake_major_version.py" "%WORKSPACE%\%VCS_DIRECTORY%\CMakeLists.txt"') do set IGN_MAJOR_VERSION=%%i +if %IGN_MAJOR_VERSION% GEQ 5 ( + set DEPEN_PKGS=%DEPEN_PKGS% gdal +) + set COLCON_PACKAGE=ignition-common set COLCON_AUTO_MAJOR_VERSION=true diff --git a/jenkins-scripts/ign_gazebo-default-devel-windows-amd64.bat b/jenkins-scripts/ign_gazebo-default-devel-windows-amd64.bat index e69391204..6bf2ee19a 100644 --- a/jenkins-scripts/ign_gazebo-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_gazebo-default-devel-windows-amd64.bat @@ -6,6 +6,10 @@ set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true :: dlfcn set DEPEN_PKGS=dlfcn-win32 cuda cppzmq curl openssl jsoncpp ffmpeg freeimage ogre ogre2 ogre22 qt5 qwt gts glib fcl eigen3 ccd assimp libyaml libzip gflags protobuf tinyxml2 zeromq +for /f %%i in ('python "%SCRIPT_DIR%\tools\detect_cmake_major_version.py" "%WORKSPACE%\%VCS_DIRECTORY%\CMakeLists.txt"') do set IGN_MAJOR_VERSION=%%i +if %IGN_MAJOR_VERSION% GEQ 7 ( + set DEPEN_PKGS=%DEPEN_PKGS% gdal +) :: This needs to be migrated to DSL to get multi-major versions correctly set COLCON_PACKAGE=ignition-gazebo set COLCON_AUTO_MAJOR_VERSION=true diff --git a/jenkins-scripts/ign_gui-default-devel-windows-amd64.bat b/jenkins-scripts/ign_gui-default-devel-windows-amd64.bat index c1de0dc65..c724e5254 100644 --- a/jenkins-scripts/ign_gui-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_gui-default-devel-windows-amd64.bat @@ -7,6 +7,10 @@ set IGN_CLEAN_WORKSPACE=true :: ogre2 from vcpkg-ports set DEPEN_PKGS=qt5 qt5-winextras qwt protobuf tinyxml2 freeimage ogre ogre22 +for /f %%i in ('python "%SCRIPT_DIR%\tools\detect_cmake_major_version.py" "%WORKSPACE%\%VCS_DIRECTORY%\CMakeLists.txt"') do set IGN_MAJOR_VERSION=%%i +if %IGN_MAJOR_VERSION% GEQ 7 ( + set DEPEN_PKGS=%DEPEN_PKGS% gdal +) :: This needs to be migrated to DSL to get multi-major versions correctly set COLCON_PACKAGE=ignition-gui set COLCON_AUTO_MAJOR_VERSION=true diff --git a/jenkins-scripts/ign_launch-default-devel-windows-amd64.bat b/jenkins-scripts/ign_launch-default-devel-windows-amd64.bat index 45d865f07..4e054415b 100644 --- a/jenkins-scripts/ign_launch-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_launch-default-devel-windows-amd64.bat @@ -5,6 +5,10 @@ set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true set DEPEN_PKGS=dlfcn-win32 cuda cppzmq curl openssl jsoncpp ffmpeg freeimage ogre ogre2 ogre22 qt5 qwt gts glib fcl eigen3 ccd assimp libyaml libzip gflags protobuf tinyxml2 zeromq +for /f %%i in ('python "%SCRIPT_DIR%\tools\detect_cmake_major_version.py" "%WORKSPACE%\%VCS_DIRECTORY%\CMakeLists.txt"') do set IGN_MAJOR_VERSION=%%i +if %IGN_MAJOR_VERSION% GEQ 6 ( + set DEPEN_PKGS=%DEPEN_PKGS% gdal +) set COLCON_PACKAGE=ignition-launch set COLCON_AUTO_MAJOR_VERSION=true diff --git a/jenkins-scripts/ign_physics-default-devel-windows-amd64.bat b/jenkins-scripts/ign_physics-default-devel-windows-amd64.bat index 38f370514..7eecb22bd 100644 --- a/jenkins-scripts/ign_physics-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_physics-default-devel-windows-amd64.bat @@ -5,6 +5,10 @@ set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true set DEPEN_PKGS=eigen3 tinyxml2 +for /f %%i in ('python "%SCRIPT_DIR%\tools\detect_cmake_major_version.py" "%WORKSPACE%\%VCS_DIRECTORY%\CMakeLists.txt"') do set IGN_MAJOR_VERSION=%%i +if %IGN_MAJOR_VERSION% GEQ 6 ( + set DEPEN_PKGS=%DEPEN_PKGS% gdal +) :: This needs to be migrated to DSL to get multi-major versions correctly set COLCON_PACKAGE=ignition-physics set COLCON_AUTO_MAJOR_VERSION=true diff --git a/jenkins-scripts/ign_rendering-default-devel-windows-amd64.bat b/jenkins-scripts/ign_rendering-default-devel-windows-amd64.bat index 1a8bee264..a76a2837a 100644 --- a/jenkins-scripts/ign_rendering-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_rendering-default-devel-windows-amd64.bat @@ -5,11 +5,14 @@ set VCS_DIRECTORY=ign-rendering set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true -set DEPEN_PKGS=boost-core dlfcn-win32 cuda eigen3 freeimage ogre ogre2 gts glib +set DEPEN_PKGS=boost-core cuda dlfcn-win32 eigen3 freeimage glib gts ogre ogre2 for /f %%i in ('python "%SCRIPT_DIR%\tools\detect_cmake_major_version.py" "%WORKSPACE%\%VCS_DIRECTORY%\CMakeLists.txt"') do set IGN_MAJOR_VERSION=%%i if %IGN_MAJOR_VERSION% GEQ 5 ( set DEPEN_PKGS=%DEPEN_PKGS% ogre22 ) +if %IGN_MAJOR_VERSION% GEQ 7 ( + set DEPEN_PKGS=%DEPEN_PKGS% gdal +) :: This needs to be migrated to DSL to get multi-major versions correctly set COLCON_PACKAGE=ignition-rendering diff --git a/jenkins-scripts/ign_sensors-default-devel-windows-amd64.bat b/jenkins-scripts/ign_sensors-default-devel-windows-amd64.bat index 627971044..7bb6fe561 100644 --- a/jenkins-scripts/ign_sensors-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_sensors-default-devel-windows-amd64.bat @@ -6,6 +6,10 @@ set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true set DEPEN_PKGS=protobuf tinyxml2 +for /f %%i in ('python "%SCRIPT_DIR%\tools\detect_cmake_major_version.py" "%WORKSPACE%\%VCS_DIRECTORY%\CMakeLists.txt"') do set IGN_MAJOR_VERSION=%%i +if %IGN_MAJOR_VERSION% GEQ 7 ( + set DEPEN_PKGS=%DEPEN_PKGS% gdal +) :: This needs to be migrated to DSL to get multi-major versions correctly set COLCON_PACKAGE=ignition-sensors set COLCON_AUTO_MAJOR_VERSION=true diff --git a/jenkins-scripts/lib/windows_library.bat b/jenkins-scripts/lib/windows_library.bat index f6ccbf395..dba048bec 100644 --- a/jenkins-scripts/lib/windows_library.bat +++ b/jenkins-scripts/lib/windows_library.bat @@ -264,7 +264,7 @@ cd %VCPKG_OSRF_DIR% git pull origin master || goto :error popd -%VCPKG_CMD% install "%1" --overlay-ports="%VCPKG_OSRF_DIR%" +%VCPKG_CMD% install --recurse "%1" --overlay-ports="%VCPKG_OSRF_DIR%" goto :EOF :: ################################## From 373425692837d0788bb23f10dfb61dafb86d4d42 Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Mon, 24 Jan 2022 13:14:43 -0800 Subject: [PATCH 067/540] brew doctor: unstable instead of fail, ignore proj@7 warnings (#627) Repeat of #626 for install_bottle jobs. Signed-off-by: Steve Peters --- .../lib/project-default-devel-homebrew-amd64.bash | 4 ++-- jenkins-scripts/lib/project-install-homebrew.bash | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash b/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash index 18ed2e689..c25483381 100644 --- a/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash +++ b/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash @@ -54,7 +54,7 @@ export HOMEBREW_NO_AUTO_UPDATE=1 # Run brew config to print system information brew config # Run brew doctor to check for problems with the system -brew doctor +brew doctor || echo MARK_AS_UNSTABLE echo '# END SECTION' echo '# BEGIN SECTION: setup the osrf/simulation tap' @@ -171,7 +171,7 @@ brew missing || brew install $(brew missing | awk '{print $2}') && brew missing # if proj@7 installed, skip brew doctor # remove this line when gdal stops depending on a deprecated version of proj # https://github.com/Homebrew/homebrew-core/issues/82441 -brew list | grep '^proj@7$' || brew doctor +brew list | grep '^proj@7$' || brew doctor || echo MARK_AS_UNSTABLE echo '# END SECTION' # CHECK PRE_TESTS_EXECUTION_HOOK AND RUN diff --git a/jenkins-scripts/lib/project-install-homebrew.bash b/jenkins-scripts/lib/project-install-homebrew.bash index 0f82abcbb..e911e3288 100644 --- a/jenkins-scripts/lib/project-install-homebrew.bash +++ b/jenkins-scripts/lib/project-install-homebrew.bash @@ -38,7 +38,7 @@ export HOMEBREW_NO_AUTO_UPDATE=1 # Run brew config to print system information brew config # Run brew doctor to check for problems with the system -brew doctor +brew doctor || echo MARK_AS_UNSTABLE echo '# END SECTION' echo '# BEGIN SECTION: setup the osrf/simulation tap' @@ -73,10 +73,10 @@ echo '# END SECTION' echo "#BEGIN SECTION: brew doctor analysis" brew missing || brew install $(brew missing | awk '{print $2}') && brew missing -# if sdl installed, skip brew doctor -# remove this line when open-scene-graph stops depending on sdl -# https://github.com/Homebrew/homebrew-core/pull/81101 -brew list | grep '^sdl$' || brew doctor +# if proj@7 installed, skip brew doctor +# remove this line when gdal stops depending on a deprecated version of proj +# https://github.com/Homebrew/homebrew-core/issues/82441 +brew list | grep '^proj@7$' || brew doctor || echo MARK_AS_UNSTABLE echo '# END SECTION' echo "# BEGIN SECTION: re-add group write permissions" From 6370d9882faeb8615c9ff5590e33948e170b29dd Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Tue, 25 Jan 2022 10:04:24 -0800 Subject: [PATCH 068/540] Don't use HOMEBREW_UPDATE_TO_TAG for now (#629) There is an unreleased fix for a brew doctor bug, so stop using HOMEBREW_UPDATE_TO_TAG for now. Signed-off-by: Steve Peters --- jenkins-scripts/lib/_homebrew_base_setup.bash | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/jenkins-scripts/lib/_homebrew_base_setup.bash b/jenkins-scripts/lib/_homebrew_base_setup.bash index 35d2308fc..09184e95b 100644 --- a/jenkins-scripts/lib/_homebrew_base_setup.bash +++ b/jenkins-scripts/lib/_homebrew_base_setup.bash @@ -15,7 +15,9 @@ else fi git -C $(${BREW_BINARY} --repo) fsck -export HOMEBREW_UPDATE_TO_TAG=1 +# don't use HOMEBREW_UPDATE_TO_TAG until brew 3.3.13 is released +# due to https://github.com/Homebrew/brew/issues/12788 +unset HOMEBREW_UPDATE_TO_TAG ${BREW_BINARY} update # manually exclude a ruby warning that jenkins thinks is from clang # https://github.com/osrf/homebrew-simulation/issues/1343 From bbcad6dc466e7c6856b8160ab604d69edff78890 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 26 Jan 2022 14:15:35 +0100 Subject: [PATCH 069/540] Try refreshing github key if auth failed for brew (#628) --- jenkins-scripts/lib/_homebrew_github_setup.bash | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/jenkins-scripts/lib/_homebrew_github_setup.bash b/jenkins-scripts/lib/_homebrew_github_setup.bash index af3e5ce1a..ae769f863 100644 --- a/jenkins-scripts/lib/_homebrew_github_setup.bash +++ b/jenkins-scripts/lib/_homebrew_github_setup.bash @@ -16,10 +16,16 @@ echo '# BEGIN SECTION: check github perms' # Github autentication. git access is provided by public key access # and hub cli needs a token if [[ -z $(ssh -T git@github.com 2>&1 | grep successfully) ]]; then - echo "The github connection seems not to be valid:" - ssh -T git@github.com - echo "Please check that the ssh key authentication is working" - exit 1 + # ssh key in github may have changed, let's try to remove and get + # the latest one + ssh-keygen -R github.com + ssh-keyscan -H github.com >> ~/.ssh/known_hosts + if [[ -z $(ssh -T git@github.com 2>&1 | grep successfully) ]]; then + echo "The github connection seems not to be valid:" + ssh -T git@github.com + echo "Please check that the ssh key authentication is working" + exit 1 + fi fi GITHUB_TOKEN_FILE="/var/lib/jenkins/.github_token" From 5ecaeb4d1b444fe01d034c6babe1f36d3f186bc5 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 26 Jan 2022 14:15:54 +0100 Subject: [PATCH 070/540] Clean the whole workspace in -debbuild jobs (#611) --- jenkins-scripts/dsl/_configs_/OSRFLinuxBuildPkgBase.groovy | 1 - 1 file changed, 1 deletion(-) diff --git a/jenkins-scripts/dsl/_configs_/OSRFLinuxBuildPkgBase.groovy b/jenkins-scripts/dsl/_configs_/OSRFLinuxBuildPkgBase.groovy index 7749e0b2e..111f57240 100644 --- a/jenkins-scripts/dsl/_configs_/OSRFLinuxBuildPkgBase.groovy +++ b/jenkins-scripts/dsl/_configs_/OSRFLinuxBuildPkgBase.groovy @@ -23,7 +23,6 @@ class OSRFLinuxBuildPkgBase wrappers { preBuildCleanup { - includePattern('pkgs/*') // the sudo does not seems to be able to remove root owned packaged deleteCommand('sudo rm -rf %s') } From 9ae4831bb73fbf0cc4a63b275f75e7aa84e72c22 Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Thu, 27 Jan 2022 09:39:09 -0800 Subject: [PATCH 071/540] Only set SKIP_usd for libsdformat 12+ (#632) Signed-off-by: Steve Peters --- jenkins-scripts/docker/sdformat-compilation.bash | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/jenkins-scripts/docker/sdformat-compilation.bash b/jenkins-scripts/docker/sdformat-compilation.bash index c7b0be5f2..67bdd9551 100644 --- a/jenkins-scripts/docker/sdformat-compilation.bash +++ b/jenkins-scripts/docker/sdformat-compilation.bash @@ -24,7 +24,9 @@ fi export GZDEV_PROJECT_NAME="sdformat${SDFORMAT_MAJOR_VERSION}" -export BUILDING_EXTRA_CMAKE_PARAMS="-DSKIP_usd=true" +if [[ ${SDFORMAT_MAJOR_VERSION} -ge 12 ]]; then + export BUILDING_EXTRA_CMAKE_PARAMS="-DSKIP_usd=true" +fi # master and major branches compilations export BUILDING_PKG_DEPENDENCIES_VAR_NAME="SDFORMAT_BASE_DEPENDENCIES" From dc9a19dbf1371a9cca4ccd6e58a50760fdeea377 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 27 Jan 2022 21:46:30 +0100 Subject: [PATCH 072/540] Fix build/ cleanup in colcon Windows build (#593) --- jenkins-scripts/lib/colcon-default-devel-windows.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/lib/colcon-default-devel-windows.bat b/jenkins-scripts/lib/colcon-default-devel-windows.bat index ef0a76b58..dfd3c408d 100644 --- a/jenkins-scripts/lib/colcon-default-devel-windows.bat +++ b/jenkins-scripts/lib/colcon-default-devel-windows.bat @@ -66,7 +66,7 @@ call %win_lib% :configure_msvc2019_compiler echo # END SECTION echo # BEGIN SECTION: setup workspace -if defined KEEP_WORKSPACE ( +if not defined KEEP_WORKSPACE ( IF exist %LOCAL_WS_BUILD% ( echo # BEGIN SECTION: preclean workspace rmdir /s /q %LOCAL_WS_BUILD% || goto :error From e1c9ed08832db40ac9c1350fed93fd114d62115b Mon Sep 17 00:00:00 2001 From: Michael Carroll Date: Fri, 28 Jan 2022 14:59:37 -0600 Subject: [PATCH 073/540] Fix shebangs on release scripts (#633) Signed-off-by: Michael Carroll --- release-repo-scripts/bump_major_version.bash | 1 + release-repo-scripts/changelog_spawn.sh | 2 +- release-repo-scripts/new_ignition_release_repos.bash | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/release-repo-scripts/bump_major_version.bash b/release-repo-scripts/bump_major_version.bash index 48ce30ce5..45421ee02 100755 --- a/release-repo-scripts/bump_major_version.bash +++ b/release-repo-scripts/bump_major_version.bash @@ -1,3 +1,4 @@ +#!/usr/bin/env bash # Copyright (C) 2019 Open Source Robotics Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/release-repo-scripts/changelog_spawn.sh b/release-repo-scripts/changelog_spawn.sh index 898528a82..cded7e418 100755 --- a/release-repo-scripts/changelog_spawn.sh +++ b/release-repo-scripts/changelog_spawn.sh @@ -1,4 +1,4 @@ -# +#!/usr/bin/env bash # Copyright (C) 2012 Open Source Robotics Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/release-repo-scripts/new_ignition_release_repos.bash b/release-repo-scripts/new_ignition_release_repos.bash index 27dfc60d0..bb1e9d36c 100755 --- a/release-repo-scripts/new_ignition_release_repos.bash +++ b/release-repo-scripts/new_ignition_release_repos.bash @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Be sure to run this script from outside a git repository. Otherwise, the # first run will fail with `fatal: remote origin already exists.` In case From 86aacda639b3e245454848bc3ab1618631b199ac Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Fri, 28 Jan 2022 13:45:55 -0800 Subject: [PATCH 074/540] Install brew :build dependencies for CI builds (#630) Signed-off-by: Steve Peters --- jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash | 2 ++ 1 file changed, 2 insertions(+) diff --git a/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash b/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash index c25483381..91044228f 100644 --- a/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash +++ b/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash @@ -75,6 +75,8 @@ fi echo "# BEGIN SECTION: install ${PROJECT_FORMULA} dependencies" # Process the package dependencies brew install ${PROJECT_FORMULA} ${PROJECT_ARGS} --only-dependencies +# the following is needed to install :build dependencies of a formula +brew install $(brew deps --1 --include-build ${PROJECT_FORMULA}) if [[ "${RERUN_FAILED_TESTS}" -gt 0 ]]; then # Install lxml for flaky_junit_merge.py From 2dd38e5f025b18f15a62f6dceb61da5caa554712 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Mon, 31 Jan 2022 15:25:42 +0100 Subject: [PATCH 075/540] Use custom apt command to avoid problems in noninteractive (#634) * Use non-interactive apt installations via common variable * Use right extension --- jenkins-scripts/docker/lib/_common_scripts.bash | 1 + jenkins-scripts/docker/lib/generic-install-base.bash | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 jenkins-scripts/docker/lib/_common_scripts.bash diff --git a/jenkins-scripts/docker/lib/_common_scripts.bash b/jenkins-scripts/docker/lib/_common_scripts.bash new file mode 100644 index 000000000..30e006bc4 --- /dev/null +++ b/jenkins-scripts/docker/lib/_common_scripts.bash @@ -0,0 +1 @@ +export APT_INSTALL="sudo DEBIAN_FRONTEND=noninteractive apt-get install -y" diff --git a/jenkins-scripts/docker/lib/generic-install-base.bash b/jenkins-scripts/docker/lib/generic-install-base.bash index a5781efa3..8a944ca00 100644 --- a/jenkins-scripts/docker/lib/generic-install-base.bash +++ b/jenkins-scripts/docker/lib/generic-install-base.bash @@ -5,6 +5,7 @@ echo '# BEGIN SECTION: setup the testing enviroment' # Define the name to be used in docker DOCKER_JOB_NAME="install_job" . ${SCRIPT_DIR}/lib/boilerplate_prepare.sh +. ${SCRIPT_DIR}/lib/_common_scripts.bash echo '# END SECTION' cat > build.sh << DELIM @@ -24,7 +25,7 @@ fi if [ `expr length "${INSTALL_JOB_PKG} "` -gt 1 ]; then echo "# BEGIN SECTION: try to install package: ${INSTALL_JOB_PKG}" sudo apt-get update -sudo apt-get install -y ${INSTALL_JOB_PKG} +${APT_INSTALL} ${INSTALL_JOB_PKG} echo '# END SECTION' fi From c878de46f662e0897c5e5b2ca2d7c4087988bd49 Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Sat, 5 Feb 2022 22:14:20 -0800 Subject: [PATCH 076/540] brew: set PKG_CONFIG_PATH if using ffmpeg@4 (#636) This formula is keg-only, so we need to give a little help for it to be found by CI compilation jobs. Signed-off-by: Steve Peters --- jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash b/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash index 91044228f..2cce75ae5 100644 --- a/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash +++ b/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash @@ -130,6 +130,10 @@ fi if brew ruby -e "exit ! '${PROJECT_FORMULA}'.f.recursive_dependencies.map(&:name).keep_if { |d| d == 'cmake@3.21.4' }.empty?"; then export PATH=/usr/local/opt/cmake@3.21.4/bin:${PATH} fi +# Workaround for ffmpeg 4: set CMAKE_PREFIX_PATH and PKG_CONFIG_PATH if we are using ffmpeg@4 +if brew ruby -e "exit ! '${PROJECT_FORMULA}'.f.recursive_dependencies.map(&:name).keep_if { |d| d == 'ffmpeg@4' }.empty?"; then + export PKG_CONFIG_PATH=${PKG_CONFIG_PATH}:/usr/local/opt/ffmpeg@4/lib/pkgconfig +fi # Workaround for tbb@2020_u3: set CPATH, LIBRARY_PATH, and CMAKE_PREFIX_PATH if brew ruby -e "exit ! '${PROJECT_FORMULA}'.f.recursive_dependencies.map(&:name).keep_if { |d| d == 'tbb@2020_u3' }.empty?"; then export CMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH}:/usr/local/opt/tbb@2020_u3 From 7b44d340c113d03783bab0a202c697515bd84392 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 9 Feb 2022 15:13:17 +0100 Subject: [PATCH 077/540] Use INVALIDATE_DOCKER_CACHE in nightly generation (#637) --- jenkins-scripts/docker/lib/debbuild-base.bash | 3 +++ 1 file changed, 3 insertions(+) diff --git a/jenkins-scripts/docker/lib/debbuild-base.bash b/jenkins-scripts/docker/lib/debbuild-base.bash index 50198d596..3c0623d1c 100644 --- a/jenkins-scripts/docker/lib/debbuild-base.bash +++ b/jenkins-scripts/docker/lib/debbuild-base.bash @@ -7,6 +7,9 @@ if [ "${UPLOAD_TO_REPO}" = "nightly" ]; then # SOURCE_TARBALL_URI is reused in nightly mode to indicate the branch # to built nightly packages from NIGHTLY_SRC_BRANCH=${SOURCE_TARBALL_URI} + # There are many problem in the nightlies with package versions preventing + # the dependency solver to work properly. Set INVALIDATE_DOCKER_CACHE + export INVALIDATE_DOCKER_CACHE=true fi # Option to use $WORKSPACE/repo as container (git or hg) for the nightly source From 9796588acbd82b69867230f441cd0c36a3a0da11 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 11 Feb 2022 17:49:41 +0100 Subject: [PATCH 078/540] Be sure of not using gcc8 on jammy (#640) * Be sure of not using gcc8 on Jammy --- jenkins-scripts/docker/lib/debbuild-base.bash | 15 +++++++++------ .../docker/lib/docker_generate_dockerfile.bash | 2 +- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/jenkins-scripts/docker/lib/debbuild-base.bash b/jenkins-scripts/docker/lib/debbuild-base.bash index 3c0623d1c..9ae86e544 100644 --- a/jenkins-scripts/docker/lib/debbuild-base.bash +++ b/jenkins-scripts/docker/lib/debbuild-base.bash @@ -190,12 +190,15 @@ g++ --version echo '# END SECTION' fi -if $NEED_C17_COMPILER; then -echo '# BEGIN SECTION: install C++17 compiler' -sudo apt-get install -y gcc-8 g++-8 -sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-8 800 --slave /usr/bin/g++ g++ /usr/bin/g++-8 --slave /usr/bin/gcov gcov /usr/bin/gcov-8 -g++ --version -echo '# END SECTION' +# Be sure that a previous bug using g++8 compiler is not present anymore +if [[ ${DISTRO} == 'jammy' ]]; then + [[ \$(/usr/bin/gcc --version | grep 'gcc-8') ]] && ( echo "gcc-8 version found. A bug." ; exit 1 ) +elif $NEED_C17_COMPILER; then + echo '# BEGIN SECTION: install C++17 compiler' + sudo apt-get install -y gcc-8 g++-8 + sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-8 800 --slave /usr/bin/g++ g++ /usr/bin/g++-8 --slave /usr/bin/gcov gcov /usr/bin/gcov-8 + g++ --version + echo '# END SECTION' fi echo '# BEGIN SECTION: create source package' \${OSRF_VERSION} diff --git a/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash b/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash index a28c1c32d..b12fb8cb8 100644 --- a/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash +++ b/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash @@ -305,7 +305,7 @@ DELIM_DOCKER31 # Beware of moving this code since it needs to run update-alternative after # installing the default compiler in PACKAGES_CACHE_AND_CHECK_UPDATES -if ${NEED_C17_COMPILER}; then +if ${NEED_C17_COMPILER} && [[ ${DISTRO} != jammy ]]; then cat >> Dockerfile << DELIM_GCC8 RUN apt-get update \\ && apt-get install -y g++-8 \\ From 21cb62d349cba7454e4bfc312af88d222bf1013f Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 11 Feb 2022 22:34:04 +0100 Subject: [PATCH 079/540] Use xz to compress .deb packages on jammy (#641) --- jenkins-scripts/docker/lib/debbuild-base.bash | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/jenkins-scripts/docker/lib/debbuild-base.bash b/jenkins-scripts/docker/lib/debbuild-base.bash index 9ae86e544..66695cc7b 100644 --- a/jenkins-scripts/docker/lib/debbuild-base.bash +++ b/jenkins-scripts/docker/lib/debbuild-base.bash @@ -209,7 +209,18 @@ if [[ ${DISTRO} == 'focal' && (${ARCH} == 'arm64' || ${ARCH} == 'armhf') ]]; the no_lintian_param="--no-lintian" fi -debuild \${no_lintian_param} --no-tgz-check -uc -us -S --source-option=--include-binaries +# our packages.o.o running xenial does not support default zstd compression of +# .deb files in jammy. Keep using xz. Not a trivial change, requires wrapper over dpkg-deb +if [[ ${DISTRO} == 'jammy' ]]; then + sudo bash -c 'echo \#\!/bin/bash > /usr/local/bin/dpkg-deb' + sudo bash -c 'echo "/usr/bin/dpkg-deb -Zxz \\\$@" >> /usr/local/bin/dpkg-deb' + sudo cat /usr/local/bin/dpkg-deb + sudo chmod +x /usr/local/bin/dpkg-deb + export PATH=/usr/local/bin:\$PATH + preserve_path='--preserve-envvar PATH' +fi + +debuild \${no_lintian_param} \${preserve_path} --no-tgz-check -uc -us -S --source-option=--include-binaries cp ../*.dsc $WORKSPACE/pkgs cp ../*.orig.* $WORKSPACE/pkgs @@ -231,7 +242,7 @@ if [[ $DISTRO != 'bionic' ]]; then fi echo '# BEGIN SECTION: create deb packages' -debuild \${no_lintian_param} --no-tgz-check -uc -us --source-option=--include-binaries -j${MAKE_JOBS} +debuild \${no_lintian_param} \${preserve_path} --no-tgz-check -uc -us --source-option=--include-binaries -j${MAKE_JOBS} echo '# END SECTION' echo '# BEGIN SECTION: export pkgs' From f0869deb5a89ad0eb26d583b74f91702d84ae02e Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 11 Feb 2022 22:35:39 +0100 Subject: [PATCH 080/540] Use autopkgtest (debian testing mechanism) in debbuild runs (#638) * Generalize autopkgtest: use it in debbuilder --- jenkins-scripts/docker/lib/_gazebo_utils.sh | 17 ++++++++++++++ jenkins-scripts/docker/lib/debbuild-base.bash | 23 +++++++++++++------ .../docker/lib/debian-git-repo-base.bash | 19 ++------------- .../lib/docker_generate_dockerfile.bash | 10 ++------ 4 files changed, 37 insertions(+), 32 deletions(-) diff --git a/jenkins-scripts/docker/lib/_gazebo_utils.sh b/jenkins-scripts/docker/lib/_gazebo_utils.sh index 01de2b575..14f5405b7 100644 --- a/jenkins-scripts/docker/lib/_gazebo_utils.sh +++ b/jenkins-scripts/docker/lib/_gazebo_utils.sh @@ -36,3 +36,20 @@ if [ \$DIFF -lt 180 ]; then fi echo '# END SECTION' """ + +DEBBUILD_AUTOPKGTEST=""" +if $RUN_AUTOPKGTEST; then +echo '# BEGIN SECTION: run autopkgtest' +cd $WORKSPACE/pkgs +set +e +sudo autopkgtest -B *.deb *.dsc -- null +# autopkgtest will return 0 if there are successful tests and 8 if there are no tests +testret=\$? +if [[ \$testret != 0 ]] && [[ \$testret != 8 ]]; then + echo 'Problem in running autopkgtest: \$testret' + exit 1 +fi +set -e +echo '# END SECTION' +fi +""" diff --git a/jenkins-scripts/docker/lib/debbuild-base.bash b/jenkins-scripts/docker/lib/debbuild-base.bash index 66695cc7b..8de9894c4 100644 --- a/jenkins-scripts/docker/lib/debbuild-base.bash +++ b/jenkins-scripts/docker/lib/debbuild-base.bash @@ -19,7 +19,12 @@ fi # testing jobs and seems to be slow at the end of jenkins jobs export ENABLE_REAPER=false +# autopkgtest is a mechanism to test the installation of the generated packages +# at the end of the package production. +RUN_AUTOPKGTEST=${RUN_AUTOPKGTEST:-true} + . ${SCRIPT_DIR}/lib/boilerplate_prepare.sh +. ${SCRIPT_DIR}/lib/_gazebo_utils.sh cat > build.sh << DELIM ################################################### @@ -259,17 +264,21 @@ done # check at least one upload test \$FOUND_PKG -eq 1 || exit 1 echo '# END SECTION' + +cat /etc/apt/sources.list +${DEBBUILD_AUTOPKGTEST} DELIM OSRF_REPOS_TO_USE=${OSRF_REPOS_TO_USE:=stable} DEPENDENCY_PKGS="devscripts \ - ubuntu-dev-tools \ - debhelper \ - wget \ - ca-certificates \ - equivs \ - dh-make \ - git" + ubuntu-dev-tools \ + debhelper \ + wget \ + ca-certificates \ + equivs \ + dh-make \ + git \ + autopkgtest" . ${SCRIPT_DIR}/lib/docker_generate_dockerfile.bash . ${SCRIPT_DIR}/lib/docker_run.bash diff --git a/jenkins-scripts/docker/lib/debian-git-repo-base.bash b/jenkins-scripts/docker/lib/debian-git-repo-base.bash index d7e8a3bd9..48ad8121f 100644 --- a/jenkins-scripts/docker/lib/debian-git-repo-base.bash +++ b/jenkins-scripts/docker/lib/debian-git-repo-base.bash @@ -5,6 +5,7 @@ export ENABLE_REAPER=false . ${SCRIPT_DIR}/lib/boilerplate_prepare.sh +. ${SCRIPT_DIR}/lib/_gazebo_utils.sh # The git plugin leaves a repository copy with a detached HEAD # state. gbp does not like it thus the need of using --git-ignore-branch @@ -128,23 +129,7 @@ done test \$FOUND_PKG -eq 1 || exit 1 echo '# END SECTION' -if $RUN_AUTOPKGTEST; then -# Ubuntu has no autopkgtest command in the autopkgtest package -if [ "$LINUX_DISTRO" != "ubuntu" ]; then -echo '# BEGIN SECTION: run tests' -cd $WORKSPACE/pkgs -set +e -autopkgtest -B *.deb *.dsc -- null -# autopkgtest will return 0 if there are successful tests and 8 if there are no tests -testret=\$? -if [[ \$testret != 0 ]] && [[ \$testret != 8 ]]; then - echo "Problem in running autopkgtest: \$testret" - exit 1 -fi -set -e -echo '# END SECTION' -fi -fi +${DEBBUILD_AUTOPKGTEST} echo '# BEGIN SECTION: clean up git build' cd $REPO_PATH diff --git a/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash b/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash index b12fb8cb8..625c8ef88 100644 --- a/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash +++ b/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash @@ -156,14 +156,8 @@ fi if [[ ${LINUX_DISTRO} == 'ubuntu' ]]; then if [[ ${ARCH} != 'armhf' && ${ARCH} != 'arm64' ]]; then cat >> Dockerfile << DELIM_DOCKER_ARCH - # Note that main,restricted and universe are not here, only multiverse - # main, restricted and unvierse are already setup in the original image - RUN echo "deb ${SOURCE_LIST_URL} ${DISTRO} multiverse" \\ - >> /etc/apt/sources.list && \\ - echo "deb ${SOURCE_LIST_URL} ${DISTRO}-updates main restricted universe multiverse" \\ - >> /etc/apt/sources.list && \\ - echo "deb ${SOURCE_LIST_URL} ${DISTRO}-security main restricted universe multiverse" && \\ - >> /etc/apt/sources.list + RUN echo "deb ${SOURCE_LIST_URL} ${DISTRO}-security main restricted universe multiverse" && \\ + >> /etc/apt/sources.list DELIM_DOCKER_ARCH fi fi From b2e94ef1ce6056a72b5129f34b1b41652e15ef8c Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Sun, 13 Feb 2022 02:05:10 +0100 Subject: [PATCH 081/540] Print ignition packages and versions in nightly generation to help debugging (#642) * Print ignition packages versions during nightly generation --- jenkins-scripts/docker/lib/debbuild-base.bash | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/jenkins-scripts/docker/lib/debbuild-base.bash b/jenkins-scripts/docker/lib/debbuild-base.bash index 8de9894c4..eb83e61ae 100644 --- a/jenkins-scripts/docker/lib/debbuild-base.bash +++ b/jenkins-scripts/docker/lib/debbuild-base.bash @@ -172,6 +172,12 @@ sudo apt-get update if [ ${DISTRO} = 'buster' ]; then sudo apt-get install -y dwz=0.13-5~bpo10+1 fi + +# Help to debug race conditions in nightly generation or other problems with versions +if ${NIGHTLY_MODE}; then + apt-cache show *ignition* | ( grep 'Package\\|Version' || true) +fi + sudo DEBIAN_FRONTEND=noninteractive mk-build-deps -r -i debian/control --tool 'apt-get --yes -o Debug::pkgProblemResolver=yes -o Debug::BuildDeps=yes' # new versions of mk-build-deps > 2.21.1 left buildinfo and changes files in the code rm -f ${PACKAGE_ALIAS}-build-deps_*.{buildinfo,changes} From 3dfe8f15ea65b955ca1c6b417f069a78096ab3d2 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Mon, 14 Feb 2022 16:21:57 +0100 Subject: [PATCH 082/540] Retry pkgs installation in nightlies if failed (#643) --- jenkins-scripts/docker/lib/debbuild-base.bash | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/jenkins-scripts/docker/lib/debbuild-base.bash b/jenkins-scripts/docker/lib/debbuild-base.bash index eb83e61ae..edbc67516 100644 --- a/jenkins-scripts/docker/lib/debbuild-base.bash +++ b/jenkins-scripts/docker/lib/debbuild-base.bash @@ -173,12 +173,24 @@ if [ ${DISTRO} = 'buster' ]; then sudo apt-get install -y dwz=0.13-5~bpo10+1 fi +timeout=1 # Help to debug race conditions in nightly generation or other problems with versions if ${NIGHTLY_MODE}; then apt-cache show *ignition* | ( grep 'Package\\|Version' || true) + timeout=60 fi -sudo DEBIAN_FRONTEND=noninteractive mk-build-deps -r -i debian/control --tool 'apt-get --yes -o Debug::pkgProblemResolver=yes -o Debug::BuildDeps=yes' +update_done=false +seconds_waiting=0 +while (! \$update_done); do + sudo DEBIAN_FRONTEND=noninteractive mk-build-deps \ + -r -i debian/control \ + --tool 'apt-get --yes -o Debug::pkgProblemResolver=yes -o Debug::BuildDeps=yes' \ + && update_done=true + sleep 1 && seconds_waiting=\$((seconds_waiting+1)) + [ \$seconds_waiting -gt \$timeout ] && exit 1 +done + # new versions of mk-build-deps > 2.21.1 left buildinfo and changes files in the code rm -f ${PACKAGE_ALIAS}-build-deps_*.{buildinfo,changes} echo '# END SECTION' From bfd53244bd17c9e982417c8db9bf457c6096da9d Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 15 Feb 2022 22:16:48 +0100 Subject: [PATCH 083/540] Remove vcpkg boost,ogre,gdal,protobuf before running Gazebo builds on Windows (#646) * Remove vcpkg packages before running Gazebo builds, except for tinyxml2, which is needed by ign-msgs5 --- jenkins-scripts/lib/gazebo-base-windows.bat | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/jenkins-scripts/lib/gazebo-base-windows.bat b/jenkins-scripts/lib/gazebo-base-windows.bat index 8dd80e5ca..81bee5b53 100644 --- a/jenkins-scripts/lib/gazebo-base-windows.bat +++ b/jenkins-scripts/lib/gazebo-base-windows.bat @@ -11,10 +11,17 @@ echo # END SECTION :: avoid conflicts with vcpkg packages call %win_lib% :disable_vcpkg_integration -call %win_lib% :remove_vcpkg_package protobuf +call %win_lib% :remove_vcpkg_package boost-uninstall +call %win_lib% :remove_vcpkg_package gdal +call %win_lib% :remove_vcpkg_package ogre +call %win_lib% :remove_vcpkg_package protobuf call %win_lib% :remove_vcpkg_package qt5 call %win_lib% :remove_vcpkg_package qwt +:: msgs5 needs tinyxml2 which was never in the win32 deps but it can be used +:: from vcpkg. +call %win_lib% :install_vcpkg_package tinyxml2 + :: IF exist %LOCAL_WS% ( rmdir /s /q %LOCAL_WS% ) || goto %win_lib% :error :: reusing the workspace From d044dd664dbc510f6aba90d2b530320e001cad0f Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 16 Feb 2022 23:51:19 +0100 Subject: [PATCH 084/540] Remove NEED_C11_COMPILER variable and simplify NEED_C17_COMPILER (#648) * Fix the logic to support C++17 compiler and remove C++11 --- .../docker/lib/boilerplate_prepare.sh | 24 +++++++------------ jenkins-scripts/docker/lib/debbuild-base.bash | 18 ++------------ .../lib/docker_generate_dockerfile.bash | 4 ++-- .../multidistribution-ignition-debbuild.bash | 1 - 4 files changed, 13 insertions(+), 34 deletions(-) diff --git a/jenkins-scripts/docker/lib/boilerplate_prepare.sh b/jenkins-scripts/docker/lib/boilerplate_prepare.sh index 8cd62b771..e04759b9b 100644 --- a/jenkins-scripts/docker/lib/boilerplate_prepare.sh +++ b/jenkins-scripts/docker/lib/boilerplate_prepare.sh @@ -102,17 +102,19 @@ if [ -z ${ENABLE_REAPER} ]; then fi # We use ignitionsrobotics or osrf. osrf by default -if [ -Z ${GITHUB_ORG} ]; then +if [ -z ${GITHUB_ORG} ]; then GITHUB_ORG="osrf" fi -# By default, do not need to use C++11 compiler -if [ -z ${NEED_C11_COMPILER} ]; then - NEED_C11_COMPILER=false +if [ -z "${NEED_C17_COMPILER}" ]; then + export INSTALL_C17_COMPILER=false fi - -if [ -z ${NEED_C17_COMPILER} ]; then - NEED_C17_COMPILER=false +# Check if we need to install a custom compiler in old distributions +export INSTALL_C17_COMPILER=false +if ${NEED_C17_COMPILER}; then + if [ ${DISTRO} = 'bionic' ] || [ ${DISTRO} = 'buster' ]; then + export INSTALL_C17_COMPILER=true + fi fi # By default, do not use ROS @@ -126,14 +128,6 @@ if [ -z ${NEED_GCC48_COMPILER} ]; then NEED_GCC48_COMPILER=false fi -# Only precise needs to install a C++11 compiler. Trusty on -# already have a supported version -if $NEED_C11_COMPILER; then - if [[ $DISTRO != 'precise' ]]; then - NEED_C11_COMPILER=false - fi -fi - # in some machines squid is returning [ -z ${NEED_SQUID_WORKAROUND} ] && NEED_SQUID_WORKAROUND=false diff --git a/jenkins-scripts/docker/lib/debbuild-base.bash b/jenkins-scripts/docker/lib/debbuild-base.bash index edbc67516..969a9c503 100644 --- a/jenkins-scripts/docker/lib/debbuild-base.bash +++ b/jenkins-scripts/docker/lib/debbuild-base.bash @@ -199,24 +199,10 @@ if [ -f /usr/bin/rosdep ]; then rosdep init fi -if $NEED_C11_COMPILER || $NEED_GCC48_COMPILER; then -echo '# BEGIN SECTION: install C++11 compiler' -if [ ${DISTRO} = 'precise' ]; then -sudo apt-get install -y python-software-propertie software-properties-common || true -sudo add-apt-repository ppa:ubuntu-toolchain-r/test -sudo apt-get update -fi -sudo apt-get install -y gcc-4.8 g++-4.8 -sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-4.8 50 -sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-4.8 50 -g++ --version -echo '# END SECTION' -fi - # Be sure that a previous bug using g++8 compiler is not present anymore -if [[ ${DISTRO} == 'jammy' ]]; then +if [[ ${DISTRO} == 'jammy' || ${DISTRO} == 'focal' ]]; then [[ \$(/usr/bin/gcc --version | grep 'gcc-8') ]] && ( echo "gcc-8 version found. A bug." ; exit 1 ) -elif $NEED_C17_COMPILER; then +elif $INSTALL_C17_COMPILER; then echo '# BEGIN SECTION: install C++17 compiler' sudo apt-get install -y gcc-8 g++-8 sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-8 800 --slave /usr/bin/g++ g++ /usr/bin/g++-8 --slave /usr/bin/gcov gcov /usr/bin/gcov-8 diff --git a/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash b/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash index 625c8ef88..c5450fad8 100644 --- a/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash +++ b/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash @@ -27,7 +27,7 @@ if [[ -z ${LINUX_DISTRO} ]]; then export LINUX_DISTRO="ubuntu" fi -[[ -z ${NEED_C17_COMPILER} ]] && NEED_C17_COMPILER=false +[[ -z ${INSTALL_C17_COMPILER} ]] && INSTALL_C17_COMPILER=false export APT_PARAMS= @@ -299,7 +299,7 @@ DELIM_DOCKER31 # Beware of moving this code since it needs to run update-alternative after # installing the default compiler in PACKAGES_CACHE_AND_CHECK_UPDATES -if ${NEED_C17_COMPILER} && [[ ${DISTRO} != jammy ]]; then +if ${INSTALL_C17_COMPILER}; then cat >> Dockerfile << DELIM_GCC8 RUN apt-get update \\ && apt-get install -y g++-8 \\ diff --git a/jenkins-scripts/docker/multidistribution-ignition-debbuild.bash b/jenkins-scripts/docker/multidistribution-ignition-debbuild.bash index 3c129a0c7..776610ae6 100644 --- a/jenkins-scripts/docker/multidistribution-ignition-debbuild.bash +++ b/jenkins-scripts/docker/multidistribution-ignition-debbuild.bash @@ -7,6 +7,5 @@ SCRIPT_DIR="${SCRIPT_DIR%/*}" export RELEASE_REPO_DIRECTORY=${DISTRO} export ENABLE_ROS=false export GITHUB_ORG=ignitionrobotics -export NEED_C11_COMPILER=true . ${SCRIPT_DIR}/lib/debbuild-base.bash From 6f1dc062f8a33f83a3116dd67e2d8de39e44d403 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 17 Feb 2022 00:01:35 +0100 Subject: [PATCH 085/540] Wait until 5min in nightly generation to workaround race conditions (#645) --- jenkins-scripts/docker/lib/debbuild-base.bash | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/jenkins-scripts/docker/lib/debbuild-base.bash b/jenkins-scripts/docker/lib/debbuild-base.bash index 969a9c503..7c95b391b 100644 --- a/jenkins-scripts/docker/lib/debbuild-base.bash +++ b/jenkins-scripts/docker/lib/debbuild-base.bash @@ -173,11 +173,12 @@ if [ ${DISTRO} = 'buster' ]; then sudo apt-get install -y dwz=0.13-5~bpo10+1 fi -timeout=1 +timeout=0 # Help to debug race conditions in nightly generation or other problems with versions if ${NIGHTLY_MODE}; then apt-cache show *ignition* | ( grep 'Package\\|Version' || true) - timeout=60 + # 5 minutes to give time to the uploader + timeout=300 fi update_done=false @@ -186,9 +187,9 @@ while (! \$update_done); do sudo DEBIAN_FRONTEND=noninteractive mk-build-deps \ -r -i debian/control \ --tool 'apt-get --yes -o Debug::pkgProblemResolver=yes -o Debug::BuildDeps=yes' \ - && update_done=true - sleep 1 && seconds_waiting=\$((seconds_waiting+1)) - [ \$seconds_waiting -gt \$timeout ] && exit 1 + && break + sleep 60 && seconds_waiting=\$((seconds_waiting+60)) + [ \$seconds_waiting -ge \$timeout ] && exit 1 done # new versions of mk-build-deps > 2.21.1 left buildinfo and changes files in the code From d98af5e5f80adb076657890efd03c5f773381a7f Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 25 Feb 2022 15:38:13 +0100 Subject: [PATCH 086/540] Use a special node tag for processing nightlies (#650) Signed-off-by: Jose Luis Rivero --- release.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/release.py b/release.py index dce19ee40..905d6f552 100755 --- a/release.py +++ b/release.py @@ -623,6 +623,15 @@ def go(argv): if (NIGHTLY and a == 'i386'): continue + # control nightly generation using a single machine to process + # all distribution builds to avoid race conditions. Note: this + # assumes that large-memory nodes are beind used for nightly + # tags. + # https://github.com/ignition-tooling/release-tools/issues/644 + if (NIGHTLY): + assert a == 'amd64', f'Nightly tag assumed amd64 but arch is {a}' + linux_platform_params['JENKINS_NODE_TAG'] = 'linux-nightly-' + d + linux_platform_params_query = urllib.parse.urlencode(linux_platform_params) url = '%s/job/%s/buildWithParameters?%s'%(JENKINS_URL, job_name, linux_platform_params_query) From 6b114c079c2454d262fc61c162ab91471079cc24 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 25 Feb 2022 15:58:04 +0100 Subject: [PATCH 087/540] Native docker platform support for armhf on Jammy (#647) * Test armhf/jammy platform docker support * Support for buildx calls --- .../docker/lib/docker_generate_dockerfile.bash | 13 +++++++++++-- jenkins-scripts/docker/lib/docker_run.bash | 12 ++++++++++-- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash b/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash index c5450fad8..ed5c00e35 100644 --- a/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash +++ b/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash @@ -97,8 +97,17 @@ case ${ARCH} in FROM_VALUE=${ARCH}/${LINUX_DISTRO}:${DISTRO} fi ;; - 'armhf' | 'arm64' ) - FROM_VALUE=osrf/${LINUX_DISTRO}_${ARCH}:${DISTRO} + 'armhf') + # There is no osrf/jammy_armhf image. Trying new + # platform support in docker + if [[ ${DISTRO} == 'jammy' ]]; then + FROM_VALUE=${LINUX_DISTRO}:${DISTRO} + else + FROM_VALUE=osrf/${LINUX_DISTRO}_${ARCH}:${DISTRO} + fi + ;; + 'arm64') + FROM_VALUE=osrf/${LINUX_DISTRO}_${ARCH}:${DISTRO} ;; *) echo "Arch unknown" diff --git a/jenkins-scripts/docker/lib/docker_run.bash b/jenkins-scripts/docker/lib/docker_run.bash index cacaa3905..5c96eaef5 100644 --- a/jenkins-scripts/docker/lib/docker_run.bash +++ b/jenkins-scripts/docker/lib/docker_run.bash @@ -23,7 +23,15 @@ fi USERID=$(id -u) USER=$(whoami) -sudo docker build ${_DOCKER_BUILD_EXTRA_ARGS} \ +# platform support starts on versions greater than 17.07 +PLAFTORM_PARAM= +DOCKER_CLI_PLUGIN= +if [[ ${DISTRO} == 'jammy' && ${ARCH} == 'armhf' ]]; then + PLAFTORM_PARAM="--platform=linux/armhf" + DOCKER_CLI_PLUGIN="buildx" +fi + +sudo docker ${DOCKER_CLI_PLUGIN} build ${PLAFTORM_PARAM} ${_DOCKER_BUILD_EXTRA_ARGS} \ --build-arg GID=$(id -g $USER) \ --build-arg USERID=$USERID \ --build-arg USER=$USER \ @@ -74,7 +82,7 @@ if [[ -d /dev/snd ]]; then fi # DOCKER_FIX is for workaround https://github.com/docker/docker/issues/14203 -sudo ${docker_cmd} run $EXTRA_PARAMS_STR \ +sudo ${docker_cmd} run ${PLAFTORM_PARAM} $EXTRA_PARAMS_STR \ -e DOCKER_FIX='' \ -e WORKSPACE=${WORKSPACE} \ -e TERM=xterm-256color \ From cf03a6a9979bb1f58de8a222409ae9977eb4cdd8 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 1 Mar 2022 16:04:52 +0100 Subject: [PATCH 088/540] Use docker-jammy TAG in jenkins for Jammy jobs (#649) --- release.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/release.py b/release.py index 905d6f552..21b3bc7a2 100755 --- a/release.py +++ b/release.py @@ -617,6 +617,8 @@ def go(argv): # Need to use JENKINS_NODE_TAG parameter for large memory nodes # since it runs qemu emulation linux_platform_params['JENKINS_NODE_TAG'] = 'linux-' + a + elif(d == 'jammy'): + linux_platform_params['JENKINS_NODE_TAG'] = 'docker-jammy' elif ('ignition-physics' in args.package_alias): linux_platform_params['JENKINS_NODE_TAG'] = 'large-memory' From 21d487698ac699e36c7df4445adad4fbcac3f4db Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Tue, 1 Mar 2022 10:56:50 -0800 Subject: [PATCH 089/540] Build ign-utils1 nightly before ign-math7 (#651) Signed-off-by: Steve Peters --- jenkins-scripts/dsl/ignition_collection.dsl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index 210b58aa9..cd8caa1cc 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -23,9 +23,9 @@ ignition_collections = [ nightly_jobs: [ 'tools' : [ debbuild: 'ign-tools' , branch: 'ign-tools1' ], 'cmake' : [ debbuild: 'ign-cmake2' , branch: 'ign-cmake2' ], + 'utils' : [ debbuild: 'ign-utils1' , branch: 'ign-utils1' ], 'math' : [ debbuild: 'ign-math7' , branch: 'main' ], 'plugin' : [ debbuild: 'ign-plugin' , branch: 'ign-plugin1' ], - 'utils' : [ debbuild: 'ign-utils1' , branch: 'ign-utils1' ], 'common' : [ debbuild: 'ign-common5' , branch: 'main' ], 'msgs' : [ debbuild: 'ign-msgs9' , branch: 'main' ], 'rendering' : [ debbuild: 'ign-rendering7' , branch: 'main' ], From 0c38102fb3bfd1893945923ed86e46e17605f39d Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Thu, 3 Mar 2022 20:33:18 -0800 Subject: [PATCH 090/540] brew: test fix for new tbb with gazebo (#652) Don't set tbb@2020_u3 variables if github branch name matches replace_deprecated_tbb_task, which allows testing of osrf/gazebo#3174. Signed-off-by: Steve Peters --- .../lib/project-default-devel-homebrew-amd64.bash | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash b/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash index 2cce75ae5..64fbce01b 100644 --- a/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash +++ b/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash @@ -136,9 +136,11 @@ if brew ruby -e "exit ! '${PROJECT_FORMULA}'.f.recursive_dependencies.map(&:name fi # Workaround for tbb@2020_u3: set CPATH, LIBRARY_PATH, and CMAKE_PREFIX_PATH if brew ruby -e "exit ! '${PROJECT_FORMULA}'.f.recursive_dependencies.map(&:name).keep_if { |d| d == 'tbb@2020_u3' }.empty?"; then - export CMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH}:/usr/local/opt/tbb@2020_u3 - export CPATH=${CPATH}:/usr/local/opt/tbb@2020_u3/include - export LIBRARY_PATH=${LIBRARY_PATH}:/usr/local/opt/tbb@2020_u3/lib + if [ "${ghprbSourceBranch}" != "replace_deprecated_tbb_task" ]; then + export CMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH}:/usr/local/opt/tbb@2020_u3 + export CPATH=${CPATH}:/usr/local/opt/tbb@2020_u3/include + export LIBRARY_PATH=${LIBRARY_PATH}:/usr/local/opt/tbb@2020_u3/lib + fi fi # Workaround for tinyxml2 6.2.0: set CMAKE_PREFIX_PATH and PKG_CONFIG_PATH if we are using tinyxml2@6.2.0 if brew ruby -e "exit ! '${PROJECT_FORMULA}'.f.recursive_dependencies.map(&:name).keep_if { |d| d == 'tinyxml2@6.2.0' }.empty?"; then From e359153385eddfb1fc287d6398245e80bd1140b5 Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Mon, 7 Mar 2022 13:57:55 -0800 Subject: [PATCH 091/540] garden view: add more main branch CI jobs (#653) Signed-off-by: Steve Peters --- jenkins-scripts/dsl/ignition_collection.dsl | 22 ++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index cd8caa1cc..e20efd496 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -290,14 +290,14 @@ ignition_collection_jobs = 'sdformat-install-sdformat12_pkg-focal-amd64' ], 'garden' : [ - 'ign_common-ign-4-win', + 'ign_common-ci-win', 'ign_fuel-tools-ci-win', 'ign_gazebo-ci-win', 'ign_gui-ci-win', 'ign_launch-ci-win', 'ign_math-ci-win', 'ign_msgs-ci-win', - 'ign_physics-ign-5-win', + 'ign_physics-ci-win', 'ign_plugin-ign-1-win', 'ign_rendering-ci-win', 'ign_sensors-ci-win', @@ -309,8 +309,8 @@ ignition_collection_jobs = 'ignition_cmake-ci-ign-cmake2-windows7-amd64', 'ignition_cmake2-install-pkg-focal-amd64', 'ignition_cmake2-install_bottle-homebrew-amd64', - 'ignition_common-ci-ign-common4-focal-amd64', - 'ignition_common-ci-ign-common4-homebrew-amd64', + 'ignition_common-ci-main-focal-amd64', + 'ignition_common-ci-main-homebrew-amd64', 'ignition_common4-install-pkg-focal-amd64', 'ignition_common4-install_bottle-homebrew-amd64', 'ignition_garden-ci-main-homebrew-amd64', @@ -332,16 +332,16 @@ ignition_collection_jobs = 'ignition_launch-ci-main-homebrew-amd64', 'ignition_launch5-install-pkg-focal-amd64', 'ignition_launch5-install_bottle-homebrew-amd64', - 'ignition_math-ci-ign-math6-focal-amd64', - 'ignition_math-ci-ign-math6-homebrew-amd64', + 'ignition_math-ci-main-focal-amd64', + 'ignition_math-ci-main-homebrew-amd64', 'ignition_math6-install-pkg-focal-amd64', 'ignition_math6-install_bottle-homebrew-amd64', 'ignition_msgs-ci-main-focal-amd64', 'ignition_msgs-ci-main-homebrew-amd64', 'ignition_msgs8-install-pkg-focal-amd64', 'ignition_msgs8-install_bottle-homebrew-amd64', - 'ignition_physics-ci-ign-physics5-focal-amd64', - 'ignition_physics-ci-ign-physics5-homebrew-amd64', + 'ignition_physics-ci-main-focal-amd64', + 'ignition_physics-ci-main-homebrew-amd64', 'ignition_physics5-install-pkg-focal-amd64', 'ignition_physics5-install_bottle-homebrew-amd64', 'ignition_plugin-ci-ign-plugin1-focal-amd64', @@ -368,9 +368,9 @@ ignition_collection_jobs = 'ignition_utils-ci-ign-utils1-homebrew-amd64', 'ignition_utils-install-pkg-focal-amd64', 'ignition_utils1-install_bottle-homebrew-amd64', - 'sdformat-ci-sdformat12-focal-amd64', - 'sdformat-ci-sdformat12-homebrew-amd64', - 'sdformat-ci-sdformat12-windows7-amd64', + 'sdformat-ci-main-focal-amd64', + 'sdformat-ci-main-homebrew-amd64', + 'sdformat-ci-main-windows7-amd64', 'sdformat-install-sdformat12_pkg-focal-amd64' ], ] From 4d27e61c2f7ba82b721a3b7035e75c86e85c40e2 Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Tue, 8 Mar 2022 09:54:31 -0800 Subject: [PATCH 092/540] Revert "Don't use HOMEBREW_UPDATE_TO_TAG for now (#629)" (#635) This reverts commit 6370d9882faeb8615c9ff5590e33948e170b29dd. It is no longer needed since brew 3.3.13 has been released. Signed-off-by: Steve Peters --- jenkins-scripts/lib/_homebrew_base_setup.bash | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/jenkins-scripts/lib/_homebrew_base_setup.bash b/jenkins-scripts/lib/_homebrew_base_setup.bash index 09184e95b..35d2308fc 100644 --- a/jenkins-scripts/lib/_homebrew_base_setup.bash +++ b/jenkins-scripts/lib/_homebrew_base_setup.bash @@ -15,9 +15,7 @@ else fi git -C $(${BREW_BINARY} --repo) fsck -# don't use HOMEBREW_UPDATE_TO_TAG until brew 3.3.13 is released -# due to https://github.com/Homebrew/brew/issues/12788 -unset HOMEBREW_UPDATE_TO_TAG +export HOMEBREW_UPDATE_TO_TAG=1 ${BREW_BINARY} update # manually exclude a ruby warning that jenkins thinks is from clang # https://github.com/osrf/homebrew-simulation/issues/1343 From e316d8497c67d5342c6f595f8a3c48363fdccb5e Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Tue, 8 Mar 2022 14:25:06 -0800 Subject: [PATCH 093/540] Set HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK for now (#656) Several bottles are failing linkage checks since Homebrew/brew#12584 was merged, so disable these checks for now. Signed-off-by: Steve Peters --- jenkins-scripts/lib/_homebrew_base_setup.bash | 1 + 1 file changed, 1 insertion(+) diff --git a/jenkins-scripts/lib/_homebrew_base_setup.bash b/jenkins-scripts/lib/_homebrew_base_setup.bash index 35d2308fc..7f85d1c02 100644 --- a/jenkins-scripts/lib/_homebrew_base_setup.bash +++ b/jenkins-scripts/lib/_homebrew_base_setup.bash @@ -16,6 +16,7 @@ fi git -C $(${BREW_BINARY} --repo) fsck export HOMEBREW_UPDATE_TO_TAG=1 +export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 ${BREW_BINARY} update # manually exclude a ruby warning that jenkins thinks is from clang # https://github.com/osrf/homebrew-simulation/issues/1343 From 3fd04cf5a221d6edc089918801c81e9723de71f9 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 11 Mar 2022 15:46:40 +0100 Subject: [PATCH 094/540] Do not fail if linux jobs don't have Dockefile (#552) --- jenkins-scripts/dsl/_configs_/OSRFLinuxBase.groovy | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/jenkins-scripts/dsl/_configs_/OSRFLinuxBase.groovy b/jenkins-scripts/dsl/_configs_/OSRFLinuxBase.groovy index 552e64ebe..bdce34f57 100644 --- a/jenkins-scripts/dsl/_configs_/OSRFLinuxBase.groovy +++ b/jenkins-scripts/dsl/_configs_/OSRFLinuxBase.groovy @@ -23,6 +23,10 @@ class OSRFLinuxBase archiveArtifacts { pattern('Dockerfile') pattern('build.sh') + // there are no guarantees that a job using OSRFLinuxBase generate + // these files (i.e: testing job). Do not fail if they are not + // present + allowEmpty(true) } } } From a7a584b9ad4cbd1f68f1a557d646b2be4bfeac85 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 11 Mar 2022 15:47:00 +0100 Subject: [PATCH 095/540] Do not run autogenerated tests in autopktest (#658) --- jenkins-scripts/docker/lib/_gazebo_utils.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/docker/lib/_gazebo_utils.sh b/jenkins-scripts/docker/lib/_gazebo_utils.sh index 14f5405b7..b7cdc8631 100644 --- a/jenkins-scripts/docker/lib/_gazebo_utils.sh +++ b/jenkins-scripts/docker/lib/_gazebo_utils.sh @@ -42,7 +42,7 @@ if $RUN_AUTOPKGTEST; then echo '# BEGIN SECTION: run autopkgtest' cd $WORKSPACE/pkgs set +e -sudo autopkgtest -B *.deb *.dsc -- null +sudo autopkgtest --no-auto-control -B *.deb *.dsc -- null # autopkgtest will return 0 if there are successful tests and 8 if there are no tests testret=\$? if [[ \$testret != 0 ]] && [[ \$testret != 8 ]]; then From 83e07d98bea908d7cb6c19d891de3fac8b825100 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 11 Mar 2022 21:17:07 +0100 Subject: [PATCH 096/540] Update vcpkg-snapshot to 2022.02.23 (#662) * Update vcpkg snapshot to 2022.02.23 * Fix problem with multiple tags on the same commit * Add patch for zeromq hash broken --- jenkins-scripts/lib/windows_env_vars.bat | 2 +- jenkins-scripts/lib/windows_library.bat | 9 ++++++--- jenkins-scripts/vcpkg-bootstrap.bat | 4 ++++ 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/jenkins-scripts/lib/windows_env_vars.bat b/jenkins-scripts/lib/windows_env_vars.bat index 011542765..5d8b1f1a2 100644 --- a/jenkins-scripts/lib/windows_env_vars.bat +++ b/jenkins-scripts/lib/windows_env_vars.bat @@ -15,6 +15,6 @@ set VCPKG_CMD=%VCPKG_DIR%\vcpkg.exe set VCPKG_CMAKE_TOOLCHAIN_FILE=%VCPKG_DIR%/scripts/buildsystems/vcpkg.cmake if NOT DEFINED VCPKG_SNAPSHOT ( :: see https://github.com/microsoft/vcpkg/releases - set VCPKG_SNAPSHOT=2021.05.12 + set VCPKG_SNAPSHOT=2022.02.23 ) goto :EOF diff --git a/jenkins-scripts/lib/windows_library.bat b/jenkins-scripts/lib/windows_library.bat index dba048bec..2a289e570 100644 --- a/jenkins-scripts/lib/windows_library.bat +++ b/jenkins-scripts/lib/windows_library.bat @@ -244,10 +244,13 @@ goto :EOF :: ################################## :check_vcpkg_snapshot setlocal EnableDelayedExpansion -for /f %%i in ('git -C %VCPKG_DIR% describe --tags HEAD') do set VCPKG_HEAD=%%i -echo "VCPKG_HEAD is %VCPKG_HEAD%" -if NOT %VCPKG_HEAD% == %VCPKG_SNAPSHOT% ( +:: look for same sha in repository HEAD and in the tag +for /f %%i in ('git -C %VCPKG_DIR% rev-parse HEAD') do set VCPKG_HEAD=%%i +for /f %%i in ('git -C %VCPKG_DIR% rev-list -n 1 %VCPKG_SNAPSHOT%') do set VCPKG_TAG=%%i +if NOT %VCPKG_HEAD% == %VCPKG_TAG% ( echo The vpckg directory is not using the expected snapshot %VCPKG_SNAPSHOT% + echo VCPKG_HEAD points to %VCPKG_HEAD% while VCPKG_TAG points to %VCPKG_TAG% + echo They should point to the same commit hash goto :error ) goto :EOF diff --git a/jenkins-scripts/vcpkg-bootstrap.bat b/jenkins-scripts/vcpkg-bootstrap.bat index 441b619f7..4f6992032 100644 --- a/jenkins-scripts/vcpkg-bootstrap.bat +++ b/jenkins-scripts/vcpkg-bootstrap.bat @@ -38,6 +38,10 @@ move %dfcln_portfile%.new %dfcln_portfile% || goto :error :: replace it to lowercase set ogre_portfile=%VCPKG_DIR%\ports\ogre\portfile.cmake powershell -Command "(Get-Content %ogre_portfile%) -replace 'Release', 'release' | Out-File -encoding ASCII %ogre_portfile%" || goto :error +:: 4. Fix for bug in hash in zeromq +:: https://github.com/microsoft/vcpkg/issues/23461 +set zeromq_portfile=%VCPKG_DIR%\ports\zeromq\portfile.cmake +powershell -Command "(Get-Content %zeromq_portfile%) -replace '64e6d37ab843e5b9aa9e56ba7904423ce0a2c6b4101dbd86b7b8b22c52c384ed7ea9764f9e0a53be04e7ade09923ca95452104e9760b66ebc0ed3ffef08a75c5', '42663c9b16a09a5c30d61a027c544ea318a9f745129579dcc0d5dd2d529be42e8dbaee1b9406497c4da7815fa60fc877d2e26f807135b2bbc0ea7ea4214b8af6' | Out-File -encoding ASCII %zeromq_portfile%" || goto :error echo # END SECTION echo "Using SNAPSHOT: bootstrap vcpkg executable" From 23c0691824b2fea19dfe527c0541496c98ccb182 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Mon, 14 Mar 2022 20:56:56 +0100 Subject: [PATCH 097/540] Do not use two ignition-cmake packages in ign-math (#665) --- jenkins-scripts/lib/dependencies_archive.sh | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/jenkins-scripts/lib/dependencies_archive.sh b/jenkins-scripts/lib/dependencies_archive.sh index 5f2cf2095..e306d19dc 100644 --- a/jenkins-scripts/lib/dependencies_archive.sh +++ b/jenkins-scripts/lib/dependencies_archive.sh @@ -384,12 +384,19 @@ fi # IGNITION # -IGN_MATH_DEPENDENCIES="libeigen3-dev \\ - libpython3-dev \\ - ruby-dev \\ - swig \\ - libignition-cmake-dev \\ - libignition-cmake2-dev" +if [[ ${IGN_MATH_MAJOR_VERSION} -eq 4 ]]; then + IGN_MATH_DEPENDENCIES="libeigen3-dev \\ + libpython3-dev \\ + ruby-dev \\ + swig \\ + libignition-cmake-dev" +else + IGN_MATH_DEPENDENCIES="libeigen3-dev \\ + libpython3-dev \\ + ruby-dev \\ + swig \\ + libignition-cmake2-dev" +fi # IGN_TRANSPORT related dependencies. Default value points to the development # version From 483ed581db584ea585a535c07e714bbb5b37233e Mon Sep 17 00:00:00 2001 From: Louise Poubel Date: Tue, 15 Mar 2022 15:57:02 -0700 Subject: [PATCH 098/540] Fix SDF links in unreleased table (#664) Signed-off-by: Louise Poubel --- terminal-dashboard/unreleased.bash | 1 + 1 file changed, 1 insertion(+) diff --git a/terminal-dashboard/unreleased.bash b/terminal-dashboard/unreleased.bash index 0e6e4488f..5c467b5a0 100755 --- a/terminal-dashboard/unreleased.bash +++ b/terminal-dashboard/unreleased.bash @@ -29,6 +29,7 @@ PACKAGE_REPO=${2:-stable} for LIB in $(get_libraries_by_collection "${COLLECTION}" ); do LIB_SHORT="${LIB/ignition-/ign-}" LIB_NAME=${LIB_SHORT//[[:digit:]]/} + LIB_SHORT="${LIB/sdformat/sdf}" PKG_VERSION=$(wget -qO- http://packages.osrfoundation.org/gazebo/ubuntu-${PACKAGE_REPO}/dists/focal/main/binary-amd64/Packages | \ grep -1 -m 1 -e "Source: ${LIB}" -e "Package: ${LIB}" | \ From c157a693cb67375d4f57d097ef23404d7ae64cfd Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 16 Mar 2022 23:08:23 +0100 Subject: [PATCH 099/540] Remove docker-jammy jenkins label, not needed anymore (#663) --- release.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/release.py b/release.py index 21b3bc7a2..905d6f552 100755 --- a/release.py +++ b/release.py @@ -617,8 +617,6 @@ def go(argv): # Need to use JENKINS_NODE_TAG parameter for large memory nodes # since it runs qemu emulation linux_platform_params['JENKINS_NODE_TAG'] = 'linux-' + a - elif(d == 'jammy'): - linux_platform_params['JENKINS_NODE_TAG'] = 'docker-jammy' elif ('ignition-physics' in args.package_alias): linux_platform_params['JENKINS_NODE_TAG'] = 'large-memory' From 2d9b194758c6fd7be01b9556f23114ca90e5c506 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 17 Mar 2022 00:41:09 +0100 Subject: [PATCH 100/540] Relocate gzdev 2nd run after cache invalidation (#667) --- .../docker/lib/docker_generate_dockerfile.bash | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash b/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash index ed5c00e35..4673a868d 100644 --- a/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash +++ b/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash @@ -279,11 +279,6 @@ cat >> Dockerfile << DELIM_DOCKER3 RUN echo "${MONTH_YEAR_STR}" DELIM_DOCKER3 -# A new install of gzdev is needed to update to possible recent changes in -# configuration and/or code and not being used since the docker cache did -# not get them. -dockerfile_install_gzdev_repos - cat >> Dockerfile << DELIM_DOCKER3_2 RUN sed -i -e 's:13\.56\.139\.45:packages.osrfoundation.org:g' /etc/apt/sources.list.d/* || true \ && (apt-get update || (rm -rf /var/lib/apt/lists/* && apt-get ${APT_PARAMS} update)) \ @@ -295,6 +290,11 @@ RUN sed -i -e 's:13\.56\.139\.45:packages.osrfoundation.org:g' /etc/apt/sources. RUN echo "Invalidating cache $(( ( RANDOM % 100000 ) + 1 ))" DELIM_DOCKER3_2 +# A new install of gzdev is needed to update to possible recent changes in +# configuration and/or code and not being used since the docker cache did +# not get them. +dockerfile_install_gzdev_repos + cat >> Dockerfile << DELIM_DOCKER31 # Note that we don't remove the apt/lists file here since it will make # to run apt-get update again From 0e4c04219a5d04c1946b60207011d3000c89627e Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Wed, 16 Mar 2022 16:47:00 -0700 Subject: [PATCH 101/540] Revert "Set HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK for now (#656)" (#660) This reverts commit e316d8497c67d5342c6f595f8a3c48363fdccb5e. This is no longer needed since Homebrew/brew#12979 was merged. Signed-off-by: Steve Peters --- jenkins-scripts/lib/_homebrew_base_setup.bash | 1 - 1 file changed, 1 deletion(-) diff --git a/jenkins-scripts/lib/_homebrew_base_setup.bash b/jenkins-scripts/lib/_homebrew_base_setup.bash index 7f85d1c02..35d2308fc 100644 --- a/jenkins-scripts/lib/_homebrew_base_setup.bash +++ b/jenkins-scripts/lib/_homebrew_base_setup.bash @@ -16,7 +16,6 @@ fi git -C $(${BREW_BINARY} --repo) fsck export HOMEBREW_UPDATE_TO_TAG=1 -export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 ${BREW_BINARY} update # manually exclude a ruby warning that jenkins thinks is from clang # https://github.com/osrf/homebrew-simulation/issues/1343 From 937e236876e7973fd834a60fc9a569f55c2ebd7e Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Mon, 21 Mar 2022 10:26:31 -0700 Subject: [PATCH 102/540] Skip brew doctor if szip is installed (#672) szip is currently deprecated in homebrew-core, so skip brew doctor checks until it is fixed. Replaces similar logic for proj@7 since that is now fixed. Signed-off-by: Steve Peters --- .../lib/project-default-devel-homebrew-amd64.bash | 8 ++++---- jenkins-scripts/lib/project-install-homebrew.bash | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash b/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash index 64fbce01b..f080fe263 100644 --- a/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash +++ b/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash @@ -176,10 +176,10 @@ echo '# END SECTION' echo "#BEGIN SECTION: brew doctor analysis" brew missing || brew install $(brew missing | awk '{print $2}') && brew missing -# if proj@7 installed, skip brew doctor -# remove this line when gdal stops depending on a deprecated version of proj -# https://github.com/Homebrew/homebrew-core/issues/82441 -brew list | grep '^proj@7$' || brew doctor || echo MARK_AS_UNSTABLE +# if szip is installed, skip brew doctor +# remove this line when hdf5 stops depending on the deprecated szip formula +# https://github.com/Homebrew/homebrew-core/issues/96930 +brew list | grep '^szip$' || brew doctor || echo MARK_AS_UNSTABLE echo '# END SECTION' # CHECK PRE_TESTS_EXECUTION_HOOK AND RUN diff --git a/jenkins-scripts/lib/project-install-homebrew.bash b/jenkins-scripts/lib/project-install-homebrew.bash index e911e3288..39dc5367c 100644 --- a/jenkins-scripts/lib/project-install-homebrew.bash +++ b/jenkins-scripts/lib/project-install-homebrew.bash @@ -73,10 +73,10 @@ echo '# END SECTION' echo "#BEGIN SECTION: brew doctor analysis" brew missing || brew install $(brew missing | awk '{print $2}') && brew missing -# if proj@7 installed, skip brew doctor -# remove this line when gdal stops depending on a deprecated version of proj -# https://github.com/Homebrew/homebrew-core/issues/82441 -brew list | grep '^proj@7$' || brew doctor || echo MARK_AS_UNSTABLE +# if szip is installed, skip brew doctor +# remove this line when hdf5 stops depending on the deprecated szip formula +# https://github.com/Homebrew/homebrew-core/issues/96930 +brew list | grep '^szip$' || brew doctor || echo MARK_AS_UNSTABLE echo '# END SECTION' echo "# BEGIN SECTION: re-add group write permissions" From 2126d8c14ae477636dd5892ec7c86008277a933f Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 23 Mar 2022 19:03:46 +0100 Subject: [PATCH 103/540] Remove old simbody script (#673) --- .../docker/lib/simbody-release-base.bash | 134 ------------------ 1 file changed, 134 deletions(-) delete mode 100644 jenkins-scripts/docker/lib/simbody-release-base.bash diff --git a/jenkins-scripts/docker/lib/simbody-release-base.bash b/jenkins-scripts/docker/lib/simbody-release-base.bash deleted file mode 100644 index f2da20d5f..000000000 --- a/jenkins-scripts/docker/lib/simbody-release-base.bash +++ /dev/null @@ -1,134 +0,0 @@ -#!/bin/bash -x - - -DOCKER_JOB_NAME="simbody_debbuild" -. ${SCRIPT_DIR}/lib/boilerplate_prepare.sh - -cat > build.sh << DELIM -################################################### -# Make project-specific changes here -# -#!/usr/bin/env bash -set -ex - -echo '# BEGIN SECTION: prepare the workspace' -# Step 0: Clean up -rm -rf $WORKSPACE/build -mkdir -p $WORKSPACE/build -cd $WORKSPACE/build - -# Clean from workspace all package related files -rm -fr $WORKSPACE/"$PACKAGE"_* -echo '# END SECTION' - -echo '# BEGIN SECTION: get simbody source ${VERSION}' -# Step 1: Get the source (nightly builds or tarball) -rm -fr $WORKSPACE/simbody -git clone https://github.com/simbody/simbody.git $WORKSPACE/simbody -cd $WORKSPACE/simbody -git checkout Simbody-${VERSION} -echo '# END SECTION' - -echo '# BEGIN SECTION: modify debian metadata' -# Use current distro -sed -i -e 's:precise:$DISTRO:g' debian/changelog -# Use current release version -sed -i -e 's:-1~:-$RELEASE_VERSION~:' debian/changelog -# Bug in saucy doxygen makes the job hangs -if [ $DISTRO = 'saucy' ]; then - sed -i -e '/.*dh_auto_build.*/d' debian/rules -fi -echo '# END SECTION' - -echo '# BEGIN SECTION: generate the .orig file' -# Step 5: use debuild to create source package -echo | dh_make -s --createorig -p ${PACKAGE}_${VERSION} || true -echo '# END SECTION' - -echo '# BEGIN SECTION: install build dependencies' -# Build dependencies -mk-build-deps -i debian/control --tool 'sudo apt-get --yes' -rm *build-deps*.deb -echo '# END SECTION' - -echo '# BEGIN SECTION: generate source pacakge' -# Step 5: use debuild to create source package -#TODO: create non-passphrase-protected keys and remove the -uc and -us args to debuild -debuild --no-tgz-check -S -uc -us --source-option=--include-binaries -j${MAKE_JOBS} -echo '# END SECTION' - -echo '# BEGIN SECTION: generate binaries' -# Step 6: use pbuilder-dist to create binary package(s) -debuild --no-tgz-check -uc -us --source-option=--include-binaries -j${MAKE_JOBS} -echo '# END SECTION' - -echo '# BEGIN SECTION: export packages' -mkdir -p $WORKSPACE/pkgs -rm -fr $WORKSPACE/pkgs/* - -PKGS=\`find .. -name '*.deb' || true\` - -FOUND_PKG=0 -for pkg in \${PKGS}; do - echo "found \$pkg" - # Check for correctly generated packages size > 3Kb - test -z \$(find \$pkg -size +3k) && echo "WARNING: empty package?" - # && exit 1 - cp \${pkg} $WORKSPACE/pkgs - FOUND_PKG=1 -done -# check at least one upload -test \$FOUND_PKG -eq 1 || exit 1 -echo '# END SECTION' -DELIM - -# -# Make project-specific changes here -################################################### - -cat > Dockerfile << DELIM_DOCKER -####################################################### -# Docker file to run build.sh - -FROM osrf/ubuntu_armhf:${DISTRO} -MAINTAINER Jose Luis Rivero - -# If host is running squid-deb-proxy on port 8000, populate /etc/apt/apt.conf.d/30proxy -# By default, squid-deb-proxy 403s unknown sources, so apt shouldn't proxy ppa.launchpad.net -RUN route -n | awk '/^0.0.0.0/ {print \$2}' > /tmp/host_ip.txt -RUN echo "HEAD /" | nc \$(cat /tmp/host_ip.txt) 8000 | grep squid-deb-proxy \ - && (echo "Acquire::http::Proxy \"http://\$(cat /tmp/host_ip.txt):8000\";" > /etc/apt/apt.conf.d/30proxy) \ - && (echo "Acquire::http::Proxy::ppa.launchpad.net DIRECT;" >> /etc/apt/apt.conf.d/30proxy) \ - || echo "No squid-deb-proxy detected on docker host" - -# Map the workspace into the container -RUN mkdir -p ${WORKSPACE} -# automatic invalidation of the cache if day is different -RUN echo "${TODAY_STR}" -RUN sudo apt-get update -RUN sudo apt-get install -y fakeroot debootstrap devscripts equivs dh-make ubuntu-dev-tools debhelper wget pkg-kde-tools bash-completion git -ADD build.sh build.sh -RUN chmod +x build.sh -DELIM_DOCKER - -sudo rm -fr ${WORKSPACE}/pkgs -sudo mkdir -p ${WORKSPACE}/pkgs - -if [[ $ARCH == armhf ]]; then - sudo docker pull osrf/ubuntu_armhf - sudo docker build -t ${DOCKER_TAG} . -else - echo "Architecture still unsupported" - exit 1 -fi - -sudo docker run \ - --cidfile=${CIDFILE} \ - -v ${WORKSPACE}/pkgs:${WORKSPACE}/pkgs \ - -t ${DOCKER_TAG} \ - /bin/bash build.sh - -CID=$(cat ${CIDFILE}) - -sudo docker stop ${CID} || true -sudo docker rm ${CID} || true From 1d86b6027bf5107d65f69b0dd9edbbf0c336ac28 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 23 Mar 2022 19:04:10 +0100 Subject: [PATCH 104/540] Add documentation about buildfarm nodes (node labels) (#661) * Add doc about buildfarm nodes (node tags) * Use label not tag * Added nightly labels --- docs/BUILDFARM_NODES.md | 42 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 docs/BUILDFARM_NODES.md diff --git a/docs/BUILDFARM_NODES.md b/docs/BUILDFARM_NODES.md new file mode 100644 index 000000000..908a623f9 --- /dev/null +++ b/docs/BUILDFARM_NODES.md @@ -0,0 +1,42 @@ +# Information on build.osrfoundation.org nodes + +## Jenkins node labels + +Jenkins labels are mainly created from chef provisioning of the nodes and mainly +used automatically from DSL code when it generates Jenkins jobs. The buildfarm +probably has more labels than the ones in the table, they are probably not being +used (can check this in the Jenkins UI) + +| Label name | Description | Requirements | +| -------- | ----------- | ------------ | +| docker | Node has capabilities to run Docker CI (standard Linux CI) | Linux system with docker installed | +| gpu-reliable | Node has a real GPU able to run simulation for Gazebo | Nvidia card and nvidia-docker installed on Linux | +| huge-memory | Node has enough RAM to run really demanding RAM compilations | Hardware has no less than 16Gb of RAM and can run abichecker on ign-physics | +| large-memory | Node has enough RAM to run non trivial compilations | Hardware has no less than 16GB of RAM on Linux | +| linux-arm64 | Node has capabilities to run native arm64 code (mostly used in packaging) | Bare-metal ARM machine | +| linux-armhf | Node has capabilities to run native armhf code (mostly used in packaging) | Bare-metal ARM machine | +| osx | Node has capabilities to run native OsX code | Apple system | +| osx_gazebo | Node has capabilities to run Gazebo classic CI and packaging | 'Powerful' Apple system | +| osx\_$distro | Node has capabilities to build code for the distribution $distro | Apple system running $distro | +| swarm | Node was created using swarm plugin in Jenkins | Chef provisioned node | +| win | Node is able to run Windows CI | Windows10 system | +| win_testing | Node is ready to test a new vcpkg snapshot | Windows10 system | + +### Node labels for nightlies + +For generating nightlies controlling the generation order of every library, build.osrfoundation.org use the approach of [using a single node to process the whole ignition family for a given Ubuntu distribution](https://github.com/ignition-tooling/release-tools/issues/644). Current assignment of nodes is as follow: + +| Label name | Description | +| -------- | ----------- | +| linux-nightly-bionic | r2d2 | +| linux-nightly-focal | optimus | +| linux-nightly-jammy | linux-ip-172-30-1-216 | + +## Provision of Node labels + +### Agents + +Chef provision of node labels is being done by code in the [osrf_jenkins_agent](https://github.com/osrf/osrf_jenkins_agent/) +repository. + +Note: check https://github.com/osrf/chef-osrf/issues/136 to see the status of the automation. From 80f3fae58af9b7c896b8ba800dc7724ab2ec9417 Mon Sep 17 00:00:00 2001 From: Louise Poubel Date: Fri, 25 Mar 2022 11:30:15 -0700 Subject: [PATCH 105/540] Fix unreleased script again (#675) Signed-off-by: Louise Poubel --- terminal-dashboard/unreleased.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terminal-dashboard/unreleased.bash b/terminal-dashboard/unreleased.bash index 5c467b5a0..92cc1f5ae 100755 --- a/terminal-dashboard/unreleased.bash +++ b/terminal-dashboard/unreleased.bash @@ -29,7 +29,7 @@ PACKAGE_REPO=${2:-stable} for LIB in $(get_libraries_by_collection "${COLLECTION}" ); do LIB_SHORT="${LIB/ignition-/ign-}" LIB_NAME=${LIB_SHORT//[[:digit:]]/} - LIB_SHORT="${LIB/sdformat/sdf}" + LIB_SHORT="${LIB_SHORT/sdformat/sdf}" PKG_VERSION=$(wget -qO- http://packages.osrfoundation.org/gazebo/ubuntu-${PACKAGE_REPO}/dists/focal/main/binary-amd64/Packages | \ grep -1 -m 1 -e "Source: ${LIB}" -e "Package: ${LIB}" | \ From d822be696f0b69dd79271c8d95193053af4cb6c1 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 30 Mar 2022 18:58:33 +0200 Subject: [PATCH 106/540] Avoid generating templates for packaging when using dh_make (#674) --- jenkins-scripts/docker/lib/debbuild-base.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/docker/lib/debbuild-base.bash b/jenkins-scripts/docker/lib/debbuild-base.bash index 7c95b391b..6486d6429 100644 --- a/jenkins-scripts/docker/lib/debbuild-base.bash +++ b/jenkins-scripts/docker/lib/debbuild-base.bash @@ -156,7 +156,7 @@ cd \`find $WORKSPACE/build -mindepth 1 -type d |head -n 1\` # If use the quilt 3.0 format for debian (drcsim) it needs a tar.gz with sources if $NIGHTLY_MODE; then rm -fr .hg* .git* - echo | dh_make -y -s --createorig -p${PACKAGE_ALIAS}_\${UPSTREAM_VERSION}+\${TIMESTAMP}+${RELEASE_VERSION}r\${REV} > /dev/null + echo | dh_make -y -s --createorig --defaultless -p${PACKAGE_ALIAS}_\${UPSTREAM_VERSION}+\${TIMESTAMP}+${RELEASE_VERSION}r\${REV} > /dev/null fi # Adding extra directories to code. debian has no problem but some extra directories From f05c146b0045db47fb1d4f6124f484883bfc212f Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 30 Mar 2022 19:07:30 +0200 Subject: [PATCH 107/540] Limit ign-physics building threads to 1 in arm native nodes (#668) --- jenkins-scripts/dsl/ignition.dsl | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index 8f43b23fd..e70d4cc21 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -307,12 +307,13 @@ ignition_software.each { ign_sw -> all_branches(ign_sw) - [ 'main']) abi_job.with { + extra_str="" if (ign_sw == 'physics') { label "huge-memory" - // on ARM native nodes in buildfarm we need to restrict to 2 the + // on ARM native nodes in buildfarm we need to restrict to 1 the // compilation threads to avoid OOM killer - GLOBAL_SHELL_CMD = GLOBAL_SHELL_CMD + '\nif [[ $(uname -m) == "aarch64" ]]; then export MAKE_JOBS=2; fi' + extra_str += '\nif [ $(uname -m) = "aarch64" ]; then export MAKE_JOBS=1; fi' } steps { @@ -322,6 +323,7 @@ ignition_software.each { ign_sw -> export DISTRO=${distro} ${GLOBAL_SHELL_CMD} + ${extra_str} export ARCH=${arch} export DEST_BRANCH=\${DEST_BRANCH:-\$ghprbTargetBranch} @@ -473,6 +475,11 @@ all_debbuilders().each { debbuilder_name -> if (debbuilder_name.contains("gazebo") || debbuilder_name == "transport7") extra_str="export NEED_C17_COMPILER=true" + // Ignition physics consumes huge amount of memory making arm node to FAIL + // Force here to use one compilation thread + if (debbuilder_name.contains("-physics")) + extra_str += '\nif [ $(uname -m) = "aarch64" ]; then export MAKE_JOBS=1; fi' + println("Generating: ${debbuilder_name}-debbuilder") def build_pkg_job = job("${debbuilder_name}-debbuilder") OSRFLinuxBuildPkg.create(build_pkg_job) From 4c590ae274841cfeb36fdb10c438f62e93ff1df5 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 31 Mar 2022 23:23:57 +0200 Subject: [PATCH 108/540] dh-make lower than 2.202003 is buggy using --defaultess (#681) * Workaround problems on old dh-make Signed-off-by: Jose Luis Rivero * De-deplicate code * Correct bash --- jenkins-scripts/docker/lib/debbuild-base.bash | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/jenkins-scripts/docker/lib/debbuild-base.bash b/jenkins-scripts/docker/lib/debbuild-base.bash index 6486d6429..366184644 100644 --- a/jenkins-scripts/docker/lib/debbuild-base.bash +++ b/jenkins-scripts/docker/lib/debbuild-base.bash @@ -156,7 +156,14 @@ cd \`find $WORKSPACE/build -mindepth 1 -type d |head -n 1\` # If use the quilt 3.0 format for debian (drcsim) it needs a tar.gz with sources if $NIGHTLY_MODE; then rm -fr .hg* .git* - echo | dh_make -y -s --createorig --defaultless -p${PACKAGE_ALIAS}_\${UPSTREAM_VERSION}+\${TIMESTAMP}+${RELEASE_VERSION}r\${REV} > /dev/null + # Versions of dh-make lower than 2.202003 are buggy using defaultless + # see: https://salsa.debian.org/debian/dh-make/-/merge_requests/8 + extra_dh_make_str='--defaultless' + if dpkg --compare-versions \$(apt-cache show dh-make | sed -n "s/Version: \\(.*\\)/\\1/p") lt 2.202003; then + extra_dh_make_str='' + fi + echo | dh_make -y -s --createorig \${extra_dh_make_str} -p${PACKAGE_ALIAS}_\${UPSTREAM_VERSION}+\${TIMESTAMP}+${RELEASE_VERSION}r\${REV} > /dev/null + rm -fr debian/ fi # Adding extra directories to code. debian has no problem but some extra directories From f141ade1ecb8030a443b41494bc54d56b85db75f Mon Sep 17 00:00:00 2001 From: Louise Poubel Date: Fri, 1 Apr 2022 11:01:06 -0700 Subject: [PATCH 109/540] =?UTF-8?q?=F0=9F=8F=81=20Edifice=20EOL=20(#676)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🏁 Edifice EOL Signed-off-by: Louise Poubel --- jenkins-scripts/dsl/ignition.dsl | 20 ++--- jenkins-scripts/dsl/ignition_collection.dsl | 86 --------------------- jenkins-scripts/dsl/sdformat.dsl | 4 +- jenkins-scripts/tools/buildcop_stats.bash | 2 +- terminal-dashboard/_dashboard_lib.sh | 19 ----- 5 files changed, 13 insertions(+), 118 deletions(-) diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index e70d4cc21..d7f875e61 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -45,18 +45,18 @@ ignition_no_test = [ ] // main, ABI check, install pkg) ignition_branches = [ 'cmake' : [ '2' ], 'common' : [ '1', '3', '4' ], - 'fuel-tools' : [ '1', '4', '6', '7' ], - 'gazebo' : [ '3', '5', '6' ], - 'gui' : [ '0', '3', '5', '6' ], - 'launch' : [ '2', '4', '5' ], + 'fuel-tools' : [ '1', '4', '7' ], + 'gazebo' : [ '3', '6' ], + 'gui' : [ '0', '3', '6' ], + 'launch' : [ '2', '5' ], 'math' : [ '4', '6' ], - 'msgs' : [ '1', '5', '7', '8' ], - 'physics' : [ '2', '4', '5' ], + 'msgs' : [ '1', '5', '8' ], + 'physics' : [ '2', '5' ], 'plugin' : [ '1' ], - 'rendering' : [ '3', '5', '6' ], - 'sensors' : [ '3', '5', '6' ], + 'rendering' : [ '3', '6' ], + 'sensors' : [ '3', '6' ], 'tools' : [ '1' ], - 'transport' : [ '4', '8', '10', '11' ], + 'transport' : [ '4', '8', '11' ], 'utils' : [ '1' ]] // DESC: prerelease branches are managed as any other supported branches for // special cases different to major branches: get compilation CI on the branch @@ -630,7 +630,7 @@ ignition_software.each { ign_sw -> // ign-gazebo only support Windows from ign-gazebo5 if (ign_sw == 'gazebo') - supported_branches = [ 'ign-gazebo5', 'ign-gazebo6', 'main' ] + supported_branches = [ 'ign-gazebo6', 'main' ] // ign-launch only support Windows from ign-launch5 if (ign_sw == 'launch') diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index e20efd496..e7492f21b 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -10,9 +10,6 @@ ignition_collections = [ [ name : 'citadel', distros : [ 'bionic' ], ], - [ name : 'edifice', - distros : [ 'focal' ], - ], [ name : 'fortress', distros : [ 'focal' ], ], @@ -122,89 +119,6 @@ ignition_collection_jobs = 'sdformat-ci-sdformat9-windows7-amd64', 'sdformat-install-sdformat9_pkg-focal-amd64' ], - 'edifice' : [ - 'ign_common-ign-4-win', - 'ign_fuel-tools-ign-6-win', - 'ign_gazebo-ign-5-win', - 'ign_gui-ign-5-win', - 'ign_math-ign-6-win', - 'ign_msgs-ign-7-win', - 'ign_physics-ign-4-win', - 'ign_plugin-ign-1-win', - 'ign_rendering-ign-5-win', - 'ign_sensors-ign-5-win', - 'ign_tools-ign-1-win', - 'ign_transport-ign-10-win', - 'ign_utils-ign-1-win', - 'ignition_cmake-ci-ign-cmake2-focal-amd64', - 'ignition_cmake-ci-ign-cmake2-homebrew-amd64', - 'ignition_cmake-ci-ign-cmake2-windows7-amd64', - 'ignition_cmake2-install-pkg-focal-amd64', - 'ignition_cmake2-install_bottle-homebrew-amd64', - 'ignition_common-ci-ign-common4-focal-amd64', - 'ignition_common-ci-ign-common4-homebrew-amd64', - 'ignition_common4-install-pkg-focal-amd64', - 'ignition_common4-install_bottle-homebrew-amd64', - 'ignition_edifice-ci-main-homebrew-amd64', - 'ignition_edifice-install-pkg-focal-amd64', - 'ignition_edifice-install_bottle-homebrew-amd64', - 'ignition_fuel-tools-ci-ign-fuel-tools6-focal-amd64', - 'ignition_fuel-tools-ci-ign-fuel-tools6-homebrew-amd64', - 'ignition_fuel-tools6-install-pkg-focal-amd64', - 'ignition_fuel-tools6-install_bottle-homebrew-amd64', - 'ignition_gazebo-ci-ign-gazebo5-focal-amd64', - 'ignition_gazebo-ci-ign-gazebo5-homebrew-amd64', - 'ignition_gazebo5-install-pkg-focal-amd64', - 'ignition_gazebo5-install_bottle-homebrew-amd64', - 'ignition_gui-ci-ign-gui5-focal-amd64', - 'ignition_gui-ci-ign-gui5-homebrew-amd64', - 'ignition_gui5-install-pkg-focal-amd64', - 'ignition_gui5-install_bottle-homebrew-amd64', - 'ignition_launch-ci-ign-launch4-focal-amd64', - 'ignition_launch-ci-ign-launch4-homebrew-amd64', - 'ignition_launch4-install-pkg-focal-amd64', - 'ignition_launch4-install_bottle-homebrew-amd64', - 'ignition_math-ci-ign-math6-focal-amd64', - 'ignition_math-ci-ign-math6-homebrew-amd64', - 'ignition_math6-install-pkg-focal-amd64', - 'ignition_math6-install_bottle-homebrew-amd64', - 'ignition_msgs-ci-ign-msgs7-focal-amd64', - 'ignition_msgs-ci-ign-msgs7-homebrew-amd64', - 'ignition_msgs7-install-pkg-focal-amd64', - 'ignition_msgs7-install_bottle-homebrew-amd64', - 'ignition_physics-ci-ign-physics4-focal-amd64', - 'ignition_physics-ci-ign-physics4-homebrew-amd64', - 'ignition_physics4-install-pkg-focal-amd64', - 'ignition_physics4-install_bottle-homebrew-amd64', - 'ignition_plugin-ci-ign-plugin1-focal-amd64', - 'ignition_plugin-ci-ign-plugin1-homebrew-amd64', - 'ignition_plugin-install-pkg-focal-amd64', - 'ignition_plugin1-install_bottle-homebrew-amd64', - 'ignition_rendering-ci-ign-rendering5-focal-amd64', - 'ignition_rendering-ci-ign-rendering5-homebrew-amd64', - 'ignition_rendering5-install-pkg-focal-amd64', - 'ignition_rendering5-install_bottle-homebrew-amd64', - 'ignition_sensors-ci-ign-sensors5-focal-amd64', - 'ignition_sensors-ci-ign-sensors5-homebrew-amd64', - 'ignition_sensors5-install-pkg-focal-amd64', - 'ignition_sensors5-install_bottle-homebrew-amd64', - 'ignition_tools-ci-ign-tools1-focal-amd64', - 'ignition_tools-ci-ign-tools1-homebrew-amd64', - 'ignition_tools-install-pkg-focal-amd64', - 'ignition_tools1-install_bottle-homebrew-amd64', - 'ignition_transport-ci-ign-transport10-focal-amd64', - 'ignition_transport-ci-ign-transport10-homebrew-amd64', - 'ignition_transport10-install-pkg-focal-amd64', - 'ignition_transport10-install_bottle-homebrew-amd64', - 'ignition_utils-ci-ign-utils1-focal-amd64', - 'ignition_utils-ci-ign-utils1-homebrew-amd64', - 'ignition_utils-install-pkg-focal-amd64', - 'ignition_utils1-install_bottle-homebrew-amd64', - 'sdformat-ci-sdformat11-focal-amd64', - 'sdformat-ci-sdformat11-homebrew-amd64', - 'sdformat-ci-sdformat11-windows7-amd64', - 'sdformat-install-sdformat11_pkg-focal-amd64' - ], 'fortress' : [ 'ign_common-ign-4-win', 'ign_fuel-tools-ign-7-win', diff --git a/jenkins-scripts/dsl/sdformat.dsl b/jenkins-scripts/dsl/sdformat.dsl index c87f04801..1d59b0d96 100644 --- a/jenkins-scripts/dsl/sdformat.dsl +++ b/jenkins-scripts/dsl/sdformat.dsl @@ -1,8 +1,8 @@ import _configs_.* import javaposse.jobdsl.dsl.Job -def sdformat_supported_branches = [ 'sdformat6' , 'sdformat9', 'sdformat11', 'sdformat12' ] -def sdformat_gz11_branches = [ 'sdformat9', 'sdformat10', 'sdformat11', 'sdformat12', 'main' ] +def sdformat_supported_branches = [ 'sdformat6' , 'sdformat9', 'sdformat12' ] +def sdformat_gz11_branches = [ 'sdformat9', 'sdformat10', 'sdformat12', 'main' ] // nightly and prereleases def extra_sdformat_debbuilder = [ 'sdformat13' ] diff --git a/jenkins-scripts/tools/buildcop_stats.bash b/jenkins-scripts/tools/buildcop_stats.bash index c2c03bbcc..88b3809f1 100755 --- a/jenkins-scripts/tools/buildcop_stats.bash +++ b/jenkins-scripts/tools/buildcop_stats.bash @@ -1,6 +1,6 @@ #!/bin/bash -views='ign-citadel ign-edifice ign-fortress ign-garden' +views='ign-citadel ign-fortress ign-garden' echo "# Build Cop Report $(date +%Y-%m-%d)" echo "## Aggregate Results as of $(date '+%Y-%m-%d %H:%M:%S')" diff --git a/terminal-dashboard/_dashboard_lib.sh b/terminal-dashboard/_dashboard_lib.sh index 2712e4c7f..c91479ada 100644 --- a/terminal-dashboard/_dashboard_lib.sh +++ b/terminal-dashboard/_dashboard_lib.sh @@ -24,25 +24,6 @@ get_libraries_by_collection() ignition-gazebo3 ignition-launch2 ignition-citadel" - elif [ "$COLLECTION" = "edifice" ]; then - LIBS=" - ignition-cmake2 - ignition-math6 - ignition-utils1 - ignition-tools - ignition-common4 - ignition-msgs7 - ignition-transport10 - ignition-fuel-tools6 - ignition-plugin - ignition-rendering5 - sdformat11 - ignition-physics4 - ignition-sensors5 - ignition-gui5 - ignition-gazebo5 - ignition-launch4 - ignition-edifice" elif [ "$COLLECTION" = "fortress" ]; then LIBS=" ignition-cmake2 From a5123f229deac94c3fcf50172926169437f2e728 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Steven!=20Ragnar=C3=B6k?= Date: Fri, 1 Apr 2022 11:01:51 -0700 Subject: [PATCH 110/540] Allow CI Workflow jobs to run concurrently. (#680) Make all compliation jobs run concurrently by default. We should be able to scale out any CI/PR jobs by default since they ought not be conflicting with the possible exception of per-host resources like GPUs but that should be handled per node since those conflicts can arise across different jobs as well as builds of the same job. --- jenkins-scripts/dsl/_configs_/GenericCompilation.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/jenkins-scripts/dsl/_configs_/GenericCompilation.groovy b/jenkins-scripts/dsl/_configs_/GenericCompilation.groovy index 182dbfa1a..8f2992a29 100644 --- a/jenkins-scripts/dsl/_configs_/GenericCompilation.groovy +++ b/jenkins-scripts/dsl/_configs_/GenericCompilation.groovy @@ -41,6 +41,7 @@ class GenericCompilation job.with { + concurrentBuild(true) properties { priority 300 } From 0f8688f932563d22dfc4b17044e7b5b515c48f2d Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Fri, 1 Apr 2022 16:19:17 -0700 Subject: [PATCH 111/540] brew: try to rescue if `brew list` fails (#679) * brew: try to rescue if `brew list` fails Sometimes `brew list` fails if the brew installation on a build machine has been corrupted. This will check for failures and invoke restore_brew if needed. Signed-off-by: Steve Peters --- jenkins-scripts/lib/_homebrew_cleanup.bash | 1 + 1 file changed, 1 insertion(+) diff --git a/jenkins-scripts/lib/_homebrew_cleanup.bash b/jenkins-scripts/lib/_homebrew_cleanup.bash index 824b44533..40d386055 100644 --- a/jenkins-scripts/lib/_homebrew_cleanup.bash +++ b/jenkins-scripts/lib/_homebrew_cleanup.bash @@ -15,6 +15,7 @@ export HOMEBREW_UPDATE_TO_TAG=1 ${BREW_BINARY} up || { restore_brew && ${BREW_BINARY} up ; } # Clear all installed homebrew packages, links, taps, and kegs +${BREW_BINARY} list --formula > /dev/null || { restore_brew && ${BREW_BINARY} list --formula > /dev/null; } BREW_LIST=$(${BREW_BINARY} list --formula) if [[ -n "${BREW_LIST}" ]]; then ${BREW_BINARY} remove --force --ignore-dependencies ${BREW_LIST} From 85cd39af2dc618fa2b91d9df6a4558fa6969241e Mon Sep 17 00:00:00 2001 From: Louise Poubel Date: Wed, 6 Apr 2022 08:24:44 -0700 Subject: [PATCH 112/540] Release table: add jammy and improve nightly (#683) Signed-off-by: Louise Poubel --- terminal-dashboard/table.bash | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/terminal-dashboard/table.bash b/terminal-dashboard/table.bash index 3950be29c..2fecd02a9 100755 --- a/terminal-dashboard/table.bash +++ b/terminal-dashboard/table.bash @@ -22,15 +22,19 @@ SCRIPT_DIR="${SCRIPT_DIR%/*}" COLLECTION=$1 PACKAGE_REPO=${2:-stable} -ARCHS=( "amd64" "i386" "arm64" "armhf") -DISTROS=( "ubuntu" "debian" ) COLUMN=" " - GREEN="\e[42m" YELLOW="\e[43m" RED="\e[101m" +ARCHS=( "amd64") +DISTROS=( "ubuntu" ) +if [[ $PACKAGE_REPO != "nightly" ]]; then + ARCHS+=( "i386" "arm64" "armhf") + DISTROS+=( "debian" ) +fi + for LIB in $(get_libraries_by_collection "${COLLECTION}" ); do echo -e "\e[107m\e[90m${LIB}\e[49m\e[39m" @@ -39,7 +43,13 @@ for LIB in $(get_libraries_by_collection "${COLLECTION}" ); do for DISTRO in "${DISTROS[@]}" do if [[ $DISTRO == "ubuntu" ]]; then - VERS=( "bionic" "focal" ) + if [[ $COLLECTION == "citadel" ]]; then + VERS=( "bionic" "focal" ) + elif [[ $COLLECTION == "fortress" ]]; then + VERS=( "bionic" "focal" "jammy" ) + elif [[ $COLLECTION == "garden" ]]; then + VERS=( "focal" "jammy" ) + fi else VERS=( "buster" ) # "sid" fi @@ -57,7 +67,7 @@ for LIB in $(get_libraries_by_collection "${COLLECTION}" ); do echo -n " " - if [[ $ARCH == "i386" && $VER == "focal" ]]; then + if [[ $ARCH == "i386" && $VER != "bionic" && $VER != "buster" ]]; then PKG_VERSION="disabled" else # The Source field is not mandatory and it is probably not present when From 90a7f6f7317e6ce8923d0704302fb90641f60ca9 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 6 Apr 2022 17:32:06 +0200 Subject: [PATCH 113/540] Check that debchange is installed in the system for changelog_spawn.sh (#684) * Check that debchange is installed in the system for changelog_spawn.sh * stop on errors for the script Signed-off-by: Jose Luis Rivero --- release-repo-scripts/changelog_spawn.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/release-repo-scripts/changelog_spawn.sh b/release-repo-scripts/changelog_spawn.sh index cded7e418..99faf1b98 100755 --- a/release-repo-scripts/changelog_spawn.sh +++ b/release-repo-scripts/changelog_spawn.sh @@ -20,6 +20,8 @@ # $ cd $software-release # $ ./changelog_spawn [msg] +set -e + version=${1} msg=${2} @@ -39,6 +41,12 @@ if [[ ${version%-*} == ${version} ]]; then exit 1 fi +if ! debchange --version > /dev/null 2>/dev/null; then + echo "debchange is not installed in the system." + echo "On Ubuntu/Debian systems please run: sudo apt-get install -y devscripts" + exit 1 +fi + changelog_example=$(find . -name changelog | head -n 1) if [[ -z ${changelog_example} ]]; then From 54d09aeadb420e6f9bbb7836fa929cda5dfd1788 Mon Sep 17 00:00:00 2001 From: methylDragon Date: Wed, 6 Apr 2022 16:48:16 -0700 Subject: [PATCH 114/540] Bumps in garden : ign-utils2 (#687) Signed-off-by: Brandon Ong --- jenkins-scripts/dsl/ignition_collection.dsl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index e7492f21b..bad29dc07 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -20,7 +20,7 @@ ignition_collections = [ nightly_jobs: [ 'tools' : [ debbuild: 'ign-tools' , branch: 'ign-tools1' ], 'cmake' : [ debbuild: 'ign-cmake2' , branch: 'ign-cmake2' ], - 'utils' : [ debbuild: 'ign-utils1' , branch: 'ign-utils1' ], + 'utils' : [ debbuild: 'ign-utils2' , branch: 'main' ], 'math' : [ debbuild: 'ign-math7' , branch: 'main' ], 'plugin' : [ debbuild: 'ign-plugin' , branch: 'ign-plugin1' ], 'common' : [ debbuild: 'ign-common5' , branch: 'main' ], From e81da1d4bbc68ef002abbf1fe14ae20f15eef06d Mon Sep 17 00:00:00 2001 From: Louise Poubel Date: Wed, 6 Apr 2022 18:15:31 -0700 Subject: [PATCH 115/540] Fix GZDEV_PROJECT_NAME for ign-utils2 (#688) * Fix GZDEV_PROJECT_NAME for ign-utils2 Signed-off-by: Louise Poubel * revert Signed-off-by: Louise Poubel --- jenkins-scripts/docker/ign_utils-compilation.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/docker/ign_utils-compilation.bash b/jenkins-scripts/docker/ign_utils-compilation.bash index 67e20c257..a91d40638 100644 --- a/jenkins-scripts/docker/ign_utils-compilation.bash +++ b/jenkins-scripts/docker/ign_utils-compilation.bash @@ -30,6 +30,6 @@ fi export NEED_C17_COMPILER=true -export GZDEV_PROJECT_NAME="ignition-utils${IGN_UTILS_DEPENDENCIES}" +export GZDEV_PROJECT_NAME="ignition-utils${IGN_UTILS_MAJOR_VERSION}" . ${SCRIPT_DIR}/lib/generic-building-base.bash From c87607779a83e38cee27690258b27ea63f761006 Mon Sep 17 00:00:00 2001 From: methylDragon Date: Thu, 7 Apr 2022 17:13:53 -0700 Subject: [PATCH 116/540] Bumps in garden : ign-cmake3 (#686) Signed-off-by: methylDragon Co-authored-by: Louise Poubel --- jenkins-scripts/dsl/ignition_collection.dsl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index bad29dc07..9d9190cfc 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -19,7 +19,7 @@ ignition_collections = [ // They're in topological order nightly_jobs: [ 'tools' : [ debbuild: 'ign-tools' , branch: 'ign-tools1' ], - 'cmake' : [ debbuild: 'ign-cmake2' , branch: 'ign-cmake2' ], + 'cmake' : [ debbuild: 'ign-cmake3' , branch: 'main' ], 'utils' : [ debbuild: 'ign-utils2' , branch: 'main' ], 'math' : [ debbuild: 'ign-math7' , branch: 'main' ], 'plugin' : [ debbuild: 'ign-plugin' , branch: 'ign-plugin1' ], From 05dbcba088e4d969d70bf2076a914ea32871f79f Mon Sep 17 00:00:00 2001 From: Louise Poubel Date: Thu, 7 Apr 2022 17:57:48 -0700 Subject: [PATCH 117/540] Fix GZDEV_PROJECT_NAME for ign-plugin2 (#692) Signed-off-by: Louise Poubel --- jenkins-scripts/docker/ign_plugin-compilation.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/docker/ign_plugin-compilation.bash b/jenkins-scripts/docker/ign_plugin-compilation.bash index 5e1f86b11..245e17ce3 100644 --- a/jenkins-scripts/docker/ign_plugin-compilation.bash +++ b/jenkins-scripts/docker/ign_plugin-compilation.bash @@ -32,6 +32,6 @@ if [[ ${IGN_PLUGIN_MAJOR_VERSION} -ge 6 ]]; then export NEED_C17_COMPILER=true fi -export GZDEV_PROJECT_NAME="ignition-plugin${IGN_PLUGIN_DEPENDENCIES}" +export GZDEV_PROJECT_NAME="ignition-plugin${IGN_PLUGIN_MAJOR_VERSION}" . ${SCRIPT_DIR}/lib/generic-building-base.bash From be1b52b5d3eb5ff47511907cca7a0639a6f69cd9 Mon Sep 17 00:00:00 2001 From: Louise Poubel Date: Fri, 8 Apr 2022 06:52:39 -0700 Subject: [PATCH 118/540] ign-tools doesn't depend on ign-cmake (#691) Signed-off-by: Louise Poubel --- jenkins-scripts/docker/ign_tools-compilation.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/docker/ign_tools-compilation.bash b/jenkins-scripts/docker/ign_tools-compilation.bash index 610eb96a5..af5196f02 100644 --- a/jenkins-scripts/docker/ign_tools-compilation.bash +++ b/jenkins-scripts/docker/ign_tools-compilation.bash @@ -16,6 +16,6 @@ fi export BUILDING_SOFTWARE_DIRECTORY="ign-tools" export BUILDING_JOB_REPOSITORIES="stable" -export BUILDING_DEPENDENCIES="libignition-cmake2-dev ruby" +export BUILDING_DEPENDENCIES="ruby" . ${SCRIPT_DIR}/lib/generic-building-base.bash From 10aa578bb4c9277ca92fb26238f5c54f353c9ec7 Mon Sep 17 00:00:00 2001 From: Louise Poubel Date: Fri, 8 Apr 2022 10:02:14 -0700 Subject: [PATCH 119/540] Update EOL template after using it for Dome and Edifice (#616) * Update EOL template after using it for Dome Signed-off-by: Louise Poubel * bullets Signed-off-by: Louise Poubel * remove bottles * bottles -> badges * updates since Edifice Signed-off-by: Louise Poubel * packages.ros Signed-off-by: Louise Poubel --- .github/ISSUE_TEMPLATE/eol_collection.md | 31 +++++++++++++++++++----- 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/eol_collection.md b/.github/ISSUE_TEMPLATE/eol_collection.md index 8f56649a7..26e7cc085 100644 --- a/.github/ISSUE_TEMPLATE/eol_collection.md +++ b/.github/ISSUE_TEMPLATE/eol_collection.md @@ -14,13 +14,32 @@ When opening PRs, add a link back to this issue for easier tracking. --> - [ ] Write a post to https://community.gazebosim.org/ announcing the community it has last X weeks for finishing their PRs. + * - [ ] Check library versions that can be EOL’d on the [versions table](https://github.com/ignitionrobotics/docs/blob/master/tools/versions.md) * -- [ ] Make releases with outstanding changes in each library about to be EOl'ed -- [ ] Archive release repos for all libraries after the last release has been made -- [ ] Update docs ([example](https://github.com/ignitionrobotics/docs/pull/124)) -- [ ] Update release-tools ([example](https://github.com/ignition-tooling/release-tools/pull/369)) -- [ ] Deprecate homebrew bottles ([example](https://github.com/osrf/homebrew-simulation/pull/1242)) -- [ ] Remove from buildfarmer ([dashboard](https://github.com/osrf/buildfarmer/pull/140) / [report](https://github.com/osrf/buildfarmer/pull/146)) +- [ ] Make releases with outstanding changes in each library about to be EOL'ed + * +- [ ] Archive release repos in [ignition-release](https://github.com/ignition-release) for all libraries after the last release has been made +- [ ] Update `release-tools` after all releases have been made ([example](https://github.com/ignition-tooling/release-tools/pull/369)) + * +- [ ] Deprecate homebrew bottles if they aren't deprecated yet (don't disable them) ([example](https://github.com/osrf/homebrew-simulation/pull/1785)) + * +- [ ] Remove badges from `homebrew-simulation` README ([example](https://github.com/osrf/homebrew-simulation/pull/1772)) + * +- [ ] Remove from buildfarmer ([example](https://github.com/osrf/homebrew-simulation/pull/1785)) + * - [ ] Remove from `pr-collection-labeler` ([example](https://github.com/ignition-tooling/pr-collection-labeler/pull/13)) + * +- [ ] Remove from `ros_ign` ([example](https://github.com/ignitionrobotics/ros_ign/pull/199)) + * +- [ ] Sync latest binaries to http://packages.ros.org ([example](https://github.com/ros-infrastructure/reprepro-updater/pull/145)) + * +- [ ] Remove from `gzdev`'s `ign_docker_env` ([example](https://github.com/ignition-tooling/gzdev/pull/56)) + * +- [ ] Close " support" ticket after final releases ([example](https://github.com/ignition-tooling/release-tools/issues/297#issuecomment-1002232980)) + * +- [ ] Update docs after EOL is complete ([example](https://github.com/ignitionrobotics/docs/pull/124)) + * +- [ ] Announce EOL on https://community.gazebosim.org/ ([example](https://community.gazebosim.org/t/ignition-blueprint-officially-end-of-life/764)) + * From 3de75f2a6258dbca377560526e636fa0fe66ba31 Mon Sep 17 00:00:00 2001 From: methylDragon Date: Fri, 8 Apr 2022 12:38:21 -0700 Subject: [PATCH 120/540] Bumps in garden : ign-plugin2 (#694) See #685 --- jenkins-scripts/dsl/ignition_collection.dsl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index 9d9190cfc..4ecb3fa8d 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -22,7 +22,7 @@ ignition_collections = [ 'cmake' : [ debbuild: 'ign-cmake3' , branch: 'main' ], 'utils' : [ debbuild: 'ign-utils2' , branch: 'main' ], 'math' : [ debbuild: 'ign-math7' , branch: 'main' ], - 'plugin' : [ debbuild: 'ign-plugin' , branch: 'ign-plugin1' ], + 'plugin' : [ debbuild: 'ign-plugin2' , branch: 'main' ], 'common' : [ debbuild: 'ign-common5' , branch: 'main' ], 'msgs' : [ debbuild: 'ign-msgs9' , branch: 'main' ], 'rendering' : [ debbuild: 'ign-rendering7' , branch: 'main' ], From 3d03e5d3f26dbb8e8c245a5b1b7b752ed6ec4395 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Sat, 9 Apr 2022 02:20:27 +0200 Subject: [PATCH 121/540] Restrict i386 on release.py to Bionic on Ubuntu (#693) Signed-off-by: Jose Luis Rivero --- release.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release.py b/release.py index 905d6f552..a6ed55006 100755 --- a/release.py +++ b/release.py @@ -600,8 +600,8 @@ def go(argv): if (PRERELEASE or NIGHTLY): if (a == 'armhf' or a == 'arm64'): continue - # No i386 for focal - if (d == 'focal' and a == 'i386'): + # Only i386 for Ubuntu in Bionic + if (a == 'i386' and l != 'debian' and d != 'bionic'): continue linux_platform_params = params.copy() From 88436da43e3db319d1f697fc8ef56d0793676edb Mon Sep 17 00:00:00 2001 From: methylDragon Date: Mon, 11 Apr 2022 17:07:53 -0700 Subject: [PATCH 122/540] Fix some bump_dependency edgecases (#699) * Add comments to standard src case * Handle IGN__VER case * Remove version numbers Adapted from: https://github.com/ignition-tooling/release-tools/commit/b2990ee2af7c5aa63d661fa41a386c69d71b356b * Handle VERSION X.X.X. case Adapted/fixed from: https://github.com/ignition-tooling/release-tools/commit/b2990ee2af7c5aa63d661fa41a386c69d71b356b Preserves other args * Increase scope of find_package matching --- release-repo-scripts/bump_dependency.bash | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/release-repo-scripts/bump_dependency.bash b/release-repo-scripts/bump_dependency.bash index c7b624924..7a37ffb77 100755 --- a/release-repo-scripts/bump_dependency.bash +++ b/release-repo-scripts/bump_dependency.bash @@ -455,8 +455,29 @@ for ((i = 0; i < "${#LIBRARIES[@]}"; i++)); do DEP_VER=${VERSIONS[$j]} DEP_PREV_VER="$((${DEP_VER}-1))" + # Rule: *plugin2 -> *plugin3 + # Replace lines like: "find_package(ignition-cmake2)" + # with: "find_package(ignition-cmake3)" find . -type f ! -name 'Changelog.md' ! -name 'Migration.md' -print0 | xargs -0 sed -i "s ${DEP_LIB}${DEP_PREV_VER} ${DEP_LIB}${DEP_VER} g" + # Rule: IGN_PLUGIN_VER 2 -> IGN_PLUGIN_VER 3 + # Replace lines like: "set(IGN_PLUGIN_VER 2)" + # with: "set(IGN_PLUGIN_VER 3)" + find . -type f ! -name 'Changelog.md' ! -name 'Migration.md' -print0 | xargs -0 sed -i "s@IGN_${DEP_LIB}_VER ${DEP_PREV_VER}@\UIGN_${DEP_LIB}_VER ${DEP_VER}@ig" + + # Replace lines like "find_package(ignition-cmake2 2.0.0)" + # with "find_package(ignition-cmake3)" + find . -type f -name 'CMakeLists.txt' -print0 | xargs -0 sed -i "s@\(find_package.*${DEP_LIB}\)${DEP_PREV_VER} \+${DEP_PREV_VER}[^ )]*@\1${DEP_VER}@g" + + # Replace lines like "ign_find_package(ignition-math6 VERSION 6.5.0)" + # with "ign_find_package(ignition-math7)" + # Preserves other args and handles edge cases: + # like "ign_find_package(ignition-math6 VERSION 6.5.0 REQUIRED)" + # with "ign_find_package(ignition-math6 REQUIRED)" + # like "ign_find_package(ignition-math6 REQUIRED COMPONENTS VERSION 6.10 eigen3)" + # with "ign_find_package(ignition-math7 REQUIRED COMPONENTS eigen3)" + find . -type f -name 'CMakeLists.txt' -print0 | xargs -0 sed -i "s@\(find_package.*${DEP_LIB}\)${DEP_PREV_VER}\(.*\) \+VERSION \+${DEP_PREV_VER}[^ )]*@\1${DEP_VER}\2@g" + # Replace collection yaml branch names with main if [[ "${LIB}" == "ign-${COLLECTION}" ]]; then find . -type f -name "collection-${COLLECTION}.yaml" -print0 | xargs -0 sed -i "s ign-${DEP_LIB}${DEP_VER} main g" @@ -490,4 +511,3 @@ for ((i = 0; i < "${#LIBRARIES[@]}"; i++)); do commitAndPR ${TOOLING_ORG} master done - From 2dc1c0e83b198cb858a750025fe12d1f5ff8b290 Mon Sep 17 00:00:00 2001 From: methylDragon Date: Tue, 12 Apr 2022 11:39:47 -0700 Subject: [PATCH 123/540] Fix ordering and remove IGN__VER pattern (#700) * Fix replacement ordering * Implement IGN__VER fix * Remove IGN__VER case instead This pattern requires that the library is found before it is invoked. And in most cases, it'll be invoked first. --- release-repo-scripts/bump_dependency.bash | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/release-repo-scripts/bump_dependency.bash b/release-repo-scripts/bump_dependency.bash index 7a37ffb77..84a7a562d 100755 --- a/release-repo-scripts/bump_dependency.bash +++ b/release-repo-scripts/bump_dependency.bash @@ -455,16 +455,6 @@ for ((i = 0; i < "${#LIBRARIES[@]}"; i++)); do DEP_VER=${VERSIONS[$j]} DEP_PREV_VER="$((${DEP_VER}-1))" - # Rule: *plugin2 -> *plugin3 - # Replace lines like: "find_package(ignition-cmake2)" - # with: "find_package(ignition-cmake3)" - find . -type f ! -name 'Changelog.md' ! -name 'Migration.md' -print0 | xargs -0 sed -i "s ${DEP_LIB}${DEP_PREV_VER} ${DEP_LIB}${DEP_VER} g" - - # Rule: IGN_PLUGIN_VER 2 -> IGN_PLUGIN_VER 3 - # Replace lines like: "set(IGN_PLUGIN_VER 2)" - # with: "set(IGN_PLUGIN_VER 3)" - find . -type f ! -name 'Changelog.md' ! -name 'Migration.md' -print0 | xargs -0 sed -i "s@IGN_${DEP_LIB}_VER ${DEP_PREV_VER}@\UIGN_${DEP_LIB}_VER ${DEP_VER}@ig" - # Replace lines like "find_package(ignition-cmake2 2.0.0)" # with "find_package(ignition-cmake3)" find . -type f -name 'CMakeLists.txt' -print0 | xargs -0 sed -i "s@\(find_package.*${DEP_LIB}\)${DEP_PREV_VER} \+${DEP_PREV_VER}[^ )]*@\1${DEP_VER}@g" @@ -478,6 +468,12 @@ for ((i = 0; i < "${#LIBRARIES[@]}"; i++)); do # with "ign_find_package(ignition-math7 REQUIRED COMPONENTS eigen3)" find . -type f -name 'CMakeLists.txt' -print0 | xargs -0 sed -i "s@\(find_package.*${DEP_LIB}\)${DEP_PREV_VER}\(.*\) \+VERSION \+${DEP_PREV_VER}[^ )]*@\1${DEP_VER}\2@g" + + # Rule: *plugin2 -> *plugin3 + # Replace lines like: "find_package(ignition-cmake2)" + # with: "find_package(ignition-cmake3)" + find . -type f ! -name 'Changelog.md' ! -name 'Migration.md' -print0 | xargs -0 sed -i "s ${DEP_LIB}${DEP_PREV_VER} ${DEP_LIB}${DEP_VER} g" + # Replace collection yaml branch names with main if [[ "${LIB}" == "ign-${COLLECTION}" ]]; then find . -type f -name "collection-${COLLECTION}.yaml" -print0 | xargs -0 sed -i "s ign-${DEP_LIB}${DEP_VER} main g" From 2eb271224570c48bc0185b71d86dcc1a220b43b1 Mon Sep 17 00:00:00 2001 From: Louise Poubel Date: Wed, 13 Apr 2022 16:39:09 -0700 Subject: [PATCH 124/540] Perform deprecation tocks when a main branch is created (#701) --- .github/ISSUE_TEMPLATE/release_collection.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/ISSUE_TEMPLATE/release_collection.md b/.github/ISSUE_TEMPLATE/release_collection.md index 420e95b6d..8b5fea99f 100644 --- a/.github/ISSUE_TEMPLATE/release_collection.md +++ b/.github/ISSUE_TEMPLATE/release_collection.md @@ -75,6 +75,8 @@ When opening PRs, add a link back to this issue for easier tracking. * - [ ] Create new `-release` repositories (use [this script](https://github.com/ignition-tooling/release-tools/blob/master/release-repo-scripts/new_ignition_release_repos.bash)) - [ ] Enable nightlies for all `main` branches on `gzdev` ([example](https://github.com/ignition-tooling/gzdev/pull/50)) + - [ ] Execute the tick-tock's "tock" for deprecations ([example](https://github.com/ignitionrobotics/ign-gazebo/pull/875)) + * - [ ] Update all repositories to default to the new stable branches. If the collection will be officially paired with a ROS 2 distro: From 7db5fa7c16b190e4506de91fbf3ea30f13e4d92c Mon Sep 17 00:00:00 2001 From: Louise Poubel Date: Thu, 14 Apr 2022 14:59:32 -0700 Subject: [PATCH 125/540] Bumps in garden: ign-tools2 (#702) Signed-off-by: Louise Poubel --- jenkins-scripts/ign_tools-default-devel-windows-amd64.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/ign_tools-default-devel-windows-amd64.bat b/jenkins-scripts/ign_tools-default-devel-windows-amd64.bat index 2bb9073d1..de4940990 100644 --- a/jenkins-scripts/ign_tools-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_tools-default-devel-windows-amd64.bat @@ -5,6 +5,6 @@ set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true set COLCON_PACKAGE=ignition-tools -set COLCON_AUTO_MAJOR_VERSION=false +set COLCON_AUTO_MAJOR_VERSION=true call "%SCRIPT_DIR%\lib\colcon-default-devel-windows.bat" From 99269231a3582f2411f37a45b73dc96c30d718d9 Mon Sep 17 00:00:00 2001 From: methylDragon Date: Thu, 14 Apr 2022 17:00:39 -0700 Subject: [PATCH 126/540] Bumps in garden : ign-tools2 (#703) --- jenkins-scripts/dsl/ignition_collection.dsl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index 4ecb3fa8d..be1dd7aa4 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -18,7 +18,7 @@ ignition_collections = [ // These are the branches currently targeted at the upcoming collection // They're in topological order nightly_jobs: [ - 'tools' : [ debbuild: 'ign-tools' , branch: 'ign-tools1' ], + 'tools' : [ debbuild: 'ign-tools2' , branch: 'main' ], 'cmake' : [ debbuild: 'ign-cmake3' , branch: 'main' ], 'utils' : [ debbuild: 'ign-utils2' , branch: 'main' ], 'math' : [ debbuild: 'ign-math7' , branch: 'main' ], From ffd4858b4c3716ae0bb72c87fb4210f6aea93c78 Mon Sep 17 00:00:00 2001 From: methylDragon Date: Thu, 14 Apr 2022 17:32:15 -0700 Subject: [PATCH 127/540] Deprecate Dependencies Archive for ign-tools installs (#704) * Change conditions to stop install procs for ignition-tools Signed-off-by: methylDragon * Fix syntax error Signed-off-by: methylDragon --- jenkins-scripts/lib/dependencies_archive.sh | 68 ++++++++++++--------- 1 file changed, 39 insertions(+), 29 deletions(-) diff --git a/jenkins-scripts/lib/dependencies_archive.sh b/jenkins-scripts/lib/dependencies_archive.sh index e306d19dc..eaecfb790 100644 --- a/jenkins-scripts/lib/dependencies_archive.sh +++ b/jenkins-scripts/lib/dependencies_archive.sh @@ -83,7 +83,7 @@ if [[ -z ${SDFORMAT_MAJOR_VERSION} ]]; then SDFORMAT_MAJOR_VERSION=6 fi -if [[ ${SDFORMAT_MAJOR_VERSION} -ge 9 ]]; then +if [[ ${SDFORMAT_MAJOR_VERSION} -ge 9 && ${SDFORMAT_MAJOR_VERSION} -lt 13 ]]; then # sdformat9 requires ignition-math6 and # uses ignition-tools for a test SDFORMAT_BASE_DEPENDENCIES="${SDFORMAT_NO_IGN_DEPENDENCIES} \\ @@ -426,7 +426,9 @@ elif [[ ${IGN_TRANSPORT_MAJOR_VERSION} -ge 8 ]]; then ruby-ffi" fi -export IGN_TRANSPORT_DEPENDENCIES="${IGN_TRANSPORT_DEPENDENCIES} libignition-tools-dev" +if [[ ${IGN_TRANSPORT_MAJOR_VERSION} -lt 12 ]]; then + export IGN_TRANSPORT_DEPENDENCIES="${IGN_TRANSPORT_DEPENDENCIES} libignition-tools-dev" +fi IGN_COMMON_NO_IGN_DEPENDENCIES="pkg-config \\ ${pythonv} \\ @@ -472,14 +474,15 @@ if [[ ${IGN_FUEL_TOOLS_MAJOR_VERSION} -ge 4 ]]; then libignition-msgs5-dev" fi - -IGN_MSGS_DEPENDENCIES="libignition-tools-dev \\ - libprotobuf-dev \\ - libprotoc-dev \\ - protobuf-compiler \\ - libtinyxml2-dev \\ - ruby \\ - ruby-dev" +if [[ ${IGN_MSGS_MAJOR_VERSION} -lt 9 ]]; then + IGN_MSGS_DEPENDENCIES="libignition-tools-dev \\ + libprotobuf-dev \\ + libprotoc-dev \\ + protobuf-compiler \\ + libtinyxml2-dev \\ + ruby \\ + ruby-dev" +fi if [[ -n ${IGN_MSGS_MAJOR_VERSION} && ${IGN_MSGS_MAJOR_VERSION} -eq 1 ]]; then IGN_MSGS_DEPENDENCIES="${IGN_MSGS_DEPENDENCIES} \\ @@ -510,12 +513,14 @@ IGN_GUI_NO_IGN_DEPENDENCIES="qtbase5-dev \\ qml-module-qtquick2 \\ qtquickcontrols2-5-dev" -IGN_GUI_DEPENDENCIES="${IGN_GUI_NO_IGN_DEPENDENCIES} \\ - libignition-cmake2-dev \\ - libignition-common3-dev \\ - libignition-math6-dev \\ - libignition-plugin-dev \\ - libignition-tools-dev" +if [[ ${IGN_GUI_MAJOR_VERSION} -lt 7 ]]; then + IGN_GUI_DEPENDENCIES="${IGN_GUI_NO_IGN_DEPENDENCIES} \\ + libignition-cmake2-dev \\ + libignition-common3-dev \\ + libignition-math6-dev \\ + libignition-plugin-dev \\ + libignition-tools-dev" +fi if [[ -n "${IGN_GUI_MAJOR_VERSION}" && ${IGN_GUI_MAJOR_VERSION} -ge 4 ]]; then IGN_GUI_DEPENDENCIES="${IGN_GUI_DEPENDENCIES} \\ @@ -532,14 +537,16 @@ IGN_PHYSICS_DART_FROM_PKGS="true" IGN_PLUGIN_DEPENDENCIES="libignition-cmake2-dev" -IGN_LAUNCH_COMMON_DEPENDENCIES="libignition-cmake2-dev \\ - libignition-common3-dev \\ - libignition-math6-dev \\ - libignition-plugin-dev \\ - libignition-tools-dev \\ - libtinyxml2-dev \\ - qtquickcontrols2-5-dev \\ - libqt5core5a" +if [[ ${IGN_LAUNCH_MAJOR_VERSION} -lt 6 ]]; then + IGN_LAUNCH_COMMON_DEPENDENCIES="libignition-cmake2-dev \\ + libignition-common3-dev \\ + libignition-math6-dev \\ + libignition-plugin-dev \\ + libignition-tools-dev \\ + libtinyxml2-dev \\ + qtquickcontrols2-5-dev \\ + libqt5core5a" +fi if [[ -n "${IGN_LAUNCH_MAJOR_VERSION}" && ${IGN_LAUNCH_MAJOR_VERSION} -ge 3 ]]; then IGN_LAUNCH_DEPENDENCIES="${IGN_LAUNCH_COMMON_DEPENDENCIES} \\ @@ -580,11 +587,14 @@ IGN_RENDERING_DEPENDENCIES="${IGN_RENDERING_NO_IGN_DEPENDENCIES} \\ libignition-plugin-dev \\ libignition-math6-dev" -IGN_SENSORS_DEPENDENCIES="libignition-common3-dev \\ - libignition-cmake2-dev \\ - libignition-math6-dev \\ - libignition-plugin-dev \\ - libignition-tools-dev" +if [[ ${IGN_SENSORS_MAJOR_VERSION} -lt 7 ]]; then + IGN_SENSORS_DEPENDENCIES="libignition-common3-dev \\ + libignition-cmake2-dev \\ + libignition-math6-dev \\ + libignition-plugin-dev \\ + libignition-tools-dev" +fi + if [[ -n "${IGN_SENSORS_MAJOR_VERSION}" && ${IGN_SENSORS_MAJOR_VERSION} -ge 4 ]]; then IGN_SENSORS_DEPENDENCIES="${IGN_SENSORS_DEPENDENCIES} \\ libignition-msgs6-dev \\ From 8c55f7a0a43a4a2dfe1917d2a1e325af842f8ed9 Mon Sep 17 00:00:00 2001 From: methylDragon Date: Thu, 14 Apr 2022 19:00:03 -0700 Subject: [PATCH 128/540] Deprecate Dependencies Archive for ign-tools installs (#705) Signed-off-by: methylDragon --- jenkins-scripts/lib/dependencies_archive.sh | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/jenkins-scripts/lib/dependencies_archive.sh b/jenkins-scripts/lib/dependencies_archive.sh index eaecfb790..508e421c0 100644 --- a/jenkins-scripts/lib/dependencies_archive.sh +++ b/jenkins-scripts/lib/dependencies_archive.sh @@ -450,13 +450,15 @@ IGN_COMMON_DEPENDENCIES="${IGN_COMMON_NO_IGN_DEPENDENCIES} \\ libignition-math4-dev \\ libignition-math6-dev" -IGN_FUEL_TOOLS_DEPENDENCIES="libignition-cmake2-dev \\ - libignition-common3-dev \\ - libignition-tools-dev \\ - libcurl4-openssl-dev \\ - libjsoncpp-dev \\ - libyaml-dev \\ - libzip-dev" +if [[ ${IGN_FUEL_TOOLS_MAJOR_VERSION} -lt 8 ]]; then + IGN_FUEL_TOOLS_DEPENDENCIES="libignition-cmake2-dev \\ + libignition-common3-dev \\ + libignition-tools-dev \\ + libcurl4-openssl-dev \\ + libjsoncpp-dev \\ + libyaml-dev \\ + libzip-dev" +fi if [[ ${IGN_FUEL_TOOLS_MAJOR_VERSION} -eq 1 ]]; then IGN_FUEL_TOOLS_DEPENDENCIES="${IGN_FUEL_TOOLS_DEPENDENCIES} \\ From 634c3118689d2c084655865c643f0f310abf8401 Mon Sep 17 00:00:00 2001 From: methylDragon Date: Thu, 14 Apr 2022 23:11:39 -0700 Subject: [PATCH 129/540] Deprecate Dependencies Archive for ign-plugin installs (#707) Signed-off-by: methylDragon --- jenkins-scripts/lib/dependencies_archive.sh | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/jenkins-scripts/lib/dependencies_archive.sh b/jenkins-scripts/lib/dependencies_archive.sh index 508e421c0..33e6aff9e 100644 --- a/jenkins-scripts/lib/dependencies_archive.sh +++ b/jenkins-scripts/lib/dependencies_archive.sh @@ -583,11 +583,13 @@ IGN_RENDERING_NO_IGN_DEPENDENCIES="${ogre_pkg}\\ mesa-common-dev \\ mesa-utils" -IGN_RENDERING_DEPENDENCIES="${IGN_RENDERING_NO_IGN_DEPENDENCIES} \\ - libignition-cmake2-dev \\ - libignition-common3-dev \\ - libignition-plugin-dev \\ - libignition-math6-dev" +if [[ ${IGN_RENDERING_MAJOR_VERSION} -lt 7 ]]; then + IGN_RENDERING_DEPENDENCIES="${IGN_RENDERING_NO_IGN_DEPENDENCIES} \\ + libignition-cmake2-dev \\ + libignition-common3-dev \\ + libignition-plugin-dev \\ + libignition-math6-dev" +fi if [[ ${IGN_SENSORS_MAJOR_VERSION} -lt 7 ]]; then IGN_SENSORS_DEPENDENCIES="libignition-common3-dev \\ From 155a82c8095104a672abe5e5aba797d6779dd899 Mon Sep 17 00:00:00 2001 From: methylDragon Date: Fri, 15 Apr 2022 00:30:19 -0700 Subject: [PATCH 130/540] Depreciate Dependencies Archive for ign-rendering4 installs (#708) * Deprecate Dependencies Archive for ign-plugin installs Signed-off-by: methylDragon * Depreciate Dependencies Archive for ign-rendering4 installs Signed-off-by: methylDragon --- jenkins-scripts/lib/dependencies_archive.sh | 73 +++++++++++---------- 1 file changed, 39 insertions(+), 34 deletions(-) diff --git a/jenkins-scripts/lib/dependencies_archive.sh b/jenkins-scripts/lib/dependencies_archive.sh index 33e6aff9e..ec30b04fb 100644 --- a/jenkins-scripts/lib/dependencies_archive.sh +++ b/jenkins-scripts/lib/dependencies_archive.sh @@ -550,30 +550,33 @@ if [[ ${IGN_LAUNCH_MAJOR_VERSION} -lt 6 ]]; then libqt5core5a" fi -if [[ -n "${IGN_LAUNCH_MAJOR_VERSION}" && ${IGN_LAUNCH_MAJOR_VERSION} -ge 3 ]]; then - IGN_LAUNCH_DEPENDENCIES="${IGN_LAUNCH_COMMON_DEPENDENCIES} \\ - libignition-gui4-dev \\ - libignition-fuel-tools5-dev \\ - libignition-gazebo4-dev \\ - libignition-msgs6-dev \\ - libignition-rendering4-dev \\ - libignition-sensors4-dev \\ - libignition-transport9-dev \\ - libsdformat10-dev" -else - IGN_LAUNCH_DEPENDENCIES="${IGN_LAUNCH_DEPENDENCIES} \\ - libignition-gui3-dev \\ - libignition-fuel-tools4-dev \\ - libignition-gazebo3-dev \\ - libignition-msgs5-dev \\ - libignition-rendering3-dev \\ - libignition-sensors3-dev \\ - libignition-transport8-dev \\ - libsdformat9-dev \\ - libwebsockets-dev \\ - binutils-dev" +if [[ ${IGN_LAUNCH_MAJOR_VERSION} -lt 6 ]]; then + if [[ -n "${IGN_LAUNCH_MAJOR_VERSION}" && ${IGN_LAUNCH_MAJOR_VERSION} -ge 3 ]]; then + IGN_LAUNCH_DEPENDENCIES="${IGN_LAUNCH_COMMON_DEPENDENCIES} \\ + libignition-gui4-dev \\ + libignition-fuel-tools5-dev \\ + libignition-gazebo4-dev \\ + libignition-msgs6-dev \\ + libignition-rendering4-dev \\ + libignition-sensors4-dev \\ + libignition-transport9-dev \\ + libsdformat10-dev" + else + IGN_LAUNCH_DEPENDENCIES="${IGN_LAUNCH_DEPENDENCIES} \\ + libignition-gui3-dev \\ + libignition-fuel-tools4-dev \\ + libignition-gazebo3-dev \\ + libignition-msgs5-dev \\ + libignition-rendering3-dev \\ + libignition-sensors3-dev \\ + libignition-transport8-dev \\ + libsdformat9-dev \\ + libwebsockets-dev \\ + binutils-dev" + fi fi + IGN_RENDERING_NO_IGN_DEPENDENCIES="${ogre_pkg}\\ freeglut3-dev \\ libfreeimage-dev \\ @@ -599,18 +602,20 @@ if [[ ${IGN_SENSORS_MAJOR_VERSION} -lt 7 ]]; then libignition-tools-dev" fi -if [[ -n "${IGN_SENSORS_MAJOR_VERSION}" && ${IGN_SENSORS_MAJOR_VERSION} -ge 4 ]]; then - IGN_SENSORS_DEPENDENCIES="${IGN_SENSORS_DEPENDENCIES} \\ - libignition-msgs6-dev \\ - libignition-rendering4-dev \\ - libignition-transport9-dev \\ - libsdformat10-dev" -elif [[ -n "${IGN_SENSORS_MAJOR_VERSION}" && ${IGN_SENSORS_MAJOR_VERSION} -ge 3 ]]; then - IGN_SENSORS_DEPENDENCIES="${IGN_SENSORS_DEPENDENCIES} \\ - libignition-msgs5-dev \\ - libignition-rendering3-dev \\ - libignition-transport8-dev \\ - libsdformat9-dev" +if [[ ${IGN_SENSORS_MAJOR_VERSION} -lt 7 ]]; then + if [[ -n "${IGN_SENSORS_MAJOR_VERSION}" && ${IGN_SENSORS_MAJOR_VERSION} -ge 4 ]]; then + IGN_SENSORS_DEPENDENCIES="${IGN_SENSORS_DEPENDENCIES} \\ + libignition-msgs6-dev \\ + libignition-rendering4-dev \\ + libignition-transport9-dev \\ + libsdformat10-dev" + elif [[ -n "${IGN_SENSORS_MAJOR_VERSION}" && ${IGN_SENSORS_MAJOR_VERSION} -ge 3 ]]; then + IGN_SENSORS_DEPENDENCIES="${IGN_SENSORS_DEPENDENCIES} \\ + libignition-msgs5-dev \\ + libignition-rendering3-dev \\ + libignition-transport8-dev \\ + libsdformat9-dev" + fi fi IGN_GAZEBO_DEPENDENCIES="libignition-common3-dev \\ From 5f43d79dcfec5e87d5dc5f94f07f1e4596720e0b Mon Sep 17 00:00:00 2001 From: Louise Poubel Date: Fri, 15 Apr 2022 09:27:56 -0700 Subject: [PATCH 131/540] Completely rely on packages.apt from Garden (#709) Signed-off-by: Louise Poubel --- jenkins-scripts/lib/dependencies_archive.sh | 303 ++++++++++---------- 1 file changed, 158 insertions(+), 145 deletions(-) diff --git a/jenkins-scripts/lib/dependencies_archive.sh b/jenkins-scripts/lib/dependencies_archive.sh index ec30b04fb..7a5b91b63 100644 --- a/jenkins-scripts/lib/dependencies_archive.sh +++ b/jenkins-scripts/lib/dependencies_archive.sh @@ -384,18 +384,21 @@ fi # IGNITION # -if [[ ${IGN_MATH_MAJOR_VERSION} -eq 4 ]]; then - IGN_MATH_DEPENDENCIES="libeigen3-dev \\ - libpython3-dev \\ - ruby-dev \\ - swig \\ - libignition-cmake-dev" -else - IGN_MATH_DEPENDENCIES="libeigen3-dev \\ - libpython3-dev \\ - ruby-dev \\ - swig \\ - libignition-cmake2-dev" +# Completely rely on packages.apt from Garden +if [[ ${IGN_MATH_MAJOR_VERSION} -lt 7 ]]; then + if [[ ${IGN_MATH_MAJOR_VERSION} -eq 4 ]]; then + IGN_MATH_DEPENDENCIES="libeigen3-dev \\ + libpython3-dev \\ + ruby-dev \\ + swig \\ + libignition-cmake-dev" + else + IGN_MATH_DEPENDENCIES="libeigen3-dev \\ + libpython3-dev \\ + ruby-dev \\ + swig \\ + libignition-cmake2-dev" + fi fi # IGN_TRANSPORT related dependencies. Default value points to the development @@ -404,52 +407,55 @@ if [[ -z ${IGN_TRANSPORT_MAJOR_VERSION} ]]; then IGN_TRANSPORT_MAJOR_VERSION=5 fi -IGN_TRANSPORT_NO_IGN_DEPENDENCIES="pkg-config \\ - ${pythonv} \\ - ruby-ronn \\ - libprotoc-dev \\ - libprotobuf-dev \\ - protobuf-compiler \\ - uuid-dev \\ - libzmq3-dev \\ - libczmq-dev" - -if [[ ${IGN_TRANSPORT_MAJOR_VERSION} -eq 4 ]]; then - export IGN_TRANSPORT_DEPENDENCIES="${IGN_TRANSPORT_NO_IGN_DEPENDENCIES} \\ - libignition-cmake-dev \\ - libignition-msgs-dev" -elif [[ ${IGN_TRANSPORT_MAJOR_VERSION} -ge 8 ]]; then - export IGN_TRANSPORT_DEPENDENCIES="${IGN_TRANSPORT_NO_IGN_DEPENDENCIES} \\ - libignition-cmake2-dev \\ - libignition-msgs5-dev \\ - libsqlite3-dev \\ - ruby-ffi" -fi - +# Completely rely on packages.apt from Garden if [[ ${IGN_TRANSPORT_MAJOR_VERSION} -lt 12 ]]; then + IGN_TRANSPORT_NO_IGN_DEPENDENCIES="pkg-config \\ + ${pythonv} \\ + ruby-ronn \\ + libprotoc-dev \\ + libprotobuf-dev \\ + protobuf-compiler \\ + uuid-dev \\ + libzmq3-dev \\ + libczmq-dev" + if [[ ${IGN_TRANSPORT_MAJOR_VERSION} -eq 4 ]]; then + export IGN_TRANSPORT_DEPENDENCIES="${IGN_TRANSPORT_NO_IGN_DEPENDENCIES} \\ + libignition-cmake-dev \\ + libignition-msgs-dev" + elif [[ ${IGN_TRANSPORT_MAJOR_VERSION} -ge 8 ]]; then + export IGN_TRANSPORT_DEPENDENCIES="${IGN_TRANSPORT_NO_IGN_DEPENDENCIES} \\ + libignition-cmake2-dev \\ + libignition-msgs5-dev \\ + libsqlite3-dev \\ + ruby-ffi" + fi export IGN_TRANSPORT_DEPENDENCIES="${IGN_TRANSPORT_DEPENDENCIES} libignition-tools-dev" fi -IGN_COMMON_NO_IGN_DEPENDENCIES="pkg-config \\ - ${pythonv} \\ - ruby-ronn \\ - uuid-dev \\ - libfreeimage-dev \\ - libgts-dev \\ - libavformat-dev \\ - libavcodec-dev \\ - libswscale-dev \\ - libavutil-dev \\ - libavdevice-dev \\ - libtinyxml2-dev \\ - uuid-dev" - -IGN_COMMON_DEPENDENCIES="${IGN_COMMON_NO_IGN_DEPENDENCIES} \\ - libignition-cmake-dev \\ - libignition-cmake2-dev \\ - libignition-math4-dev \\ - libignition-math6-dev" - +# Completely rely on packages.apt from Garden +if [[ ${IGN_COMMON_MAJOR_VERSION} -lt 5 ]]; then + IGN_COMMON_NO_IGN_DEPENDENCIES="pkg-config \\ + ${pythonv} \\ + ruby-ronn \\ + uuid-dev \\ + libfreeimage-dev \\ + libgts-dev \\ + libavformat-dev \\ + libavcodec-dev \\ + libswscale-dev \\ + libavutil-dev \\ + libavdevice-dev \\ + libtinyxml2-dev \\ + uuid-dev" + + IGN_COMMON_DEPENDENCIES="${IGN_COMMON_NO_IGN_DEPENDENCIES} \\ + libignition-cmake-dev \\ + libignition-cmake2-dev \\ + libignition-math4-dev \\ + libignition-math6-dev" +fi + +# Completely rely on packages.apt from Garden if [[ ${IGN_FUEL_TOOLS_MAJOR_VERSION} -lt 8 ]]; then IGN_FUEL_TOOLS_DEPENDENCIES="libignition-cmake2-dev \\ libignition-common3-dev \\ @@ -458,24 +464,25 @@ if [[ ${IGN_FUEL_TOOLS_MAJOR_VERSION} -lt 8 ]]; then libjsoncpp-dev \\ libyaml-dev \\ libzip-dev" -fi -if [[ ${IGN_FUEL_TOOLS_MAJOR_VERSION} -eq 1 ]]; then - IGN_FUEL_TOOLS_DEPENDENCIES="${IGN_FUEL_TOOLS_DEPENDENCIES} \\ - libignition-cmake-dev \\ - libignition-common-dev" -else - IGN_FUEL_TOOLS_DEPENDENCIES="${IGN_FUEL_TOOLS_DEPENDENCIES} \\ - libignition-cmake2-dev \\ - libignition-common3-dev \\ - libtinyxml2-dev" -fi + if [[ ${IGN_FUEL_TOOLS_MAJOR_VERSION} -eq 1 ]]; then + IGN_FUEL_TOOLS_DEPENDENCIES="${IGN_FUEL_TOOLS_DEPENDENCIES} \\ + libignition-cmake-dev \\ + libignition-common-dev" + else + IGN_FUEL_TOOLS_DEPENDENCIES="${IGN_FUEL_TOOLS_DEPENDENCIES} \\ + libignition-cmake2-dev \\ + libignition-common3-dev \\ + libtinyxml2-dev" + fi -if [[ ${IGN_FUEL_TOOLS_MAJOR_VERSION} -ge 4 ]]; then - IGN_FUEL_TOOLS_DEPENDENCIES="${IGN_FUEL_TOOLS_DEPENDENCIES} \\ - libignition-msgs5-dev" + if [[ ${IGN_FUEL_TOOLS_MAJOR_VERSION} -ge 4 ]]; then + IGN_FUEL_TOOLS_DEPENDENCIES="${IGN_FUEL_TOOLS_DEPENDENCIES} \\ + libignition-msgs5-dev" + fi fi +# Completely rely on packages.apt from Garden if [[ ${IGN_MSGS_MAJOR_VERSION} -lt 9 ]]; then IGN_MSGS_DEPENDENCIES="libignition-tools-dev \\ libprotobuf-dev \\ @@ -484,61 +491,65 @@ if [[ ${IGN_MSGS_MAJOR_VERSION} -lt 9 ]]; then libtinyxml2-dev \\ ruby \\ ruby-dev" -fi -if [[ -n ${IGN_MSGS_MAJOR_VERSION} && ${IGN_MSGS_MAJOR_VERSION} -eq 1 ]]; then - IGN_MSGS_DEPENDENCIES="${IGN_MSGS_DEPENDENCIES} \\ - libignition-cmake-dev \\ - libignition-math4-dev" -elif [[ -n ${IGN_MSGS_MAJOR_VERSION} && ${IGN_MSGS_MAJOR_VERSION} -ge 5 ]]; then - IGN_MSGS_DEPENDENCIES="${IGN_MSGS_DEPENDENCIES} \\ - libignition-cmake2-dev \\ - libignition-math6-dev" + if [[ -n ${IGN_MSGS_MAJOR_VERSION} && ${IGN_MSGS_MAJOR_VERSION} -eq 1 ]]; then + IGN_MSGS_DEPENDENCIES="${IGN_MSGS_DEPENDENCIES} \\ + libignition-cmake-dev \\ + libignition-math4-dev" + elif [[ -n ${IGN_MSGS_MAJOR_VERSION} && ${IGN_MSGS_MAJOR_VERSION} -ge 5 ]]; then + IGN_MSGS_DEPENDENCIES="${IGN_MSGS_DEPENDENCIES} \\ + libignition-cmake2-dev \\ + libignition-math6-dev" + fi fi -IGN_GUI_NO_IGN_DEPENDENCIES="qtbase5-dev \\ - qtdeclarative5-dev \\ - libtinyxml2-dev \\ - libqwt-qt5-dev \\ - qml-module-qt-labs-folderlistmodel \\ - qml-module-qt-labs-platform \\ - qml-module-qt-labs-settings \\ - qml-module-qtcharts \\ - qml-module-qtgraphicaleffects \\ - qml-module-qtqml-models2 \\ - qml-module-qtquick-controls \\ - qml-module-qtquick-controls2 \\ - qml-module-qtquick-dialogs \\ - qml-module-qtquick-layouts \\ - qml-module-qtquick-templates2 \\ - qml-module-qtquick-window2 \\ - qml-module-qtquick2 \\ - qtquickcontrols2-5-dev" - +# Completely rely on packages.apt from Garden if [[ ${IGN_GUI_MAJOR_VERSION} -lt 7 ]]; then + IGN_GUI_NO_IGN_DEPENDENCIES="qtbase5-dev \\ + qtdeclarative5-dev \\ + libtinyxml2-dev \\ + libqwt-qt5-dev \\ + qml-module-qt-labs-folderlistmodel \\ + qml-module-qt-labs-platform \\ + qml-module-qt-labs-settings \\ + qml-module-qtcharts \\ + qml-module-qtgraphicaleffects \\ + qml-module-qtqml-models2 \\ + qml-module-qtquick-controls \\ + qml-module-qtquick-controls2 \\ + qml-module-qtquick-dialogs \\ + qml-module-qtquick-layouts \\ + qml-module-qtquick-templates2 \\ + qml-module-qtquick-window2 \\ + qml-module-qtquick2 \\ + qtquickcontrols2-5-dev" IGN_GUI_DEPENDENCIES="${IGN_GUI_NO_IGN_DEPENDENCIES} \\ libignition-cmake2-dev \\ libignition-common3-dev \\ libignition-math6-dev \\ libignition-plugin-dev \\ libignition-tools-dev" -fi -if [[ -n "${IGN_GUI_MAJOR_VERSION}" && ${IGN_GUI_MAJOR_VERSION} -ge 4 ]]; then - IGN_GUI_DEPENDENCIES="${IGN_GUI_DEPENDENCIES} \\ - libignition-common3-dev" -elif [[ -n "${IGN_GUI_MAJOR_VERSION}" && ${IGN_GUI_MAJOR_VERSION} -eq 3 ]]; then - IGN_GUI_DEPENDENCIES="${IGN_GUI_DEPENDENCIES} \\ - libignition-common3-dev \\ - libignition-msgs5-dev \\ - libignition-rendering3-dev \\ - libignition-transport8-dev" + if [[ -n "${IGN_GUI_MAJOR_VERSION}" && ${IGN_GUI_MAJOR_VERSION} -ge 4 ]]; then + IGN_GUI_DEPENDENCIES="${IGN_GUI_DEPENDENCIES} \\ + libignition-common3-dev" + elif [[ -n "${IGN_GUI_MAJOR_VERSION}" && ${IGN_GUI_MAJOR_VERSION} -eq 3 ]]; then + IGN_GUI_DEPENDENCIES="${IGN_GUI_DEPENDENCIES} \\ + libignition-common3-dev \\ + libignition-msgs5-dev \\ + libignition-rendering3-dev \\ + libignition-transport8-dev" + fi fi IGN_PHYSICS_DART_FROM_PKGS="true" -IGN_PLUGIN_DEPENDENCIES="libignition-cmake2-dev" +# Completely rely on packages.apt from Garden +if [[ ${IGN_PLUGIN_MAJOR_VERSION} -lt 2 ]]; then + IGN_PLUGIN_DEPENDENCIES="libignition-cmake2-dev" +fi +# Completely rely on packages.apt from Garden if [[ ${IGN_LAUNCH_MAJOR_VERSION} -lt 6 ]]; then IGN_LAUNCH_COMMON_DEPENDENCIES="libignition-cmake2-dev \\ libignition-common3-dev \\ @@ -548,9 +559,7 @@ if [[ ${IGN_LAUNCH_MAJOR_VERSION} -lt 6 ]]; then libtinyxml2-dev \\ qtquickcontrols2-5-dev \\ libqt5core5a" -fi -if [[ ${IGN_LAUNCH_MAJOR_VERSION} -lt 6 ]]; then if [[ -n "${IGN_LAUNCH_MAJOR_VERSION}" && ${IGN_LAUNCH_MAJOR_VERSION} -ge 3 ]]; then IGN_LAUNCH_DEPENDENCIES="${IGN_LAUNCH_COMMON_DEPENDENCIES} \\ libignition-gui4-dev \\ @@ -576,17 +585,17 @@ if [[ ${IGN_LAUNCH_MAJOR_VERSION} -lt 6 ]]; then fi fi - -IGN_RENDERING_NO_IGN_DEPENDENCIES="${ogre_pkg}\\ - freeglut3-dev \\ - libfreeimage-dev \\ - libglew-dev \\ - libogre-1.9-dev \\ - libx11-dev \\ - mesa-common-dev \\ - mesa-utils" - +# Completely rely on packages.apt from Garden if [[ ${IGN_RENDERING_MAJOR_VERSION} -lt 7 ]]; then + IGN_RENDERING_NO_IGN_DEPENDENCIES="${ogre_pkg}\\ + freeglut3-dev \\ + libfreeimage-dev \\ + libglew-dev \\ + libogre-1.9-dev \\ + libx11-dev \\ + mesa-common-dev \\ + mesa-utils" + IGN_RENDERING_DEPENDENCIES="${IGN_RENDERING_NO_IGN_DEPENDENCIES} \\ libignition-cmake2-dev \\ libignition-common3-dev \\ @@ -594,15 +603,14 @@ if [[ ${IGN_RENDERING_MAJOR_VERSION} -lt 7 ]]; then libignition-math6-dev" fi +# Completely rely on packages.apt from Garden if [[ ${IGN_SENSORS_MAJOR_VERSION} -lt 7 ]]; then IGN_SENSORS_DEPENDENCIES="libignition-common3-dev \\ libignition-cmake2-dev \\ libignition-math6-dev \\ libignition-plugin-dev \\ libignition-tools-dev" -fi -if [[ ${IGN_SENSORS_MAJOR_VERSION} -lt 7 ]]; then if [[ -n "${IGN_SENSORS_MAJOR_VERSION}" && ${IGN_SENSORS_MAJOR_VERSION} -ge 4 ]]; then IGN_SENSORS_DEPENDENCIES="${IGN_SENSORS_DEPENDENCIES} \\ libignition-msgs6-dev \\ @@ -618,29 +626,34 @@ if [[ ${IGN_SENSORS_MAJOR_VERSION} -lt 7 ]]; then fi fi -IGN_GAZEBO_DEPENDENCIES="libignition-common3-dev \\ - libignition-cmake2-dev \\ - libgflags-dev \\ - libignition-math6-dev \\ - libignition-math6-eigen3-dev \\ - libbenchmark-dev" - -if [[ -n "${IGN_GAZEBO_MAJOR_VERSION}" && ${IGN_GAZEBO_MAJOR_VERSION} -ge 4 ]]; then - IGN_GAZEBO_DEPENDENCIES="${IGN_GAZEBO_DEPENDENCIES} \\ - libsdformat10-dev" -elif [[ -n "${IGN_GAZEBO_MAJOR_VERSION}" && ${IGN_GAZEBO_MAJOR_VERSION} -eq 3 ]]; then - IGN_GAZEBO_DEPENDENCIES="${IGN_GAZEBO_DEPENDENCIES} \\ - libignition-fuel-tools4-dev \\ - libignition-gui3-dev \\ - libignition-msgs5-dev \\ - libignition-rendering3-dev \\ - libignition-sensors3-dev \\ - libignition-physics2 \\ - libignition-physics2-dartsim \\ - libignition-physics2-dartsim-dev \\ - libignition-physics2-dev \\ - libignition-transport8-dev \\ - libsdformat9-dev" +# Completely rely on packages.apt from Garden +if [[ ${IGN_GAZEBO_MAJOR_VERSION} -lt 7 ]]; then + IGN_GAZEBO_DEPENDENCIES="libignition-common3-dev \\ + libignition-cmake2-dev \\ + libgflags-dev \\ + libignition-math6-dev \\ + libignition-math6-eigen3-dev \\ + libbenchmark-dev" + if [[ -n "${IGN_GAZEBO_MAJOR_VERSION}" && ${IGN_GAZEBO_MAJOR_VERSION} -ge 4 ]]; then + IGN_GAZEBO_DEPENDENCIES="${IGN_GAZEBO_DEPENDENCIES} \\ + libsdformat10-dev" + elif [[ -n "${IGN_GAZEBO_MAJOR_VERSION}" && ${IGN_GAZEBO_MAJOR_VERSION} -eq 3 ]]; then + IGN_GAZEBO_DEPENDENCIES="${IGN_GAZEBO_DEPENDENCIES} \\ + libignition-fuel-tools4-dev \\ + libignition-gui3-dev \\ + libignition-msgs5-dev \\ + libignition-rendering3-dev \\ + libignition-sensors3-dev \\ + libignition-physics2 \\ + libignition-physics2-dartsim \\ + libignition-physics2-dartsim-dev \\ + libignition-physics2-dev \\ + libignition-transport8-dev \\ + libsdformat9-dev" + fi fi -IGN_UTILS_DEPENDENCIES="libignition-cmake2-dev" +# Completely rely on packages.apt from Garden +if [[ ${IGN_UTILS_MAJOR_VERSION} -lt 2 ]]; then + IGN_UTILS_DEPENDENCIES="libignition-cmake2-dev" +fi From 870786d33b81b53534f76a92009ae3db94899a3b Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Mon, 18 Apr 2022 18:06:07 +0200 Subject: [PATCH 132/540] ASAN CI builds for ignition and sdformat (#678) * ASAN CI builds for ignition and sdformat * Do not ignore OSRF_REPOS_TO_USE if GZDEV_PROJECT_NAME is used * Skip IGN tests for asan builds Signed-off-by: Jose Luis Rivero --- .../docker/ign_cmake-compilation.bash | 2 +- .../docker/ign_rendering-compilation.bash | 2 +- .../_generic_linux_compilation_build.sh.bash | 3 +- .../lib/docker_generate_dockerfile.bash | 5 +- .../docker/sdformat-compilation.bash | 2 +- jenkins-scripts/dsl/_configs_/Globals.groovy | 4 +- jenkins-scripts/dsl/ignition.dsl | 67 +++++++++++++------ jenkins-scripts/dsl/sdformat.dsl | 61 ++++++++++++----- 8 files changed, 101 insertions(+), 45 deletions(-) diff --git a/jenkins-scripts/docker/ign_cmake-compilation.bash b/jenkins-scripts/docker/ign_cmake-compilation.bash index b79901ccf..d5b396489 100644 --- a/jenkins-scripts/docker/ign_cmake-compilation.bash +++ b/jenkins-scripts/docker/ign_cmake-compilation.bash @@ -18,7 +18,7 @@ export BUILDING_SOFTWARE_DIRECTORY="ign-cmake" export BUILDING_DEPENDENCIES="pkg-config" # Enable long-running ign-cmake tests in CI. -export BUILDING_EXTRA_CMAKE_PARAMS="-DBUILDSYSTEM_TESTING=True" +export BUILDING_EXTRA_CMAKE_PARAMS+=" -DBUILDSYSTEM_TESTING=True" # Identify IGN_CMAKE_MAJOR_VERSION to help with dependency resolution IGN_CMAKE_MAJOR_VERSION=$(\ diff --git a/jenkins-scripts/docker/ign_rendering-compilation.bash b/jenkins-scripts/docker/ign_rendering-compilation.bash index 197e928f5..d9607edb3 100644 --- a/jenkins-scripts/docker/ign_rendering-compilation.bash +++ b/jenkins-scripts/docker/ign_rendering-compilation.bash @@ -35,6 +35,6 @@ fi export GPU_SUPPORT_NEEDED=true export GZDEV_PROJECT_NAME="ignition-rendering${IGN_RENDERING_MAJOR_VERSION}" -export BUILDING_EXTRA_CMAKE_PARAMS="-DSKIP_optix=true" +export BUILDING_EXTRA_CMAKE_PARAMS+=" -DSKIP_optix=true" . ${SCRIPT_DIR}/lib/generic-building-base.bash diff --git a/jenkins-scripts/docker/lib/_generic_linux_compilation_build.sh.bash b/jenkins-scripts/docker/lib/_generic_linux_compilation_build.sh.bash index 3e4f523aa..511807060 100644 --- a/jenkins-scripts/docker/lib/_generic_linux_compilation_build.sh.bash +++ b/jenkins-scripts/docker/lib/_generic_linux_compilation_build.sh.bash @@ -6,6 +6,7 @@ # - GENERIC_ENABLE_CPPCHECK (optional) [default true] run cppcheck # - GENERIC_ENABLE_TESTS (optional) [default true] run tests # - BUILDING_EXTRA_CMAKE_PARAMS (optional) extra cmake params +# - BUILDING_EXTRA_MAKETEST_PARAMS (optional) extra "make test ARGS=" params # - BUILD_ (optional) build dependency from source, for example, BUILD_IGN_MATH # - _BRANCH (optional [default: master]) branch for BUILD_ @@ -95,7 +96,7 @@ if $GENERIC_ENABLE_TESTS; then echo '# BEGIN SECTION: running tests' init_stopwatch TEST mkdir -p \$HOME - make test ARGS="-VV" || true + make test ARGS="-VV ${BUILDING_EXTRA_MAKETEST_PARAMS}" || true stop_stopwatch TEST echo '# END SECTION' else diff --git a/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash b/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash index 4673a868d..81436efc1 100644 --- a/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash +++ b/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash @@ -57,7 +57,9 @@ RUN ${GZDEV_DIR}/gzdev.py repository enable --project=${GZDEV_PROJECT_NAME} --fo if [ -n $GZDEV_TRY_BRANCH ]; then git -C ${GZDEV_DIR} checkout $GZDEV_TRY_BRANCH; fi || true && \ ${GZDEV_DIR}/gzdev.py repository enable --project=${GZDEV_PROJECT_NAME} --force-linux-distro=${DISTRO} ) DELIM_OSRF_REPO_GZDEV -else +fi + +# This could duplicate repositories enabled in the step above. gzdev should warn about it without failing. for repo in ${OSRF_REPOS_TO_USE}; do cat >> Dockerfile << DELIM_OSRF_REPO RUN ${GZDEV_DIR}/gzdev.py repository enable osrf ${repo} --force-linux-distro=${DISTRO} || ( git -C ${GZDEV_DIR} pull origin ${GZDEV_BRANCH} && \ @@ -65,7 +67,6 @@ RUN ${GZDEV_DIR}/gzdev.py repository enable osrf ${repo} --force-linux-distro=${ ${GZDEV_DIR}/gzdev.py repository enable osrf ${repo} --force-linux-distro=${DISTRO} ) DELIM_OSRF_REPO done -fi } case ${LINUX_DISTRO} in diff --git a/jenkins-scripts/docker/sdformat-compilation.bash b/jenkins-scripts/docker/sdformat-compilation.bash index 67bdd9551..21d03de1f 100644 --- a/jenkins-scripts/docker/sdformat-compilation.bash +++ b/jenkins-scripts/docker/sdformat-compilation.bash @@ -25,7 +25,7 @@ fi export GZDEV_PROJECT_NAME="sdformat${SDFORMAT_MAJOR_VERSION}" if [[ ${SDFORMAT_MAJOR_VERSION} -ge 12 ]]; then - export BUILDING_EXTRA_CMAKE_PARAMS="-DSKIP_usd=true" + export BUILDING_EXTRA_CMAKE_PARAMS+=" -DSKIP_usd=true" fi # master and major branches compilations diff --git a/jenkins-scripts/dsl/_configs_/Globals.groovy b/jenkins-scripts/dsl/_configs_/Globals.groovy index 8fd517ae9..068a9a4c6 100644 --- a/jenkins-scripts/dsl/_configs_/Globals.groovy +++ b/jenkins-scripts/dsl/_configs_/Globals.groovy @@ -10,8 +10,10 @@ class Globals static rtools_description = true static gazebodistro_branch = false - static CRON_EVERY_THREE_DAYS = 'H H * * H/3' + static CRON_ON_WEEKEND = 'H H * * 6-7' + + static MAKETEST_SKIP_IGN = "-E _ign_TEST" static gpu_by_distro = [ bionic : [ 'nvidia' ]] diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index d7f875e61..85fcd8602 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -429,38 +429,65 @@ ignition_software.each { ign_sw -> } } -// OTHER CI SUPPORTED JOBS / DAILY +void generate_asan_ci_job(ignition_ci_job, ign_sw, branch, distro, arch) +{ + generate_ci_job(ignition_ci_job, ign_sw, branch, distro, arch, + '-DIGN_SANITIZER=Address', + Globals.MAKETEST_SKIP_IGN) +} + +void generate_ci_job(ignition_ci_job, ign_sw, branch, distro, arch, + extra_cmake = '', extra_test = '') +{ + OSRFLinuxCompilation.create(ignition_ci_job, enable_testing(ign_sw)) + OSRFGitHub.create(ignition_ci_job, + "ignitionrobotics/ign-${ign_sw}", + "${branch}", "ign-${ign_sw}") + + include_gpu_label_if_needed(ignition_ci_job, ign_sw) + ignition_ci_job.with + { + if (ign_sw == 'physics') + label "huge-memory" + + steps { + shell("""\ + #!/bin/bash -xe + + ${GLOBAL_SHELL_CMD} + export BUILDING_EXTRA_CMAKE_PARAMS="${extra_cmake}" + export BUILDING_EXTRA_MAKETEST_PARAMS="${extra_test}" + export DISTRO=${distro} + export ARCH=${arch} + /bin/bash -xe ./scripts/jenkins-scripts/docker/ign_${ign_sw}-compilation.bash + """.stripIndent()) + } + } +} + +// OTHER CI SUPPORTED JOBS ignition_software.each { ign_sw -> all_supported_distros.each { distro -> supported_arches.each { arch -> // -------------------------------------------------------------- // branches CI job scm@daily all_branches("${ign_sw}").each { branch -> + // 1. Standard CI def ignition_ci_job = job("ignition_${ign_sw}-ci-${branch}-${distro}-${arch}") - OSRFLinuxCompilation.create(ignition_ci_job, enable_testing(ign_sw)) - OSRFGitHub.create(ignition_ci_job, - "ignitionrobotics/ign-${ign_sw}", - "${branch}", "ign-${ign_sw}") - - include_gpu_label_if_needed(ignition_ci_job, ign_sw) + generate_ci_job(ignition_ci_job, ign_sw, branch, distro, arch) ignition_ci_job.with { - if (ign_sw == 'physics') - label "huge-memory" - triggers { scm('@daily') } - - steps { - shell("""\ - #!/bin/bash -xe - - ${GLOBAL_SHELL_CMD} - export DISTRO=${distro} - export ARCH=${arch} - /bin/bash -xe ./scripts/jenkins-scripts/docker/ign_${ign_sw}-compilation.bash - """.stripIndent()) + } + // 2. ASAN CI + def ignition_ci_asan_job = job("ignition_${ign_sw}-ci_asan-${branch}-${distro}-${arch}") + generate_asan_ci_job(ignition_ci_asan_job, ign_sw, branch, distro, arch) + ignition_ci_asan_job.with + { + triggers { + scm(Globals.CRON_ON_WEEKEND) } } } diff --git a/jenkins-scripts/dsl/sdformat.dsl b/jenkins-scripts/dsl/sdformat.dsl index 1d59b0d96..80e024096 100644 --- a/jenkins-scripts/dsl/sdformat.dsl +++ b/jenkins-scripts/dsl/sdformat.dsl @@ -136,35 +136,60 @@ other_supported_distros.each { distro -> } // end of arch } // end of distro -// BRANCHES CI JOB @ SCM/DAILY +void generate_asan_ci_job(sdformat_ci_job, branch, distro, arch) +{ + generate_ci_job(sdformat_ci_job, branch, distro, arch, + '-DIGN_SANITIZER=Address', + Globals.MAKETEST_SKIP_IGN) +} + +void generate_ci_job(sdformat_ci_job, branch, distro, arch, + extra_cmake = '', extra_test = '') +{ + OSRFLinuxCompilation.create(sdformat_ci_job) + OSRFGitHub.create(sdformat_ci_job, "ignitionrobotics/sdformat", + get_sdformat_branch_name(branch)) + sdformat_ci_job.with + { + steps { + shell("""\ + #!/bin/bash -xe + + export BUILDING_EXTRA_CMAKE_PARAMS="${extra_cmake}" + export BUILDING_EXTRA_MAKETEST_PARAMS="${extra_test}" + export DISTRO=${distro} + export ARCH=${arch} + /bin/bash -xe ./scripts/jenkins-scripts/docker/sdformat-compilation.bash + """.stripIndent()) + } + } +} + +// BRANCHES CI JOB @ SCM sdformat_supported_branches.each { branch -> ci_distro.each { distro -> supported_arches.each { arch -> - // ci_main job for the rest of arches / scm@daily + // ci job for the rest of arches / scm@daily def sdformat_ci_job = job("sdformat-ci-${branch}-${distro}-${arch}") - OSRFLinuxCompilation.create(sdformat_ci_job) - OSRFGitHub.create(sdformat_ci_job, "ignitionrobotics/sdformat", - get_sdformat_branch_name(branch)) + generate_ci_job(sdformat_ci_job, branch, distro, arch) sdformat_ci_job.with { triggers { scm('@daily') } - - steps { - shell("""\ - #!/bin/bash -xe - - export DISTRO=${distro} - export ARCH=${arch} - /bin/bash -xe ./scripts/jenkins-scripts/docker/sdformat-compilation.bash - """.stripIndent()) + } + // ci_asan job for the rest of arches / scm@weekend + def sdformat_ci_asan_job = job("sdformat-ci_asan-${branch}-${distro}-${arch}") + generate_asan_ci_job(sdformat_ci_asan_job, branch, distro, arch) + sdformat_ci_asan_job.with + { + triggers { + scm(Globals.CRON_ON_WEEKEND) } } - } // end of arch - } // end of distro -} // end of distro -// + } + } +} // EXPERIMENTAL ARCHES @ SCM/WEEKLY ci_distro.each { distro -> From 5e99985b02ad4f47e87b20845dd3f4c4a8ae5728 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 19 Apr 2022 15:35:52 +0200 Subject: [PATCH 133/540] Change linux-nightly-jammy from ip-* node to to drogon (#697) --- docs/BUILDFARM_NODES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/BUILDFARM_NODES.md b/docs/BUILDFARM_NODES.md index 908a623f9..8b625675c 100644 --- a/docs/BUILDFARM_NODES.md +++ b/docs/BUILDFARM_NODES.md @@ -30,7 +30,7 @@ For generating nightlies controlling the generation order of every library, buil | -------- | ----------- | | linux-nightly-bionic | r2d2 | | linux-nightly-focal | optimus | -| linux-nightly-jammy | linux-ip-172-30-1-216 | +| linux-nightly-jammy | drogon | ## Provision of Node labels From 987a45738a2b2276603d4bd2266380d4d4424ebc Mon Sep 17 00:00:00 2001 From: Jorge Perez Date: Tue, 26 Apr 2022 10:49:25 -0500 Subject: [PATCH 134/540] Fix build regression on ign-tools1 for windows on CI (#713) * Fix build regression on ign-tools1 for windows on CI Signed-off-by: Jorge Perez --- .../ign_tools-default-devel-windows-amd64.bat | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/jenkins-scripts/ign_tools-default-devel-windows-amd64.bat b/jenkins-scripts/ign_tools-default-devel-windows-amd64.bat index de4940990..afcb992a3 100644 --- a/jenkins-scripts/ign_tools-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_tools-default-devel-windows-amd64.bat @@ -5,6 +5,17 @@ set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true set COLCON_PACKAGE=ignition-tools -set COLCON_AUTO_MAJOR_VERSION=true + +:: override logic @ colcon-default-devel-windows.bat to handle ign-tools1 case on windows +setlocal ENABLEDELAYEDEXPANSION +for /f %%i in ('python "%SCRIPT_DIR%\tools\detect_cmake_major_version.py" "%WORKSPACE%\%VCS_DIRECTORY%\CMakeLists.txt"') do set PKG_MAJOR_VERSION=%%i +echo "MAJOR_VERSION detected: !PKG_MAJOR_VERSION!" +if "!PKG_MAJOR_VERSION!" == "1" ( + set COLCON_PACKAGE=%COLCON_PACKAGE% +) else ( + set COLCON_PACKAGE=%COLCON_PACKAGE%!PKG_MAJOR_VERSION! +) + +set COLCON_AUTO_MAJOR_VERSION=false call "%SCRIPT_DIR%\lib\colcon-default-devel-windows.bat" From dbb5462bdea0d12bb0a6c7a976136329703e1040 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Sat, 30 Apr 2022 00:05:01 +0200 Subject: [PATCH 135/540] Remove libglvnd in docker for jammy (broken) and fix gazebo CI (#715) * Do not install glvnd on jammy * Exclude OpenAL errors from failing Gazebo buil --- jenkins-scripts/docker/lib/docker_generate_dockerfile.bash | 7 ++++++- jenkins-scripts/dsl/gazebo.dsl | 3 +++ jenkins-scripts/parser_rules/gazebo_err.parser | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash b/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash index 81436efc1..f1223cc25 100644 --- a/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash +++ b/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash @@ -348,6 +348,10 @@ cat >> Dockerfile << DELIM_NVIDIA2_GPU ${NVIDIA_VISIBLE_DEVICES:-all} ENV NVIDIA_DRIVER_CAPABILITIES \ ${NVIDIA_DRIVER_CAPABILITIES:+$NVIDIA_DRIVER_CAPABILITIES,}graphics +DELIM_NVIDIA2_GPU + +if [[ ${LINUX_DISTRO} == 'ubuntu' ]] && [[ ${DISTRO} == 'bionic' || ${DISTRO} == 'focal' ]]; then +cat >> Dockerfile << DELIM_NVIDIA3_GPU # Install libglvnd for OpenGL using nvidia-docker2 RUN apt-get update && apt-get install -y --no-install-recommends \ git \ @@ -369,7 +373,8 @@ RUN mkdir -p /opt/libglvnd && cd /opt/libglvnd && \ make install-strip && \ find /usr/local/lib/x86_64-linux-gnu -type f -name 'lib*.la' -delete ENV LD_LIBRARY_PATH /usr/local/lib/x86_64-linux-gnu\${LD_LIBRARY_PATH:+:\${LD_LIBRARY_PATH}} -DELIM_NVIDIA2_GPU +DELIM_NVIDIA3_GPU +fi fi else # No NVIDIA cards needs to have the same X stack than the host diff --git a/jenkins-scripts/dsl/gazebo.dsl b/jenkins-scripts/dsl/gazebo.dsl index bbb02cb5a..af5236628 100644 --- a/jenkins-scripts/dsl/gazebo.dsl +++ b/jenkins-scripts/dsl/gazebo.dsl @@ -49,6 +49,9 @@ void generate_install_job(Job job, gz_branch, distro, arch, use_osrf_repos = fal // Branch is exactly in the form of gazeboN def dev_packages = "lib${gz_branch}-dev ${gz_branch}" + // Ubuntu Jammy has no versioned gazebo package + if (distro == 'jammy') + dev_packages = "libgazebo-dev gazebo" def gzdev_str = "" if (use_osrf_repos) diff --git a/jenkins-scripts/parser_rules/gazebo_err.parser b/jenkins-scripts/parser_rules/gazebo_err.parser index 92fca5621..388e06f17 100644 --- a/jenkins-scripts/parser_rules/gazebo_err.parser +++ b/jenkins-scripts/parser_rules/gazebo_err.parser @@ -1 +1 @@ -error /.*\[Err\].*/ +error /.*\[Err\].*(!OpenAL.cc).*/ From 06dcb15e5bc4645e15e9d15d069aaaecec664cad Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 3 May 2022 20:48:12 +0200 Subject: [PATCH 136/540] Support for Rolling on Jammy for gazebo_ros_pkgs (#717) * Move ROS 2 Rolling CI to Jammy * Use unversioned gazebo packages in Jammy Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/_configs_/Globals.groovy | 2 +- jenkins-scripts/lib/dependencies_archive.sh | 13 ++++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/jenkins-scripts/dsl/_configs_/Globals.groovy b/jenkins-scripts/dsl/_configs_/Globals.groovy index 068a9a4c6..dd439dde0 100644 --- a/jenkins-scripts/dsl/_configs_/Globals.groovy +++ b/jenkins-scripts/dsl/_configs_/Globals.groovy @@ -21,7 +21,7 @@ class Globals 'noetic' : ['focal'] , 'foxy' : ['focal'] , 'galactic' : ['focal'] , - 'rolling' : ['focal']] + 'rolling' : ['jammy']] // This should be in sync with archive_library static gz_version_by_rosdistro = [ 'melodic' : ['9'] , diff --git a/jenkins-scripts/lib/dependencies_archive.sh b/jenkins-scripts/lib/dependencies_archive.sh index 7a5b91b63..6d0531ec9 100644 --- a/jenkins-scripts/lib/dependencies_archive.sh +++ b/jenkins-scripts/lib/dependencies_archive.sh @@ -246,9 +246,16 @@ else case ${GAZEBO_VERSION_FOR_ROS} in *) - # both packages see http://answers.ros.org/question/217970 - _GZ_ROS_PACKAGES="libgazebo${GAZEBO_VERSION_FOR_ROS}-dev \\ - gazebo${GAZEBO_VERSION_FOR_ROS}" + # Rolling is supported only on Jammy. Gazebo11 packages from Jammy + # are coming directly from Ubuntu repositories, package names are + # unversioned + if [[ ${DISTRO} == 'jammy' ]]; then + _GZ_ROS_PACKAGES="libgazebo-dev gazebo" + else + # both packages see http://answers.ros.org/question/217970 + _GZ_ROS_PACKAGES="libgazebo${GAZEBO_VERSION_FOR_ROS}-dev \\ + gazebo${GAZEBO_VERSION_FOR_ROS}" + fi ;; esac From 1302f7e80666ee6be9f78aea8b91cc00f59e4770 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 6 May 2022 14:28:59 +0200 Subject: [PATCH 137/540] Monitor the new gz- names in github repositories (#721) --- jenkins-scripts/dsl/_configs_/GenericAnyJobGitHub.groovy | 2 ++ jenkins-scripts/dsl/_configs_/Globals.groovy | 6 ++++++ jenkins-scripts/dsl/_configs_/OSRFGitHub.groovy | 2 ++ 3 files changed, 10 insertions(+) diff --git a/jenkins-scripts/dsl/_configs_/GenericAnyJobGitHub.groovy b/jenkins-scripts/dsl/_configs_/GenericAnyJobGitHub.groovy index 5cc78732c..fe768ac38 100644 --- a/jenkins-scripts/dsl/_configs_/GenericAnyJobGitHub.groovy +++ b/jenkins-scripts/dsl/_configs_/GenericAnyJobGitHub.groovy @@ -25,6 +25,8 @@ class GenericAnyJobGitHub // Get repo name for relativeTargetDirectory String github_repo_name = github_repo.substring(github_repo.lastIndexOf("/") + 1) + // Use new gz- repositories + github_repo = Globals.ign2gz(github_repo) job.with { diff --git a/jenkins-scripts/dsl/_configs_/Globals.groovy b/jenkins-scripts/dsl/_configs_/Globals.groovy index dd439dde0..ee82588aa 100644 --- a/jenkins-scripts/dsl/_configs_/Globals.groovy +++ b/jenkins-scripts/dsl/_configs_/Globals.groovy @@ -30,6 +30,12 @@ class Globals 'galactic' : ['11'] , 'rolling' : ['11']] + static String ign2gz(String str) { + str = str.replaceAll("ignitionrobotics","gazebosim") + str = str.replaceAll("ign-gazebo","gz-sim") + return str.replaceAll("ign-","gz-") + } + static ArrayList get_ros_distros_by_ubuntu_distro(String ubuntu_distro) { ArrayList result = [] diff --git a/jenkins-scripts/dsl/_configs_/OSRFGitHub.groovy b/jenkins-scripts/dsl/_configs_/OSRFGitHub.groovy index da04ec43a..3ead839bb 100644 --- a/jenkins-scripts/dsl/_configs_/OSRFGitHub.groovy +++ b/jenkins-scripts/dsl/_configs_/OSRFGitHub.groovy @@ -13,6 +13,8 @@ class OSRFGitHub if (subdir == 'NOT-DEFINED-USE-DEFAULT') subdir = software_name + repo = Globals.ign2gz(repo) + job.with { scm { From b213ef89f81bd4320e13c4ec06342613a28a26a4 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 6 May 2022 19:05:50 +0200 Subject: [PATCH 138/540] Use gazebo-forks instead of ignition-forks (#722) --- jenkins-scripts/dsl/extra.dsl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/extra.dsl b/jenkins-scripts/dsl/extra.dsl index e89be2500..2a395aceb 100644 --- a/jenkins-scripts/dsl/extra.dsl +++ b/jenkins-scripts/dsl/extra.dsl @@ -61,7 +61,7 @@ gbp_repo_debbuilds.each { software -> scm { git { remote { - github("ignition-forks/${software}-release", 'https') + github("gazebo-forks/${software}-release", 'https') branch('${BRANCH}') } From 956ef284c52bd6261b46ac7c37f25071b663d70d Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 12 May 2022 01:34:07 +0200 Subject: [PATCH 139/540] Document release-repo scripts (#716) * Document release-repo-scripts Signed-off-by: Jose Luis Rivero * Move scripts to source-repo-scripts * Move script description to source-repo-scripts * Document source_changelog --- README.md | 5 +- release-repo-scripts/README.md | 111 ++++++++++++++++++ .../new_ignition_release_repos.bash | 2 +- source-repo-scripts/README.md | 65 ++++++++++ .../bump_dependency.bash | 0 .../source_changelog.bash | 0 6 files changed, 179 insertions(+), 4 deletions(-) create mode 100644 release-repo-scripts/README.md create mode 100644 source-repo-scripts/README.md rename {release-repo-scripts => source-repo-scripts}/bump_dependency.bash (100%) rename source_changelog.bash => source-repo-scripts/source_changelog.bash (100%) mode change 100644 => 100755 diff --git a/README.md b/README.md index d06427cc9..c898da678 100644 --- a/README.md +++ b/README.md @@ -7,9 +7,8 @@ Gazebo Classic and Ignition software. * [release.py](release.py): Triggers new debian and homebrew releases (major, minor, patch, pre-release...). * [source_changelog.bash](source_changelog.bash): Generates Changelog.md entries newer than a specified tag based on git commit descriptions. -* [changelog_spawn.sh](release-repo-scripts/changelog_spawn.sh): Adds changelog information to `*-release` repositories. Must be used before running `release.py`. -* [new_ignition_release_repos.bash](release-repo-scripts/new_ignition_release_repos.bash): Create new `*-release` repositories. -* [bump_dependency.bash](release-repo-scripts/bump_dependency.bash): Makes all the updates needed when bumping a library's major version for the upcoming collection. + +* For -release repository scripts, please see [README.md](release-repo-scripts/README.md) ### Making releases diff --git a/release-repo-scripts/README.md b/release-repo-scripts/README.md new file mode 100644 index 000000000..2df2a842e --- /dev/null +++ b/release-repo-scripts/README.md @@ -0,0 +1,111 @@ +# -release repositories scripts + +The directory hosts the scripts designed to work with Gazebo -release +repositories, the release repositories hosting the metadata information for +packaging Debian and Ubuntu. Mostly hosted under +https://github.com/gazebo-release/. + +## Scripts available + +### bump_major_version.bash + +The script will bump the major version of a library for the packaging files +inside a -release repository. Mostly designed to be run when a new -release +repository is forked for a new major release. + +*Note: this script is being used from new_ignition_release_repos script in this +repository which also creates new repositories.* + +#### Usage + +Usually forking an existing -release repository, clone it and navigate to the +local clone using a shell is the step zero. + +Require `debchange` command from the package `devscripts`. `debchange` requires +`DEBEMAIL` and `DEBFULLNAME` environment variables to be set for using it. + +```bash +cd +bump_major_version.bash +``` +Changes required involve file names and code inside the configuration files. +The script will create an initial changelog entry with a nightly version lower +than any other expected in the repository. + +#### Example + +To bump `ign-rendering6` and create a new `ign-rendering7` release repositories, +first fork the `ign-rendering6` in GitHub. + +```bash +git clone https://github.com/gz-release/gz-rendering6-release +cd gz-rendering6-release +./path/to/release-tools/release-repo-scripts/bump_major_version 6 7 +``` + +### changelog_spawn.sh + +Update all changelogs inside a -release repository to an specific new version. +Used while releasing a new version of any of the libraries or software. + +#### Usage + +Require `debchange` command from the package `devscripts`. `debchange` requires +`DEBEMAIL` and `DEBFULLNAME` environment variables to be set for using it. + +The new version needs to be in the form of `X.Y.Z-R` Where `X.Y.Z` is the version +of the code and `R` is the revision corresponding to the version of packaging +metadata. + +```bash +cd +changelog_spawn.sh +``` + +#### Example + +To bump `ign-rendering7` from the version `7.0.0-1` to prepare the new release of +`7.1.0`: + +```bash +git clone https://github.com/gz-release/gz-rendering7-release +cd gz-rendering7-release +./path/to/release-tools/release-repo-scripts/changelog_spawn.sh 7.1.0-1 +``` +### new_ignition_release_repos.bash + +The script will create new -release repositories as forks from the previous version +and run the `bump_major_version` script in the repository to update all files and +packaging metadata to the new version. + +#### Usage + +Requires the 'gh' CLI to be installed and the requirements of the `bump_major_version` +script in this repository. + +```bash +cd /tmp/ +./path/to/release-tools/release-repo-scripts/new_ignition_release_repos.bash +``` + +`list_of_new_ignition_names_space_separated` is composed from one or more +repository names to be created. + +Sequence of actions expected: + * Create ign-fooY-release if it does not exits + * Clone to the current directory + * Pull from main/master branch from previous version, ign-fooX-release repository to have the whole history + * Run the bump_major_version from X to Y + * Show the results + * Commit/push all the changes + +The -release repositories used by this tool will be created on current directory. + +#### Example + +To create `ign-rendering7-release` repository copying files in `ign-rendering6-release`: + +```bash +cd /tmp/ +./path/to/release-tools/release-repo-scripts/new_ignition_release_repos.bash ign-rendering7 +``` diff --git a/release-repo-scripts/new_ignition_release_repos.bash b/release-repo-scripts/new_ignition_release_repos.bash index bb1e9d36c..3d7f151ed 100755 --- a/release-repo-scripts/new_ignition_release_repos.bash +++ b/release-repo-scripts/new_ignition_release_repos.bash @@ -10,7 +10,7 @@ set -e SCRIPT_DIR="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" if [[ $# -lt 1 ]]; then - echo "$0 " echo " example: $0 ign-cmake3 ign-common22" exit 1 fi diff --git a/source-repo-scripts/README.md b/source-repo-scripts/README.md new file mode 100644 index 000000000..4f13d6898 --- /dev/null +++ b/source-repo-scripts/README.md @@ -0,0 +1,65 @@ +# Source repositories scripts + +The directory hosts the scripts designed to work with Gazebo source +repositories, real code repositories. Mostly hosted under +https://github.com/gazebosim/ GitHub organization. + +Some of them could also do another kind of operations outside of +this repositories (like updated -release repositories info). + +## Scripts available + +### bump_dependency.bash + +The script will bump the major version of a library for a given collection: + +#### Usage + +Requires the 'gh' CLI to be installed. +```bash +bump_dependency.bash ; ; [] +``` + +The `docs_branch` parameter is optional and defaults to `master` if not specified. + +The script clones all the necessary repositories under /tmp/bump_dependency. + +Before committing to each repository, the script asks "Commit ?". Before saying yes, navigate to the repository and check if the diff +looks reasonable. When you say yes, the changes will be committed and pushed. +Click on the link printed by GitHub to open the pull request. + +#### Example + +To bump to `gz-rendering6` and all its dependencies, as well as `sdf12` and +its dependencies on fortress using the `chapulina/fortress` branch for `docs`: + +```bash +./bump_dependency.bash fortress "gz-rendering;sdformat" "6;12" 428 edifice +"chapulina/fortress" +``` + +### source_changelog.bash + +The script will generate updates for `Changelog.md` files useful when preparing a +new release. + +#### Usage + +Go to the source directory containing a local checkout of a Gazebo repository: + +``` +source_changelog.bash +``` + +The script will parse git log entries and generate Markdown as console output +ready to be injected to `Changelog.md` files. + +### Example + +To update the Changelog for `gz-math` new version 6.11.0: + +``` +cd gz-math +~/release-tools/source-repo-scripts/source_changelog.bash 6.10.0 +``` diff --git a/release-repo-scripts/bump_dependency.bash b/source-repo-scripts/bump_dependency.bash similarity index 100% rename from release-repo-scripts/bump_dependency.bash rename to source-repo-scripts/bump_dependency.bash diff --git a/source_changelog.bash b/source-repo-scripts/source_changelog.bash old mode 100644 new mode 100755 similarity index 100% rename from source_changelog.bash rename to source-repo-scripts/source_changelog.bash From d64769b4b4bd2e728b08bb365f50a8c3120d009d Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 14 Jan 2022 19:21:52 +0100 Subject: [PATCH 140/540] New experimental script to have new distros in -release repos --- release-repo-scripts/new_distribution.bash | 176 +++++++++++++++++++++ 1 file changed, 176 insertions(+) create mode 100644 release-repo-scripts/new_distribution.bash diff --git a/release-repo-scripts/new_distribution.bash b/release-repo-scripts/new_distribution.bash new file mode 100644 index 000000000..99ec56d5a --- /dev/null +++ b/release-repo-scripts/new_distribution.bash @@ -0,0 +1,176 @@ +#!/bin/bash + +# Create a new distribution inside a -release repo by copying from previous + +set -e + +# SCRIPT_DIR="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" + +if [[ $# -lt 1 ]]; then + echo "$0 " + echo " example: $0 ammy focal" + exit 1 +fi + +NEW_DISTRO=${1} + +get_latest_distribution_in_dir() +{ + ls | sort | grep -v ubuntu | grep -v debian | tail -1 +} + +get_source_package_name() +{ + dpkg-parsechangelog -S Source +} + +latest_distro=$(get_latest_distribution_in_dir) +if [[ ! -d ${latest_distro} ]]; then + echo "Failed to detect latest distribution dir: ${latest_distro}" + exit 1 +fi + +get_root_repo_dir() +{ + root_check=${PWD} + + if [[ -d ${root_check}/ubuntu ]]; then + echo "${root_check}" + elif [[ -d ${root_check}/../ubuntu ]]; then + echo "${root_check}/../" + else + echo "Failed to detect root_dir" + exit 1 + fi +} + +list_packages_in_repo() +{ + control_file=${1} src_package=${2} + + grep-dctrl -n -s Package --field Package "${src_package}" ${control_file} | sort +} + +echo -n " + Detecting package release-repo root: " +root_dir=$(get_root_repo_dir) +echo "${root_dir}" +echo -n " + Detecting package source name: " +pushd "${latest_distro}" >/dev/null +source_pkg=$(get_source_package_name) +echo "${source_pkg}" +source_pkg_debian=${source_pkg//[0-9]/} +debian_repo=/tmp/${source_pkg_debian} +echo " + Clone debian repository ${debian_repo}" +if [[ ! -d ${debian_repo} ]]; then + git clone "https://salsa.debian.org/science-team/${source_pkg_debian}" "${debian_repo}" +fi +popd >/dev/null + +echo " + Creating ${NEW_DISTRO} from ${latest_distro}" +cp -a "${latest_distro}" "${NEW_DISTRO}" + +pushd "${NEW_DISTRO}" >/dev/null +echo " -- Changes:" +echo " - Update changelog" +dch --distribution="${NEW_DISTRO}" \ + --force-distribution \ + --check-dirname-level=0 \ + "First release for ${NEW_DISTRO}" +sed -i -e "s:~${latest_distro}.*):~${NEW_DISTRO}):" debian/changelog +head -n 5 debian/changelog > debian/changelog.new +mv debian/changelog.new debian/changelog + +echo " - Remove not needed format file" +rm -f debian/format + +echo " - Import watch file" +cp "${debian_repo}/debian/watch" "${root_dir}/ubuntu/debian/watch" +git add "${root_dir}/ubuntu/debian/watch" +ln -s "${root_dir}/ubuntu/debian/watch" debian/watch +git add debian/watch + +echo " - Replace Vcs info" +sed -i -e "s:^Vcs-Browser.*:Vcs-Browser\: https\://github.com/ignitionrobotics/${source_pkg_debian}:" \ + "${root_dir}/ubuntu/debian/control" +sed -i -e "s:^Vcs-Hg.*:Vcs-Git\: https\://github.com/ignitionrobotics/${source_pkg_debian}.git:" \ + "${root_dir}/ubuntu/debian/control" + +echo " - Set compat 13" +rm -f debian/compat +echo "13" > debian/compat +git add debian/compat + +# Introduce standards. Bionic is in 4.1.4. Focal in 4.5.0 +echo " - Set standards to 4.5.1" +sed -i -e '/^Homepage: */i Standards-Version: 4.5.1' "${root_dir}/ubuntu/debian/control" +# Bionic has debhelpert 11. Focal and Buster 12. +sed -i -e 's:debhelper (>= 9):debhelper (>= 11):' "${root_dir}/ubuntu/debian/control" + +echo " - Import new copyright" +rm -f debian/copyright +new_copyright_path="${root_dir}/ubuntu/debian/debian_copyright_2022" +cp "${debian_repo}/debian/copyright" "${new_copyright_path}" +git add "${new_copyright_path}" +ln -s "${new_copyright_path}" debian/copyright + +echo " - Import autopkgtest" +autopkgtest_path="${root_dir}/ubuntu/debian/tests/" +mkdir -p "${autopkgtest_path}" +cp -a "${debian_repo}"/debian/tests/* "${autopkgtest_path}" +git add "${autopkgtest_path}" +ln -s "${autopkgtest_path}" debian/tests +git add debian/tests + +# Import docs file if found +if [[ -f ${debian_repo}/debian/docs ]]; then + echo " - Import docs file" + cp "${debian_repo}/debian/docs" "${root_dir}/ubuntu/debian/" + git add "${root_dir}/ubuntu/debian/docs" + ln -s "${root_dir}/ubuntu/debian/docs" debian/docs + git add debian/docs +fi + +# Import examples files. Asumming just one file +for f in ${debian_repo}/debian/*.examples; do + echo " - Import examples file ${f}" + cp "${f}" "${root_dir}/ubuntu/debian/" + git add "${root_dir}"/ubuntu/debian/*.examples + ln -s "$(ls "${root_dir}"/ubuntu/debian/*.examples)" debian/ + # use versioned dev if needed + if [[ -f $(ls debian/*-dev*.examples) ]]; then + examples_file=${f##*/} + mv debian/${examples_file} debian/${examples_file/${source_pkg_debian}/${source_pkg}} + git add debian/*.examples + fi +done + +# Check broken symlinks +echo " - Check broken symlinks" +find debian/ -xtype l +# Finally add new files +git add debian/* + +# Info sections +BLUE='\033[1;34m' +NC='\033[0m\n' # No Color +echo +printf "${BLUE}INFO in packages generated${NC}" +or_pkgs_file=/tmp/or_pkgs_file +echo "=Open Robotics=" > ${or_pkgs_file} +list_packages_in_repo "debian/control" >> ${or_pkgs_file} +debian_pkgs_file=/tmp/debian_pkgs_file +echo "=Debian=" > ${debian_pkgs_file} +list_packages_in_repo "${debian_repo}/debian/control" >> ${debian_pkgs_file} +pr -Tm ${or_pkgs_file} ${debian_pkgs_file} +echo +printf "${BLUE}INFO: diff in rules${NC}" +diff -u --color debian/rules "${debian_repo}/debian/rules" || true +echo +printf "${BLUE}INFO: patches in debian${NC}" +ls "${debian_repo}/debian/patches/" +echo +printf "${BLUE}INFO: diff in debian directories${NC}" +diff -qr debian "${debian_repo}/debian" || true +echo +printf "${BLUE}INFO: description in Debian for -dev package${NC}" +grep-dctrl -n -s Description --field=Package "${source_pkg_debian}" "${debian_repo}/debian/control" From 6dc1f706e3d34601af30f178701c0a06990d9b62 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 10 Feb 2022 18:42:42 +0100 Subject: [PATCH 141/540] Use relative paths to make symlinks happy --- release-repo-scripts/new_distribution.bash | 29 +++++++++++++++++----- 1 file changed, 23 insertions(+), 6 deletions(-) mode change 100644 => 100755 release-repo-scripts/new_distribution.bash diff --git a/release-repo-scripts/new_distribution.bash b/release-repo-scripts/new_distribution.bash old mode 100644 new mode 100755 index 99ec56d5a..da778bd1f --- a/release-repo-scripts/new_distribution.bash +++ b/release-repo-scripts/new_distribution.bash @@ -44,6 +44,21 @@ get_root_repo_dir() fi } +get_relative_path() +{ + common_file=${1} + + pushd debian >/dev/null + ubuntu_distros="../../ubuntu/debian/${common_file}" + if [[ -e ${ubuntu_distros} ]]; then + echo ${ubuntu_distros} + else + # debian is two level ups + echo "../ubuntu/debian/${common_file}" + fi + popd >/dev/null +} + list_packages_in_repo() { control_file=${1} src_package=${2} @@ -86,7 +101,7 @@ rm -f debian/format echo " - Import watch file" cp "${debian_repo}/debian/watch" "${root_dir}/ubuntu/debian/watch" git add "${root_dir}/ubuntu/debian/watch" -ln -s "${root_dir}/ubuntu/debian/watch" debian/watch +ln -sf $(get_relative_path watch) debian/watch git add debian/watch echo " - Replace Vcs info" @@ -102,6 +117,7 @@ git add debian/compat # Introduce standards. Bionic is in 4.1.4. Focal in 4.5.0 echo " - Set standards to 4.5.1" +sed -i -e '/Standards-Version:.*/d' "${root_dir}/ubuntu/debian/control" sed -i -e '/^Homepage: */i Standards-Version: 4.5.1' "${root_dir}/ubuntu/debian/control" # Bionic has debhelpert 11. Focal and Buster 12. sed -i -e 's:debhelper (>= 9):debhelper (>= 11):' "${root_dir}/ubuntu/debian/control" @@ -111,14 +127,14 @@ rm -f debian/copyright new_copyright_path="${root_dir}/ubuntu/debian/debian_copyright_2022" cp "${debian_repo}/debian/copyright" "${new_copyright_path}" git add "${new_copyright_path}" -ln -s "${new_copyright_path}" debian/copyright +ln -sf $(get_relative_path copyright) debian/copyright echo " - Import autopkgtest" autopkgtest_path="${root_dir}/ubuntu/debian/tests/" mkdir -p "${autopkgtest_path}" cp -a "${debian_repo}"/debian/tests/* "${autopkgtest_path}" git add "${autopkgtest_path}" -ln -s "${autopkgtest_path}" debian/tests +ln -sf $(get_relative_path tests) debian/tests git add debian/tests # Import docs file if found @@ -126,16 +142,17 @@ if [[ -f ${debian_repo}/debian/docs ]]; then echo " - Import docs file" cp "${debian_repo}/debian/docs" "${root_dir}/ubuntu/debian/" git add "${root_dir}/ubuntu/debian/docs" - ln -s "${root_dir}/ubuntu/debian/docs" debian/docs + ln -sf $(get_relative_path docs) debian/docs git add debian/docs fi # Import examples files. Asumming just one file -for f in ${debian_repo}/debian/*.examples; do +shopt -s nullglob +for f in "${debian_repo}"/debian/*.examples; do echo " - Import examples file ${f}" cp "${f}" "${root_dir}/ubuntu/debian/" git add "${root_dir}"/ubuntu/debian/*.examples - ln -s "$(ls "${root_dir}"/ubuntu/debian/*.examples)" debian/ + ln -sf $(get_relative_path $f) debian/$f # use versioned dev if needed if [[ -f $(ls debian/*-dev*.examples) ]]; then examples_file=${f##*/} From 3ca80470516d7d9450b7ce8f7bc85517f8fcad58 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 13 May 2022 19:11:30 +0200 Subject: [PATCH 142/540] Fixes for the script --- ...tion.bash => new_ubuntu_distribution.bash} | 44 ++++++++++--------- 1 file changed, 24 insertions(+), 20 deletions(-) rename release-repo-scripts/{new_distribution.bash => new_ubuntu_distribution.bash} (81%) diff --git a/release-repo-scripts/new_distribution.bash b/release-repo-scripts/new_ubuntu_distribution.bash similarity index 81% rename from release-repo-scripts/new_distribution.bash rename to release-repo-scripts/new_ubuntu_distribution.bash index da778bd1f..83ad4ccfd 100755 --- a/release-repo-scripts/new_distribution.bash +++ b/release-repo-scripts/new_ubuntu_distribution.bash @@ -4,11 +4,9 @@ set -e -# SCRIPT_DIR="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" - if [[ $# -lt 1 ]]; then echo "$0 " - echo " example: $0 ammy focal" + echo " example: $0 jammy" exit 1 fi @@ -105,9 +103,11 @@ ln -sf $(get_relative_path watch) debian/watch git add debian/watch echo " - Replace Vcs info" -sed -i -e "s:^Vcs-Browser.*:Vcs-Browser\: https\://github.com/ignitionrobotics/${source_pkg_debian}:" \ +sed -i -e "s:^Vcs-Browser.*:Vcs-Browser\: https\://github.com/gazebosim/${source_pkg_debian}:" \ + "${root_dir}/ubuntu/debian/control" +sed -i -e "s:^Vcs-Hg.*:Vcs-Git\: https\://github.com/gazebosim/${source_pkg_debian}.git:" \ "${root_dir}/ubuntu/debian/control" -sed -i -e "s:^Vcs-Hg.*:Vcs-Git\: https\://github.com/ignitionrobotics/${source_pkg_debian}.git:" \ +sed -i -e 's:github.com/ignitionrobotics:github.com/gazebosim:g' \ "${root_dir}/ubuntu/debian/control" echo " - Set compat 13" @@ -129,13 +129,15 @@ cp "${debian_repo}/debian/copyright" "${new_copyright_path}" git add "${new_copyright_path}" ln -sf $(get_relative_path copyright) debian/copyright -echo " - Import autopkgtest" autopkgtest_path="${root_dir}/ubuntu/debian/tests/" -mkdir -p "${autopkgtest_path}" -cp -a "${debian_repo}"/debian/tests/* "${autopkgtest_path}" -git add "${autopkgtest_path}" -ln -sf $(get_relative_path tests) debian/tests -git add debian/tests +if [[ ! -d ${autopkgtest_path} ]]; then + echo " - Import autopkgtest" + mkdir -p "${autopkgtest_path}" + cp -a "${debian_repo}"/debian/tests/* "${autopkgtest_path}" + git add "${autopkgtest_path}" + ln -sf $(get_relative_path tests) debian/tests + git add debian/tests +fi # Import docs file if found if [[ -f ${debian_repo}/debian/docs ]]; then @@ -149,15 +151,17 @@ fi # Import examples files. Asumming just one file shopt -s nullglob for f in "${debian_repo}"/debian/*.examples; do - echo " - Import examples file ${f}" - cp "${f}" "${root_dir}/ubuntu/debian/" - git add "${root_dir}"/ubuntu/debian/*.examples - ln -sf $(get_relative_path $f) debian/$f - # use versioned dev if needed - if [[ -f $(ls debian/*-dev*.examples) ]]; then - examples_file=${f##*/} - mv debian/${examples_file} debian/${examples_file/${source_pkg_debian}/${source_pkg}} - git add debian/*.examples + if [[ ! -f ${f} ]]; then + echo " - Import examples file ${f}" + cp "${f}" "${root_dir}/ubuntu/debian/" + git add "${root_dir}"/ubuntu/debian/*.examples + #ln -sf $(get_relative_path $f) debian/${examples_file} + # use versioned dev if needed + if [[ -f $(ls debian/*-dev*.examples) ]]; then + examples_file=${f##*/} + mv debian/${examples_file} debian/${examples_file/${source_pkg_debian}/${source_pkg}} + git add debian/*.examples + fi fi done From 738aa6f7027bdcf92ee26651696284fefc789639 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 13 May 2022 19:11:51 +0200 Subject: [PATCH 143/540] Add documentation --- release-repo-scripts/README.md | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/release-repo-scripts/README.md b/release-repo-scripts/README.md index 2df2a842e..e4ec4d349 100644 --- a/release-repo-scripts/README.md +++ b/release-repo-scripts/README.md @@ -72,6 +72,39 @@ git clone https://github.com/gz-release/gz-rendering7-release cd gz-rendering7-release ./path/to/release-tools/release-repo-scripts/changelog_spawn.sh 7.1.0-1 ``` +### new_ubuntu_distribution.bash + +The script will create a new directory to host a new Ubuntu distribution inside +the -release repository where it is being run. The new directory will be mostly +a copy of the latest Ubuntu distribution available. + +A local clone of the same Debian official package is being done in /tmp to import +some changes from it and produce a diff between the metadata in Debian official +and the one in the local repo. + +#### Usage + +Require the package `dctrl-tools` installed. The script will perform different +operations to update different metadata. + +Some git add operations are performed but there won't be a commit or a permanent +change. + +```bash +cd +./new_ubuntu_distribution +``` + +#### Example + +To create the kinetic Ubuntu distribution inside `gz-cmake3-release`: + +```bash +git clone https://github.com/gz-release/gz-cmake3-release +cd gz-cmake3-release +./path/to/release-tools/release-repo-scripts/new_ubuntu_distribution.bash kinetic +``` + ### new_ignition_release_repos.bash The script will create new -release repositories as forks from the previous version From 36ace7a3a752ea60737249783ff2de0760b81470 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Mon, 16 May 2022 18:31:54 +0200 Subject: [PATCH 144/540] Replace ignition- github organizations (#723) * Replace ignition- github organizations * Move scripts to source-repo-scripts * Document release-repo scripts (#716) * Document release-repo-scripts * Document source_changelog --- .github/ISSUE_TEMPLATE/bump_libraries.md | 6 +++--- .github/ISSUE_TEMPLATE/eol_collection.md | 8 ++++---- .github/ISSUE_TEMPLATE/new_collection.md | 6 +++--- .github/ISSUE_TEMPLATE/release_collection.md | 14 ++++++------- README.md | 6 +++--- docs/BUILDFARM_NODES.md | 2 +- .../lib/docker_generate_dockerfile.bash | 2 +- .../dsl/_configs_/GenericAnyJobGitHub.groovy | 2 +- .../dsl/_configs_/OSRFLinuxCompilation.groovy | 4 ++-- .../dsl/_configs_/OSRFUNIXBase.groovy | 2 +- .../dsl/_configs_/OSRFWinBase.groovy | 2 +- jenkins-scripts/dsl/brew_release.dsl | 2 +- jenkins-scripts/dsl/core.dsl | 2 +- jenkins-scripts/dsl/gazebo.dsl | 2 +- jenkins-scripts/dsl/gzdev.dsl | 4 ++-- jenkins-scripts/dsl/ignition.dsl | 16 +++++++-------- jenkins-scripts/dsl/ignition_collection.dsl | 2 +- jenkins-scripts/dsl/ros1_ign_bridge.dsl | 2 +- jenkins-scripts/dsl/sdformat.dsl | 20 +++++++++---------- jenkins-scripts/dsl/test.dsl | 2 +- .../gazebo-one_liner-homebrew.bash | 2 +- jenkins-scripts/lib/windows_library.bat | 2 +- one-line-installations/gazebo.sh | 2 +- release.py | 2 +- source-repo-scripts/bump_dependency.bash | 6 +++--- terminal-dashboard/_dashboard_lib.sh | 2 +- 26 files changed, 61 insertions(+), 61 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bump_libraries.md b/.github/ISSUE_TEMPLATE/bump_libraries.md index ab41324fb..29c40ba82 100644 --- a/.github/ISSUE_TEMPLATE/bump_libraries.md +++ b/.github/ISSUE_TEMPLATE/bump_libraries.md @@ -15,7 +15,7 @@ instead. This script can be used to do most of the work: -https://github.com/ignition-tooling/release-tools/blob/master/release-repo-scripts/bump_dependency.bash +https://github.com/gazebo-tooling/release-tools/blob/master/release-repo-scripts/bump_dependency.bash When opening PRs, add a link back to this issue for easier tracking. @@ -28,8 +28,8 @@ Libraries being bumped: * : - [ ] Source code of all downstream libraries ([example](https://github.com/ignitionrobotics/ign-transport/pull/149)) - [ ] Release repositories of all downstream libraries ([example](https://github.com/ignition-release/ign-fuel-tools7-release/pull/1)) - - [ ] Use `main` branch on `gazebodistro` ([example](https://github.com/ignition-tooling/gazebodistro/pull/42)) - - [ ] Build nightlies from the `main` branch ([example](https://github.com/ignition-tooling/release-tools/pull/437)) + - [ ] Use `main` branch on `gazebodistro` ([example](https://github.com/gazebo-tooling/gazebodistro/pull/42)) + - [ ] Build nightlies from the `main` branch ([example](https://github.com/gazebo-tooling/release-tools/pull/437)) - [ ] homebrew-simulation: create formula and update dependencies ([example](https://github.com/osrf/homebrew-simulation/pull/14230)) - [ ] docs (the collection’s page) ([example](https://github.com/ignitionrobotics/docs/pull/175)) * diff --git a/.github/ISSUE_TEMPLATE/eol_collection.md b/.github/ISSUE_TEMPLATE/eol_collection.md index 26e7cc085..a2b9c27dd 100644 --- a/.github/ISSUE_TEMPLATE/eol_collection.md +++ b/.github/ISSUE_TEMPLATE/eol_collection.md @@ -20,7 +20,7 @@ When opening PRs, add a link back to this issue for easier tracking. - [ ] Make releases with outstanding changes in each library about to be EOL'ed * - [ ] Archive release repos in [ignition-release](https://github.com/ignition-release) for all libraries after the last release has been made -- [ ] Update `release-tools` after all releases have been made ([example](https://github.com/ignition-tooling/release-tools/pull/369)) +- [ ] Update `release-tools` after all releases have been made ([example](https://github.com/gazebo-tooling/release-tools/pull/369)) * - [ ] Deprecate homebrew bottles if they aren't deprecated yet (don't disable them) ([example](https://github.com/osrf/homebrew-simulation/pull/1785)) * @@ -28,15 +28,15 @@ When opening PRs, add a link back to this issue for easier tracking. * - [ ] Remove from buildfarmer ([example](https://github.com/osrf/homebrew-simulation/pull/1785)) * -- [ ] Remove from `pr-collection-labeler` ([example](https://github.com/ignition-tooling/pr-collection-labeler/pull/13)) +- [ ] Remove from `pr-collection-labeler` ([example](https://github.com/gazebo-tooling/pr-collection-labeler/pull/13)) * - [ ] Remove from `ros_ign` ([example](https://github.com/ignitionrobotics/ros_ign/pull/199)) * - [ ] Sync latest binaries to http://packages.ros.org ([example](https://github.com/ros-infrastructure/reprepro-updater/pull/145)) * -- [ ] Remove from `gzdev`'s `ign_docker_env` ([example](https://github.com/ignition-tooling/gzdev/pull/56)) +- [ ] Remove from `gzdev`'s `ign_docker_env` ([example](https://github.com/gazebo-tooling/gzdev/pull/56)) * -- [ ] Close " support" ticket after final releases ([example](https://github.com/ignition-tooling/release-tools/issues/297#issuecomment-1002232980)) +- [ ] Close " support" ticket after final releases ([example](https://github.com/gazebo-tooling/release-tools/issues/297#issuecomment-1002232980)) * - [ ] Update docs after EOL is complete ([example](https://github.com/ignitionrobotics/docs/pull/124)) * diff --git a/.github/ISSUE_TEMPLATE/new_collection.md b/.github/ISSUE_TEMPLATE/new_collection.md index 394bd2231..88e3576d0 100644 --- a/.github/ISSUE_TEMPLATE/new_collection.md +++ b/.github/ISSUE_TEMPLATE/new_collection.md @@ -13,13 +13,13 @@ When opening PRs, add a link back to this issue for easier tracking. --> -- [ ] Create `gazebodistro` file for new collection ([example](https://github.com/ignition-tooling/gazebodistro/pull/11)) +- [ ] Create `gazebodistro` file for new collection ([example](https://github.com/gazebo-tooling/gazebodistro/pull/11)) - [ ] Create `ign-` and `ign--release` repositories ([example source](https://github.com/ignitionrobotics/ign-edifice), [example release](https://github.com/ignition-release/ign-edifice-release)) - [ ] Create homebrew-simulation formula for new collection ([example](https://github.com/osrf/homebrew-simulation/pull/1173)) -- [ ] Create collection view on release-tools and build nightlies ([example](https://github.com/ignition-tooling/release-tools/pull/422/files)) +- [ ] Create collection view on release-tools and build nightlies ([example](https://github.com/gazebo-tooling/release-tools/pull/422/files)) - [ ] Add collection to buildfarmer dashboard ([example](https://github.com/osrf/buildfarmer/pull/183)) / scripts ([example](https://github.com/osrf/buildfarmer/pull/184)) - [ ] Add docs for new collection ([example](https://github.com/ignitionrobotics/docs/pull/95)) - [ ] Add new collection label: - - [ ] Add to pr-collection-labeler ([example](https://github.com/ignition-tooling/pr-collection-labeler/pull/11)) + - [ ] Add to pr-collection-labeler ([example](https://github.com/gazebo-tooling/pr-collection-labeler/pull/11)) - [ ] Add to all libraries ([handy script](https://github.com/chapulina/github-label-maker/tree/chapulina/ign#ignition)) - [ ] Add new collection tag to Gazebo Answers [here](https://answers.gazebosim.org/settings/FORUM_DATA_RULES/) diff --git a/.github/ISSUE_TEMPLATE/release_collection.md b/.github/ISSUE_TEMPLATE/release_collection.md index 8b5fea99f..483af6d24 100644 --- a/.github/ISSUE_TEMPLATE/release_collection.md +++ b/.github/ISSUE_TEMPLATE/release_collection.md @@ -33,7 +33,7 @@ When opening PRs, add a link back to this issue for easier tracking. * - [ ] Update public documentation to install from pre-releases instead of nightlies ([example](https://github.com/ignitionrobotics/docs/pull/196/files#diff-ebeee4adce7cb444f663b59020fb9f43f6f9adf36f63657b8afdeeea7a8530d1)). * -- [ ] Change nightlies to pre-release for release branches on `gzdev` ([example](https://github.com/ignition-tooling/gzdev/pull/35)) +- [ ] Change nightlies to pre-release for release branches on `gzdev` ([example](https://github.com/gazebo-tooling/gzdev/pull/35)) * ## Code freeze @@ -52,7 +52,7 @@ When opening PRs, add a link back to this issue for easier tracking. - [ ] Make collection stable release after all libraries are released. * -- [ ] Update `gazebodistro` ([example](https://github.com/ignition-tooling/gazebodistro/pull/31)) +- [ ] Update `gazebodistro` ([example](https://github.com/gazebo-tooling/gazebodistro/pull/31)) * - [ ] Make GitHub releases for all the X.0.0 versions ([example](https://github.com/ignitionrobotics/ign-transport/releases/tag/ignition-transport9_9.0.0)) - [ ] Update docs to say release is stable ([example](https://github.com/ignitionrobotics/docs/pull/171)) @@ -62,19 +62,19 @@ When opening PRs, add a link back to this issue for easier tracking. ## Post-release -- [ ] Enable Jenkins CI for new stable branches ([example](https://github.com/ignition-tooling/release-tools/pull/299)) +- [ ] Enable Jenkins CI for new stable branches ([example](https://github.com/gazebo-tooling/release-tools/pull/299)) * -- [ ] Remove pre-release for release branches on `gzdev` ([example](https://github.com/ignition-tooling/gzdev/pull/36)) +- [ ] Remove pre-release for release branches on `gzdev` ([example](https://github.com/gazebo-tooling/gzdev/pull/36)) * - [ ] Bump all `main` branches to the next major versioni (`X.0.0~pre1`) - [ ] Source code ([example](https://github.com/ignitionrobotics/ign-common/pull/193)) * - - [ ] Add files to `gazebodistro` ([example](https://github.com/ignition-tooling/gazebodistro/pull/12)) + - [ ] Add files to `gazebodistro` ([example](https://github.com/gazebo-tooling/gazebodistro/pull/12)) * - [ ] Add aliases to `homebrew-simulation` ([example](https://github.com/osrf/homebrew-simulation/commit/1f8602af6f52e06e0542eebfbdbe97f5f6cf950c)) * - - [ ] Create new `-release` repositories (use [this script](https://github.com/ignition-tooling/release-tools/blob/master/release-repo-scripts/new_ignition_release_repos.bash)) - - [ ] Enable nightlies for all `main` branches on `gzdev` ([example](https://github.com/ignition-tooling/gzdev/pull/50)) + - [ ] Create new `-release` repositories (use [this script](https://github.com/gazebo-tooling/release-tools/blob/master/release-repo-scripts/new_ignition_release_repos.bash)) + - [ ] Enable nightlies for all `main` branches on `gzdev` ([example](https://github.com/gazebo-tooling/gzdev/pull/50)) - [ ] Execute the tick-tock's "tock" for deprecations ([example](https://github.com/ignitionrobotics/ign-gazebo/pull/875)) * - [ ] Update all repositories to default to the new stable branches. diff --git a/README.md b/README.md index c898da678..4579ec1fe 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ Gazebo Classic and Ignition software. * [Example pull request.](https://github.com/ignition-release/ign-gazebo3-release/pull/4) 1. Wait for all pull requests opened above to be approved and merged. 1. Clone `release-tools` - * `git clone https://github.com/ignition-tooling/release-tools` + * `git clone https://github.com/gazebo-tooling/release-tools` 1. Update the release-repo's changelog: 1. `cd ` 1. `/release-repo-scripts/changelog_spawn.sh ` @@ -69,10 +69,10 @@ Gazebo Classic and Ignition software. 1. If the dry run succeeds, run the same command again, now without `--dry-run`. 1. Check that: * Several `-debbuilder` jobs have been queued in https://build.osrfoundation.org/ and watch those jobs to see if any of them fail. - While it would be easier to track the status of these jobs if there was a [dashboard](https://github.com/ignition-tooling/release-tools/issues/295), + While it would be easier to track the status of these jobs if there was a [dashboard](https://github.com/gazebo-tooling/release-tools/issues/295), you can watch the page for a specific debbuild, such as https://build.osrfoundation.org/job/ign-gui3-debbuilder/. 1. If you observe a failure of an Ignition Dome package, check the list of supported - architectures in [issue #297](https://github.com/ignition-tooling/release-tools/issues/297); + architectures in [issue #297](https://github.com/gazebo-tooling/release-tools/issues/297); for example Bionic/armhf and Focal/armhf are not supported for Dome. To disable builds for a specific architecture, add a file to the `-release` repository that starts with `.releasepy_NO_ARCH_` and append the name of the architecture to be excluded. The file can be added to the diff --git a/docs/BUILDFARM_NODES.md b/docs/BUILDFARM_NODES.md index 8b625675c..7f1bb9914 100644 --- a/docs/BUILDFARM_NODES.md +++ b/docs/BUILDFARM_NODES.md @@ -24,7 +24,7 @@ used (can check this in the Jenkins UI) ### Node labels for nightlies -For generating nightlies controlling the generation order of every library, build.osrfoundation.org use the approach of [using a single node to process the whole ignition family for a given Ubuntu distribution](https://github.com/ignition-tooling/release-tools/issues/644). Current assignment of nodes is as follow: +For generating nightlies controlling the generation order of every library, build.osrfoundation.org use the approach of [using a single node to process the whole ignition family for a given Ubuntu distribution](https://github.com/gazebo-tooling/release-tools/issues/644). Current assignment of nodes is as follow: | Label name | Description | | -------- | ----------- | diff --git a/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash b/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash index f1223cc25..1e2613061 100644 --- a/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash +++ b/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash @@ -41,7 +41,7 @@ dockerfile_install_gzdev_repos() { cat >> Dockerfile << DELIM_OSRF_REPO_GIT RUN rm -fr ${GZDEV_DIR} -RUN git clone https://github.com/ignition-tooling/gzdev -b ${GZDEV_BRANCH} ${GZDEV_DIR} +RUN git clone https://github.com/gazebo-tooling/gzdev -b ${GZDEV_BRANCH} ${GZDEV_DIR} RUN if [ -n $GZDEV_TRY_BRANCH ]; then \ git -C ${GZDEV_DIR} fetch origin $GZDEV_TRY_BRANCH || true; \ git -C ${GZDEV_DIR} checkout $GZDEV_TRY_BRANCH || true; \ diff --git a/jenkins-scripts/dsl/_configs_/GenericAnyJobGitHub.groovy b/jenkins-scripts/dsl/_configs_/GenericAnyJobGitHub.groovy index fe768ac38..ecc3cd7d6 100644 --- a/jenkins-scripts/dsl/_configs_/GenericAnyJobGitHub.groovy +++ b/jenkins-scripts/dsl/_configs_/GenericAnyJobGitHub.groovy @@ -54,7 +54,7 @@ class GenericAnyJobGitHub configure { project -> project / triggers / 'org.jenkinsci.plugins.ghprb.GhprbTrigger' { adminlist 'osrf-jenkins j-rivero' - orgslist 'osrf ignitionrobotics' + orgslist 'osrf gazebosim' allowMembersOfWhitelistedOrgsAsAdmin(true) useGitHubHooks(true) onlyTriggerPhrase(false) diff --git a/jenkins-scripts/dsl/_configs_/OSRFLinuxCompilation.groovy b/jenkins-scripts/dsl/_configs_/OSRFLinuxCompilation.groovy index 603772ee4..a8baa6677 100644 --- a/jenkins-scripts/dsl/_configs_/OSRFLinuxCompilation.groovy +++ b/jenkins-scripts/dsl/_configs_/OSRFLinuxCompilation.groovy @@ -19,7 +19,7 @@ class OSRFLinuxCompilation extends OSRFLinuxBase job.with { // preclean of build/*_results. Please see - // https://github.com/ignition-tooling/release-tools/issues/75" + // https://github.com/gazebo-tooling/release-tools/issues/75" // to know why preBuildCleanup is not working and the use // of shell as workaround // wrappers { @@ -34,7 +34,7 @@ class OSRFLinuxCompilation extends OSRFLinuxBase #!/bin/bash -xe echo "Workaround for cleaning up workspace" - echo "check: https://github.com/ignition-tooling/release-tools/issues/75" + echo "check: https://github.com/gazebo-tooling/release-tools/issues/75" sudo rm -fr "\${WORKSPACE}/build/*_results" sudo rm -fr "\${WORKSPACE}/*_results" diff --git a/jenkins-scripts/dsl/_configs_/OSRFUNIXBase.groovy b/jenkins-scripts/dsl/_configs_/OSRFUNIXBase.groovy index 544c28ade..037a59c06 100644 --- a/jenkins-scripts/dsl/_configs_/OSRFUNIXBase.groovy +++ b/jenkins-scripts/dsl/_configs_/OSRFUNIXBase.groovy @@ -35,7 +35,7 @@ class OSRFUNIXBase extends OSRFBase #!/bin/bash -xe [[ -d ./scripts ]] && rm -fr ./scripts - git clone https://github.com/ignition-tooling/release-tools scripts -b \$RTOOLS_BRANCH + git clone https://github.com/gazebo-tooling/release-tools scripts -b \$RTOOLS_BRANCH """.stripIndent()) } } diff --git a/jenkins-scripts/dsl/_configs_/OSRFWinBase.groovy b/jenkins-scripts/dsl/_configs_/OSRFWinBase.groovy index 8971292d7..cd6d8f0fe 100644 --- a/jenkins-scripts/dsl/_configs_/OSRFWinBase.groovy +++ b/jenkins-scripts/dsl/_configs_/OSRFWinBase.groovy @@ -20,7 +20,7 @@ class OSRFWinBase extends OSRFBase { batchFile("""\ IF exist scripts ( rmdir scripts /s /q ) - git clone https://github.com/ignition-tooling/release-tools scripts -b %RTOOLS_BRANCH% + git clone https://github.com/gazebo-tooling/release-tools scripts -b %RTOOLS_BRANCH% """.stripIndent()) } } diff --git a/jenkins-scripts/dsl/brew_release.dsl b/jenkins-scripts/dsl/brew_release.dsl index 240056530..5f1592741 100644 --- a/jenkins-scripts/dsl/brew_release.dsl +++ b/jenkins-scripts/dsl/brew_release.dsl @@ -180,7 +180,7 @@ bottle_job_builder.with } project / triggers / 'org.jenkinsci.plugins.ghprb.GhprbTrigger' { adminlist 'osrf-jenkins j-rivero scpeters' - orgslist 'ignitionrobotics' + orgslist 'gazebosim' whitelist 'osrfbuild' useGitHubHooks(true) allowMembersOfWhitelistedOrgsAsAdmin(true) diff --git a/jenkins-scripts/dsl/core.dsl b/jenkins-scripts/dsl/core.dsl index 20734cac9..f63f32238 100644 --- a/jenkins-scripts/dsl/core.dsl +++ b/jenkins-scripts/dsl/core.dsl @@ -34,7 +34,7 @@ update_vcpkg_snapshot_job.with def ignition_testing_software = 'gazebo' def testing_vcpkg_job = job("_vcpkg_testing_snapshot") OSRFWinCompilationAnyGitHub.create(testing_vcpkg_job, - "ignitionrobotics/ign-${ignition_testing_software}", + "gazebosim/ign-${ignition_testing_software}", NO_TESTING, NO_BRANCHES, NO_GITHUB_PR_INTEGRATION) testing_vcpkg_job.with { diff --git a/jenkins-scripts/dsl/gazebo.dsl b/jenkins-scripts/dsl/gazebo.dsl index af5236628..336fa2d6e 100644 --- a/jenkins-scripts/dsl/gazebo.dsl +++ b/jenkins-scripts/dsl/gazebo.dsl @@ -313,7 +313,7 @@ ci_distro.each { distro -> label "gpu-reliable && large-memory" // Problem with the compilation of Gazebo under bullseyes - // See: https://github.com/ignition-tooling/release-tools/issues/129 + // See: https://github.com/gazebo-tooling/release-tools/issues/129 disabled() steps { diff --git a/jenkins-scripts/dsl/gzdev.dsl b/jenkins-scripts/dsl/gzdev.dsl index 4a72a06b5..e76b2fef3 100644 --- a/jenkins-scripts/dsl/gzdev.dsl +++ b/jenkins-scripts/dsl/gzdev.dsl @@ -21,7 +21,7 @@ supported_distros.each { distro -> scm { git { remote { - github('ignition-tooling/gzdev', 'https') + github('gazebo-tooling/gzdev', 'https') branch('refs/heads/master') extensions { @@ -50,7 +50,7 @@ supported_distros.each { distro -> // -------------------------------------------------------------- // 3. Create the testing any job def gzdev_any_job = job("gzdev-ci-pr_any-${distro}-${arch}") - OSRFLinuxCompilationAnyGitHub.create(gzdev_any_job, "ignition-tooling/gzdev", false, false) + OSRFLinuxCompilationAnyGitHub.create(gzdev_any_job, "gazebo-tooling/gzdev", false, false) gzdev_any_job.with { diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index 85fcd8602..a5abc360f 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -79,7 +79,7 @@ ignition_extra_debbuild = [ 'cmake3', 'sensors7', 'tools2', 'transport12', - 'utils1', // see comment https://github.com/ignition-tooling/release-tools/pull/431#issuecomment-815099918 + 'utils1', // see comment https://github.com/gazebo-tooling/release-tools/pull/431#issuecomment-815099918 'utils2'] // DESC: exclude ignition from generate any install testing job ignition_no_pkg_yet = [ ] @@ -303,7 +303,7 @@ ignition_software.each { ign_sw -> checkout_subdir = "ign-${ign_sw}" OSRFLinuxABIGitHub.create(abi_job) GenericAnyJobGitHub.create(abi_job, - "ignitionrobotics/ign-${ign_sw}", + "gazebosim/ign-${ign_sw}", all_branches(ign_sw) - [ 'main']) abi_job.with { @@ -347,7 +347,7 @@ ignition_software.each { ign_sw -> def ignition_ci_any_job = job(ignition_ci_job_name) def ignition_checkout_dir = "ign-${ign_sw}" OSRFLinuxCompilationAnyGitHub.create(ignition_ci_any_job, - "ignitionrobotics/${ignition_checkout_dir}", + "gazebosim/${ignition_checkout_dir}", enable_testing(ign_sw)) include_gpu_label_if_needed(ignition_ci_any_job, ign_sw) ignition_ci_any_job.with @@ -441,7 +441,7 @@ void generate_ci_job(ignition_ci_job, ign_sw, branch, distro, arch, { OSRFLinuxCompilation.create(ignition_ci_job, enable_testing(ign_sw)) OSRFGitHub.create(ignition_ci_job, - "ignitionrobotics/ign-${ign_sw}", + "gazebosim/ign-${ign_sw}", "${branch}", "ign-${ign_sw}") include_gpu_label_if_needed(ignition_ci_job, ign_sw) @@ -542,7 +542,7 @@ ignition_software.each { ign_sw -> def ignition_brew_ci_any_job = job(ignition_brew_ci_any_job_name) OSRFBrewCompilationAnyGitHub.create(ignition_brew_ci_any_job, - "ignitionrobotics/ign-${ign_sw}", + "gazebosim/ign-${ign_sw}", enable_testing(ign_sw), GITHUB_SUPPORT_ALL_BRANCHES, ENABLE_GITHUB_PR_INTEGRATION, @@ -574,7 +574,7 @@ ignition_software.each { ign_sw -> enable_testing(ign_sw), enable_cmake_warnings(ign_sw)) OSRFGitHub.create(ignition_brew_ci_job, - "ignitionrobotics/ign-${ign_sw}", + "gazebosim/ign-${ign_sw}", "${branch}", "ign-${ign_sw}") ignition_brew_ci_job.with { @@ -665,7 +665,7 @@ ignition_software.each { ign_sw -> def ignition_win_ci_any_job = job(ignition_win_ci_any_job_name) OSRFWinCompilationAnyGitHub.create(ignition_win_ci_any_job, - "ignitionrobotics/ign-${ign_sw}", + "gazebosim/ign-${ign_sw}", enable_testing(ign_sw), supported_branches, ENABLE_GITHUB_PR_INTEGRATION, @@ -701,7 +701,7 @@ ignition_software.each { ign_sw -> enable_testing(ign_sw), enable_cmake_warnings(ign_sw)) OSRFGitHub.create(ignition_win_ci_job, - "ignitionrobotics/ign-${ign_sw}", + "gazebosim/ign-${ign_sw}", "${branch}", "ign-${ign_sw}") ignition_win_ci_job.with diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index be1dd7aa4..a574ce5a3 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -346,7 +346,7 @@ ignition_collections.each { ign_collection -> def ignition_brew_ci_job = job("ignition_${ign_collection_name}-ci-main-homebrew-amd64") OSRFBrewCompilation.create(ignition_brew_ci_job, DISABLE_TESTS) OSRFGitHub.create(ignition_brew_ci_job, - "ignitionrobotics/ign-${ign_collection_name}", + "gazebosim/ign-${ign_collection_name}", "main", "ign-${ign_collection_name}") ignition_brew_ci_job.with diff --git a/jenkins-scripts/dsl/ros1_ign_bridge.dsl b/jenkins-scripts/dsl/ros1_ign_bridge.dsl index e4b061b29..114ed3eeb 100644 --- a/jenkins-scripts/dsl/ros1_ign_bridge.dsl +++ b/jenkins-scripts/dsl/ros1_ign_bridge.dsl @@ -42,7 +42,7 @@ bridge_packages.each { pkg -> stringParam("ARCH", "amd64", "Architecture to build packages for") stringParam('ROS_DISTRO', 'noetic','ROS DISTRO to build pakcages for') stringParam("UPLOAD_TO_REPO", 'stable', "OSRF repo name to upload the package to") - stringParam('UPSTREAM_RELEASE_REPO', 'https://github.com/ignition-release/ros1_ign_bridge-release', 'Release repository url') + stringParam('UPSTREAM_RELEASE_REPO', 'https://github.com/gazebo-release/ros1_ign_bridge-release', 'Release repository url') } steps { diff --git a/jenkins-scripts/dsl/sdformat.dsl b/jenkins-scripts/dsl/sdformat.dsl index 80e024096..de9e37c15 100644 --- a/jenkins-scripts/dsl/sdformat.dsl +++ b/jenkins-scripts/dsl/sdformat.dsl @@ -38,7 +38,7 @@ abi_distro.each { distro -> abi_job_name = "sdformat-abichecker-any_to_any-ubuntu_auto-${arch}" def abi_job = job(abi_job_name) OSRFLinuxABIGitHub.create(abi_job) - GenericAnyJobGitHub.create(abi_job, 'ignitionrobotics/sdformat', abi_branches) + GenericAnyJobGitHub.create(abi_job, 'gazebosim/sdformat', abi_branches) abi_job.with { steps { @@ -67,7 +67,7 @@ ci_distro.each { distro -> // 1. Create the main ci jobs def sdformat_ci_job = job("sdformat-ci-main-${distro}-${arch}") OSRFLinuxCompilation.create(sdformat_ci_job) - OSRFGitHub.create(sdformat_ci_job, "ignitionrobotics/sdformat", "main") + OSRFGitHub.create(sdformat_ci_job, "gazebosim/sdformat", "main") sdformat_ci_job.with { @@ -88,7 +88,7 @@ ci_distro.each { distro -> // -------------------------------------------------------------- // 2. Create the any job - String sdf_repo = "ignitionrobotics/sdformat" + String sdf_repo = "gazebosim/sdformat" def sdformat_ci_any_job = job(ci_build_any_job_name_linux) OSRFLinuxCompilationAnyGitHub.create(sdformat_ci_any_job, sdf_repo) @@ -115,7 +115,7 @@ other_supported_distros.each { distro -> // ci_main job for the rest of arches / scm@daily def sdformat_ci_job = job("sdformat-ci-main-${distro}-${arch}") OSRFLinuxCompilation.create(sdformat_ci_job) - OSRFGitHub.create(sdformat_ci_job, "ignitionrobotics/sdformat", "main") + OSRFGitHub.create(sdformat_ci_job, "gazebosim/sdformat", "main") sdformat_ci_job.with { @@ -147,7 +147,7 @@ void generate_ci_job(sdformat_ci_job, branch, distro, arch, extra_cmake = '', extra_test = '') { OSRFLinuxCompilation.create(sdformat_ci_job) - OSRFGitHub.create(sdformat_ci_job, "ignitionrobotics/sdformat", + OSRFGitHub.create(sdformat_ci_job, "gazebosim/sdformat", get_sdformat_branch_name(branch)) sdformat_ci_job.with { @@ -196,7 +196,7 @@ ci_distro.each { distro -> experimental_arches.each { arch -> def sdformat_ci_job = job("sdformat-ci-main-${distro}-${arch}") OSRFLinuxCompilation.create(sdformat_ci_job) - OSRFGitHub.create(sdformat_ci_job, "ignitionrobotics/sdformat", "main") + OSRFGitHub.create(sdformat_ci_job, "gazebosim/sdformat", "main") sdformat_ci_job.with { @@ -281,7 +281,7 @@ all_debbuild_branches.each { branch -> String ci_build_any_job_name_brew = "sdformat-ci-pr_any-homebrew-amd64" def sdformat_brew_ci_any_job = job(ci_build_any_job_name_brew) OSRFBrewCompilationAnyGitHub.create(sdformat_brew_ci_any_job, - "ignitionrobotics/sdformat") + "gazebosim/sdformat") sdformat_brew_ci_any_job.with { steps { @@ -298,7 +298,7 @@ all_branches = sdformat_supported_branches + 'main' all_branches.each { branch -> def sdformat_brew_ci_job = job("sdformat-ci-${branch}-homebrew-amd64") OSRFBrewCompilation.create(sdformat_brew_ci_job) - OSRFGitHub.create(sdformat_brew_ci_job, "ignitionrobotics/sdformat", + OSRFGitHub.create(sdformat_brew_ci_job, "gazebosim/sdformat", get_sdformat_branch_name(branch)) sdformat_brew_ci_job.with @@ -356,7 +356,7 @@ sdformat_supported_branches.each { branch -> String ci_build_any_job_name_win7 = "sdformat-ci-pr_any-windows7-amd64" def sdformat_win_ci_any_job = job(ci_build_any_job_name_win7) OSRFWinCompilationAnyGitHub.create(sdformat_win_ci_any_job, - "ignitionrobotics/sdformat") + "gazebosim/sdformat") sdformat_win_ci_any_job.with { steps { @@ -371,7 +371,7 @@ all_branches = sdformat_supported_branches + 'main' all_branches.each { branch -> def sdformat_win_ci_job = job("sdformat-ci-${branch}-windows7-amd64") OSRFWinCompilation.create(sdformat_win_ci_job) - OSRFGitHub.create(sdformat_win_ci_job, "ignitionrobotics/sdformat", + OSRFGitHub.create(sdformat_win_ci_job, "gazebosim/sdformat", get_sdformat_branch_name(branch)) sdformat_win_ci_job.with { diff --git a/jenkins-scripts/dsl/test.dsl b/jenkins-scripts/dsl/test.dsl index 5f9d809df..7d63fffb5 100644 --- a/jenkins-scripts/dsl/test.dsl +++ b/jenkins-scripts/dsl/test.dsl @@ -11,7 +11,7 @@ def ignition_ci_job_win = job("_test_job_from_dsl_win") OSRFWinBase.create(ignition_ci_job_win) def ignition_ci_pr_job = job("_test_pr_job_from_dsl") OSRFLinuxCompilationAnyGitHub.create(ignition_ci_pr_job, - 'ignitionrobotics/testing', + 'gazebosim/testing', false, false, ['main']) diff --git a/jenkins-scripts/gazebo-one_liner-homebrew.bash b/jenkins-scripts/gazebo-one_liner-homebrew.bash index 9a1784da7..8716ede3b 100644 --- a/jenkins-scripts/gazebo-one_liner-homebrew.bash +++ b/jenkins-scripts/gazebo-one_liner-homebrew.bash @@ -10,5 +10,5 @@ echo '# END SECTION' echo '# BEGIN SECTION: run the one-liner installation' # TODO: change this to 'curl -ssL https://get.gazebosim.org' once that supports https -curl -ssL https://github.com/ignition-tooling/release-tools/raw/master/one-line-installations/gazebo.sh | sh -x +curl -ssL https://github.com/gazebo-tooling/release-tools/raw/master/one-line-installations/gazebo.sh | sh -x echo '# END SECTION' diff --git a/jenkins-scripts/lib/windows_library.bat b/jenkins-scripts/lib/windows_library.bat index 2a289e570..f5a8e2527 100644 --- a/jenkins-scripts/lib/windows_library.bat +++ b/jenkins-scripts/lib/windows_library.bat @@ -145,7 +145,7 @@ set gzdistro_dir=gazebodistro if "%GAZEBODISTRO_BRANCH%" == "" (set GAZEBODISTRO_BRANCH=master) if exist %gzdistro_dir% (rmdir /s /q %gzdistro_dir%) -git clone https://github.com/ignition-tooling/gazebodistro %gzdistro_dir% -b %GAZEBODISTRO_BRANCH% +git clone https://github.com/gazebo-tooling/gazebodistro %gzdistro_dir% -b %GAZEBODISTRO_BRANCH% :: Check if ci_matching_branch name is used if "%ghprbSourceBranch%" == "" (echo ghprbSourceBranch is unset) else ( python "%SCRIPT_DIR%\tools\detect_ci_matching_branch.py" "%ghprbSourceBranch%" diff --git a/one-line-installations/gazebo.sh b/one-line-installations/gazebo.sh index 6206ef687..35ac89f08 100644 --- a/one-line-installations/gazebo.sh +++ b/one-line-installations/gazebo.sh @@ -315,7 +315,7 @@ do_install() { cat >&2 <<-'EOF_END' Either your platform is not easily detectable, is not supported by this - installer script (yet - PRs welcome! [https://github.com/ignition-tooling/release-tools]) + installer script (yet - PRs welcome! [https://github.com/gazebo-tooling/release-tools]) or does not yet have a package for gazebo. Please visit the following URL for more detailed installation instructions: diff --git a/release.py b/release.py index a6ed55006..df0412a6d 100755 --- a/release.py +++ b/release.py @@ -627,7 +627,7 @@ def go(argv): # all distribution builds to avoid race conditions. Note: this # assumes that large-memory nodes are beind used for nightly # tags. - # https://github.com/ignition-tooling/release-tools/issues/644 + # https://github.com/gazebo-tooling/release-tools/issues/644 if (NIGHTLY): assert a == 'amd64', f'Nightly tag assumed amd64 but arch is {a}' linux_platform_params['JENKINS_NODE_TAG'] = 'linux-nightly-' + d diff --git a/source-repo-scripts/bump_dependency.bash b/source-repo-scripts/bump_dependency.bash index 84a7a562d..d7e86274c 100755 --- a/source-repo-scripts/bump_dependency.bash +++ b/source-repo-scripts/bump_dependency.bash @@ -47,10 +47,10 @@ WHITE_BG="\e[107m" BLUE_BG="\e[44m" GREEN_BG="\e[42m" -IGN_ORG="ignitionrobotics" +IGN_ORG="gazebosim" OSRF_ORG="osrf" -TOOLING_ORG="ignition-tooling" -RELEASE_ORG="ignition-release" +TOOLING_ORG="gazebo-tooling" +RELEASE_ORG="gazebo-release" COLLECTION=${1} LIBRARY_INPUT=${2} diff --git a/terminal-dashboard/_dashboard_lib.sh b/terminal-dashboard/_dashboard_lib.sh index c91479ada..6ec8f1d64 100644 --- a/terminal-dashboard/_dashboard_lib.sh +++ b/terminal-dashboard/_dashboard_lib.sh @@ -1,6 +1,6 @@ #!/bin/bash -# TODO:jrivero https://github.com/ignition-tooling/release-tools/issues/551 +# TODO:jrivero https://github.com/gazebo-tooling/release-tools/issues/551 get_libraries_by_collection() { From f5c2ef4224b59994c01c40e978413f31c4671251 Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Mon, 16 May 2022 13:55:58 -0700 Subject: [PATCH 145/540] merge_forward_pull_request.bash: new script (#720) Add a script to simplify creation of forward port pull requests that uses the gh tool and automatically fills in the template. Signed-off-by: Steve Peters --- source-repo-scripts/README.md | 29 ++++++++++ .../merge_forward_pull_request.bash | 55 +++++++++++++++++++ 2 files changed, 84 insertions(+) create mode 100755 source-repo-scripts/merge_forward_pull_request.bash diff --git a/source-repo-scripts/README.md b/source-repo-scripts/README.md index 4f13d6898..226c13c91 100644 --- a/source-repo-scripts/README.md +++ b/source-repo-scripts/README.md @@ -39,6 +39,35 @@ its dependencies on fortress using the `chapulina/fortress` branch for `docs`: "chapulina/fortress" ``` +### merge_forward_pull_request.bash + +The script will open a pull request for a forward port after the user has +already created the branch, run `git merge`, resolved conflicts, and committed +the result. + +#### Usage + +Requires the 'gh' CLI to be installed. +```bash +./merge_forward_pull_request.bash +``` + +#### Example + +To merge `ign-rendering6` to `main`: + +```bash +cd ign-rendering +git fetch origin +git checkout main +git reset --hard origin/main +git checkout -b merge_6_to_main +git merge origin/ign-rendering6 +# manually resolve conflicts if necessary +git merge --continue +/path/to/merge_forward_pull_request.bash ign-rendering6 main +``` + ### source_changelog.bash The script will generate updates for `Changelog.md` files useful when preparing a diff --git a/source-repo-scripts/merge_forward_pull_request.bash b/source-repo-scripts/merge_forward_pull_request.bash new file mode 100755 index 000000000..8e1b8328b --- /dev/null +++ b/source-repo-scripts/merge_forward_pull_request.bash @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# Copyright (C) 2022 Open Source Robotics Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The script will open a pull request for a forward port. +# +# Requires the 'gh' CLI to be installed. +# +# Usage: +# $ ./merge_forward_pull_request.bash +# +# For example, to merge `ign-rendering6` forward to `main`: +# +# ./merge_forward_pull_request.bash ign-rendering6 main + +FROM_BRANCH=${1} +TO_BRANCH=${2} + +if [[ $# -ne 2 ]]; then + echo "./merge_forward_pull_request.bash " + exit 1 +fi + +set -e + +ORIGIN_URL=$(git remote get-url origin) +ORIGIN_ORG_REPO=$(echo ${ORIGIN_URL} | sed -e 's@.*github\.com.@@') + +TITLE="Merge ${FROM_BRANCH} ➡️ ${TO_BRANCH}" + +BODY="# ➡️ Forward port + +Port \`${FROM_BRANCH} \` ➡️ \`${TO_BRANCH}\` + +Branch comparision: https://github.com/${ORIGIN_ORG_REPO}/compare/${TO_BRANCH}...${FROM_BRANCH} + +**Note to maintainers**: Remember to **Merge** with commit (not squash-merge or rebase)" + +gh pr create \ + --title "$TITLE" \ + --repo "$ORIGIN_ORG_REPO" \ + --base "$TO_BRANCH" \ + --body "$BODY" From 127033372e4d633f06998a98283465da29b754a9 Mon Sep 17 00:00:00 2001 From: Louise Poubel Date: Tue, 17 May 2022 09:31:30 -0700 Subject: [PATCH 146/540] Script to generate release summaries (#677) * Script to generate release summaries Signed-off-by: Louise Poubel * Right shebang and stop on errors * Auto detect project name and add some safety checks * Update release_summary.bash Co-authored-by: Jose Luis Rivero * fix SDF Signed-off-by: Louise Poubel * Move files Signed-off-by: Louise Poubel * fix readme Signed-off-by: Louise Poubel Co-authored-by: Jose Luis Rivero --- README.md | 4 +- source-repo-scripts/README.md | 26 ++++++++ source-repo-scripts/release_summary.bash | 80 ++++++++++++++++++++++++ 3 files changed, 108 insertions(+), 2 deletions(-) create mode 100644 source-repo-scripts/release_summary.bash diff --git a/README.md b/README.md index 4579ec1fe..45f27cbbb 100644 --- a/README.md +++ b/README.md @@ -6,9 +6,9 @@ Gazebo Classic and Ignition software. ## Scripts * [release.py](release.py): Triggers new debian and homebrew releases (major, minor, patch, pre-release...). -* [source_changelog.bash](source_changelog.bash): Generates Changelog.md entries newer than a specified tag based on git commit descriptions. -* For -release repository scripts, please see [README.md](release-repo-scripts/README.md) +* For -release repository scripts, please see [release-repo-scripts/README.md](release-repo-scripts/README.md) +* For source repository scripts, please see [source-repo-scripts/README.md](source-repo-scripts/README.md) ### Making releases diff --git a/source-repo-scripts/README.md b/source-repo-scripts/README.md index 226c13c91..aedb7723f 100644 --- a/source-repo-scripts/README.md +++ b/source-repo-scripts/README.md @@ -92,3 +92,29 @@ To update the Changelog for `gz-math` new version 6.11.0: cd gz-math ~/release-tools/source-repo-scripts/source_changelog.bash 6.10.0 ``` + +### Release summary + +Print a markdown summary of a release (not `Changelog.md` entries), with its +changelog and contributors. The script is designed to publish release summaries +from the internal Open Robotics team to the Community (usually in the public forum). + +#### Usage + +Run the script, then copy the end of the output into the announcement body. + +```bash +cd +bash release_summary.bash +``` + +The release versions can be checked on `Changelog.md`. They don't need to be +consecutive (useful in cases when multiple releases happen between announcements). + +#### Example + +To announce the changes between Sensors 6.2.0 and 6.3.0: + +```bash +cd +bash release_summary.bash 6.2.0 6.3.0 diff --git a/source-repo-scripts/release_summary.bash b/source-repo-scripts/release_summary.bash new file mode 100644 index 000000000..c717c73be --- /dev/null +++ b/source-repo-scripts/release_summary.bash @@ -0,0 +1,80 @@ +#!/usr/bin/env bash + +set -e + +# Print a markdown summary of a release (not Changelog.md entries), with its +# changelog and contributors. The script is designed to publish release summaries +# from the internal Open Robotics team to the Community (usually in the public forum). +# +# cd +# bash release_summary.bash 5.3.0 5.4.0 + +PREV=$1 +NEW=$2 + +MAJOR=${PREV%.*.*} + +if [[ ! -f CMakeLists.txt ]]; then + echo "No CMakeLists.txt detected. Are you in an source repository?" + exit 1 +fi + +# Parse CMakeLists project () for sdformat and ignitions +LIB=$(sed -n 's/^project[[:space:]]*(\(ignition-\)\?\([a-Z]*\)[0-9]*.*)/\2/p' CMakeLists.txt) + +if [ -z "${LIB}" ]; then + echo "Parsing of CMakeLists.txt project tag failed" + echo "Probably an internal bug" + exit 1 +fi + +echo "---------------------------------" +NAME_FOR_TAGS=ignition-${LIB} +NAME_FOR_TAGS="${NAME_FOR_TAGS/ignition-sdformat/sdformat}" +echo "NAME_FOR_TAGS: $NAME_FOR_TAGS" + +NAME_FOR_REPO=ign-${LIB} +NAME_FOR_REPO="${NAME_FOR_REPO/ign-sdformat/sdformat}" +echo "NAME_FOR_REPO: $NAME_FOR_REPO" + +NAME_FOR_BRANCH=ign-${LIB} +NAME_FOR_BRANCH="${NAME_FOR_BRANCH/ign-sdformat/sdf}" +echo "NAME_FOR_BRANCH: $NAME_FOR_BRANCH" + +NAME_FOR_TITLE="Ignition ${LIB^}" +NAME_FOR_TITLE="${NAME_FOR_TITLE/Ignition Sdformat/SDFormat}" +echo "NAME_FOR_TITLE: $NAME_FOR_TITLE" + +echo "Previous version: ${PREV}" +echo "New version: ${NEW}" +echo "Major version: ${MAJOR}" +echo "---------------------------------" +echo "" + +git fetch --all --tags + +if ! git checkout "${NAME_FOR_BRANCH}${MAJOR}"; then + echo "Branch ${NAME_FOR_BRANCH}${MAJOR} was not found in the repository" + exit 1 +fi + +git pull origin ${NAME_FOR_BRANCH}${MAJOR} + +echo "" +echo "" +echo "" +echo "" +echo "## $NAME_FOR_TITLE $NEW" +echo "" +echo "## Changelog" +echo "" +echo "[Full changelog](https://github.com/ignitionrobotics/${NAME_FOR_REPO}/blob/${NAME_FOR_BRANCH}${MAJOR}/Changelog.md)" +echo "" +#awk '/${NEW}/{ f = 1; next } /${PREV}/{ f = 0 } f' Changelog.md +sed -n "/${NEW}/, /${PREV}/{ /${NEW}/! { /${PREV}/! p } }" Changelog.md +echo "" +echo "## Contributors" +echo "" +git log --pretty="%an" ${NAME_FOR_TAGS}${MAJOR}_${PREV}...${NAME_FOR_TAGS}${MAJOR}_${NEW} | sort | uniq | sed "s/.*/\*&\*/" +echo "" +echo "---" From 6097067fa08a917f0843fc1e3651e83b8943a963 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 19 May 2022 18:52:11 +0200 Subject: [PATCH 147/540] Auto detect PACKAGE_ALIAS to help with gz transition (#730) * Auto detect PACKAGE_ALIAS to help with gz transition * Inject parameter into an script variable --- jenkins-scripts/docker/lib/debbuild-base.bash | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/jenkins-scripts/docker/lib/debbuild-base.bash b/jenkins-scripts/docker/lib/debbuild-base.bash index 366184644..64339a073 100644 --- a/jenkins-scripts/docker/lib/debbuild-base.bash +++ b/jenkins-scripts/docker/lib/debbuild-base.bash @@ -140,10 +140,17 @@ esac cd \${PACKAGE_RELEASE_DIR} +# Helper for transition of ign to gz +PACKAGE_ALIAS=${PACKAGE_ALIAS} +SRC_PACKAGE_NAME=\$(grep-dctrl -sSource -n '' debian/control) +if [[ \${SRC_PACKAGE_NAME} != \${SRC_PACKAGE_NAME/gz-} ]]; then + PACKAGE_ALIAS=\${SRC_PACKAGE_NAME} +fi + # [nightly] Adjust version in nightly mode if $NIGHTLY_MODE; then NIGHTLY_VERSION_SUFFIX=\${UPSTREAM_VERSION}+\${TIMESTAMP}+${RELEASE_VERSION}r\${REV}-${RELEASE_VERSION}~${DISTRO} - debchange --package ${PACKAGE_ALIAS} \\ + debchange --package \${PACKAGE_ALIAS} \\ --newversion \${NIGHTLY_VERSION_SUFFIX} \\ --distribution ${DISTRO} \\ --force-distribution \\ @@ -162,7 +169,7 @@ if $NIGHTLY_MODE; then if dpkg --compare-versions \$(apt-cache show dh-make | sed -n "s/Version: \\(.*\\)/\\1/p") lt 2.202003; then extra_dh_make_str='' fi - echo | dh_make -y -s --createorig \${extra_dh_make_str} -p${PACKAGE_ALIAS}_\${UPSTREAM_VERSION}+\${TIMESTAMP}+${RELEASE_VERSION}r\${REV} > /dev/null + echo | dh_make -y -s --createorig \${extra_dh_make_str} -p\${PACKAGE_ALIAS}_\${UPSTREAM_VERSION}+\${TIMESTAMP}+${RELEASE_VERSION}r\${REV} > /dev/null rm -fr debian/ fi @@ -200,7 +207,7 @@ while (! \$update_done); do done # new versions of mk-build-deps > 2.21.1 left buildinfo and changes files in the code -rm -f ${PACKAGE_ALIAS}-build-deps_*.{buildinfo,changes} +rm -f \${PACKAGE_ALIAS}-build-deps_*.{buildinfo,changes} echo '# END SECTION' if [ -f /usr/bin/rosdep ]; then From a0319f33ed45839d815294229ae7fc0b4e14e649 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Mon, 16 May 2022 17:16:54 +0200 Subject: [PATCH 148/540] Add script to rename packages from ignition to gazebo --- .../rename_ignition_to_gazebo.bash | 105 ++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100755 release-repo-scripts/rename_ignition_to_gazebo.bash diff --git a/release-repo-scripts/rename_ignition_to_gazebo.bash b/release-repo-scripts/rename_ignition_to_gazebo.bash new file mode 100755 index 000000000..d517caf27 --- /dev/null +++ b/release-repo-scripts/rename_ignition_to_gazebo.bash @@ -0,0 +1,105 @@ +#!/bin/bash + +set -e + +# 1. Block: change in the control file +find . -name control -type f | while IFS= read -r f; do + if [[ $f == ./ubuntu/debian/tests/control ]]; then + continue + fi + echo "Processing control file: ${f}" + ignition_pkgs=$(grep-dctrl -sPackage -n '' ${f}) + echo " - Replace ignition ocurrences" + # Rename urls and packages + sed -i -e 's:ignitionrobotics:gazebosim:g' "${f}" + sed -i -e 's:libignition:libgz:g' "${f}" + sed -i -e 's:ignition:gz:g' "${f}" + sed -i -e 's:Ignition:Gazebo:g' "${f}" + sed -i -e 's:ign-:gz-:g' "${f}" + # Inject transitional (alias) packages + echo " - Transitional packages in control file" + for pkg in ${ignition_pkgs}; do + echo " * Processing transitional package for ${pkg}" + new_pkg_name=${pkg/ignition/gz} + cat << EOF >> "${f}" + +Package: ${pkg} +Depends: ${new_pkg_name}, \${misc:Depends} +Architecture: all +Priority: optional +Section: oldlibs +Description: transitional package + This is a transitional package. It can safely be removed. +EOF + done + echo +done + +# 2. Copyright files +find . -name '*copyright*' -type f | while IFS= read -r f; do + echo "Procesing copyright file: ${f}" + sed -i -e 's:bitbucket.org/:github.org/:g' "${f}" + sed -i -e 's:/osrf/:/ignitionrobotics/:g' "${f}" + sed -i -e 's:ignitionrobotics:gazebosim:g' "${f}" + sed -i -e 's:ignition:gz:g' "${f}" + sed -i -e 's:ign_:gz-:g' "${f}" + echo +done + +# 3. Test file if needed +if [[ -f ubuntu/debian/tests/ ]]; then + echo "Procesing copyright tests control" + sed -i -e 's:ignition:gz:g' ubuntu/debian/tests/control + echo +fi + +# 4. Changelog files +find . -name 'changelog' -type f | while IFS= read -r f; do + echo "Bumping changelog file ${f}" + distro=$(dpkg-parsechangelog -l${f} -S distribution) + version=$(dpkg-parsechangelog -l${f} -S Version | sed 's:~.*::g') + version_without_rev=${version/-*} + bump_rev=$(( ${version/*-} + 1 )) + dch --changelog "${f}" \ + --distribution="${distro}" \ + --force-distribution \ + --newversion "${version_without_rev}-${bump_rev}~${distro}" \ + "Change package source name" + sed -i -e '1 s:ignition-gazebo:gz-sim:g' "${f}" + sed -i -e '1 s:ignition:gz:g' "${f}" +done + +echo + +# 5. Rename all ignition symlinks and destinations +find . -name '*ignition*' -type l | while IFS= read -r f; do + echo "Renaming ${f} -> ${f/ignition-/gz-}" + link=$(readlink "$f") + ln -sfT "${link//ignition-/gz-}" "${f}" + git mv "${f}" "${f/ignition-/gz-}" +done + +echo + +# 6. Rename all ignition filenames not symlinks +find . -name '*ignition*' -type f | while IFS= read -r f; do + echo "Renaming ${f} -> ${f/ignition-/gz-}" + git mv "${f}" "${f/ignition-/gz-}" +done + +# 6. watch files +find . -name 'watch' -type f | while IFS= read -r f; do + echo "Changeing watch file ${f}" + sed -i -e 's:/osrf/:/ignitionrobotics/:g' "${f}" + sed -i -e 's:ignitionrobotics:gazebosim:g' "${f}" + sed -i -e 's:ignition:gz:g' "${f}" +done + +echo + +# 7. Revert Provides, Breaks, Replaces +find . -name control -type f | while IFS= read -r f; do + sed -i -e 's:\(Breaks\:.*\)gz-\(.*\):\1ignition-\2:' "${f}" + sed -i -e 's:\(Provides\:.*\)gz-\(.*\):\1ignition-\2:' "${f}" + sed -i -e 's:\(Replaces\:.*\)gz-\(.*\):\1ignition-\2:' "${f}" +done From 1bd6ab50b449914f3ac5da81dd82a782a84dc4fa Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Mon, 16 May 2022 18:48:59 +0200 Subject: [PATCH 149/540] Add instruction to README --- release-repo-scripts/README.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/release-repo-scripts/README.md b/release-repo-scripts/README.md index 2df2a842e..a2856c5bd 100644 --- a/release-repo-scripts/README.md +++ b/release-repo-scripts/README.md @@ -109,3 +109,27 @@ To create `ign-rendering7-release` repository copying files in `ign-rendering6-r cd /tmp/ ./path/to/release-tools/release-repo-scripts/new_ignition_release_repos.bash ign-rendering7 ``` + +### rename_ignition_to_gazebo.bash + +Script used for renaming all ignition package names to gz. It creates +transition packages as aliases. + +#### Usage + +The script uses `dch` from the package `debhelper`. `grep-dctrl` from the `dctrl-tools` package. +The script will make all the changes without making any commit so a manual inspection +can be done after the run. + +```bash +cd +./path/to/release-tools/release-repo-scripts/rename_ignition_to_gazebo.bash +``` + +#### Example + +```bash +git clone https://github.com/gazebo-release/gz-cmake3-release +cd gz-cmake3-release +./path/to/release-tools/release-repo-scripts/rename_ignition_to_gazebo.bash +``` From d4261e2d3ea0b1dfb8bef17903a7aa2cd2003f31 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Mon, 16 May 2022 18:51:38 +0200 Subject: [PATCH 150/540] Use gazebo-release --- jenkins-scripts/docker/lib/debbuild-base.bash | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/jenkins-scripts/docker/lib/debbuild-base.bash b/jenkins-scripts/docker/lib/debbuild-base.bash index 64339a073..e1b6dcce4 100644 --- a/jenkins-scripts/docker/lib/debbuild-base.bash +++ b/jenkins-scripts/docker/lib/debbuild-base.bash @@ -83,7 +83,7 @@ fi # Step 4: add debian/ subdirectory with necessary metadata files to unpacked source tarball rm -rf /tmp/$PACKAGE-release -git clone https://github.com/ignition-release/$PACKAGE-release -b $RELEASE_REPO_BRANCH /tmp/$PACKAGE-release +git clone https://github.com/gazebo-release/$PACKAGE-release -b $RELEASE_REPO_BRANCH /tmp/$PACKAGE-release cd /tmp/$PACKAGE-release # In nightly get the default latest version from default changelog if $NIGHTLY_MODE; then @@ -109,7 +109,7 @@ fi case \${BUILD_METHOD} in "OVERWRITE_BASE") # 1. Clone the base branch - git clone https://github.com/ignition-release/\${PACKAGE}-release \\ + git clone https://github.com/gazebo-release/\${PACKAGE}-release \\ -b \${RELEASE_BASE_BRANCH} \\ /tmp/base_$PACKAGE-release # 2. Overwrite the information @@ -190,7 +190,7 @@ fi timeout=0 # Help to debug race conditions in nightly generation or other problems with versions if ${NIGHTLY_MODE}; then - apt-cache show *ignition* | ( grep 'Package\\|Version' || true) + apt-cache show *gz-* | ( grep 'Package\\|Version' || true) # 5 minutes to give time to the uploader timeout=300 fi From b1754ccec879c4e2d504897ccdcc637f0f8ec5b5 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 17 May 2022 23:49:22 +0200 Subject: [PATCH 151/540] Remove GITHUB_ORG and use gazebosim --- jenkins-scripts/docker/lib/boilerplate_prepare.sh | 5 ----- jenkins-scripts/docker/lib/debbuild-base.bash | 2 +- .../docker/multidistribution-ignition-debbuild.bash | 1 - 3 files changed, 1 insertion(+), 7 deletions(-) diff --git a/jenkins-scripts/docker/lib/boilerplate_prepare.sh b/jenkins-scripts/docker/lib/boilerplate_prepare.sh index e04759b9b..e46b10006 100644 --- a/jenkins-scripts/docker/lib/boilerplate_prepare.sh +++ b/jenkins-scripts/docker/lib/boilerplate_prepare.sh @@ -101,11 +101,6 @@ if [ -z ${ENABLE_REAPER} ]; then ENABLE_REAPER=true fi -# We use ignitionsrobotics or osrf. osrf by default -if [ -z ${GITHUB_ORG} ]; then - GITHUB_ORG="osrf" -fi - if [ -z "${NEED_C17_COMPILER}" ]; then export INSTALL_C17_COMPILER=false fi diff --git a/jenkins-scripts/docker/lib/debbuild-base.bash b/jenkins-scripts/docker/lib/debbuild-base.bash index e1b6dcce4..e5a95ed0b 100644 --- a/jenkins-scripts/docker/lib/debbuild-base.bash +++ b/jenkins-scripts/docker/lib/debbuild-base.bash @@ -48,7 +48,7 @@ if ${NIGHTLY_MODE}; then if ${USE_REPO_DIRECTORY_FOR_NIGHTLY}; then mv ${WORKSPACE}/repo \$REAL_PACKAGE_NAME else - git clone https://github.com/${GITHUB_ORG}/\$REAL_PACKAGE_NAME -b ${NIGHTLY_SRC_BRANCH} + git clone https://github.com/gazebosim/\$REAL_PACKAGE_NAME -b ${NIGHTLY_SRC_BRANCH} fi PACKAGE_SRC_BUILD_DIR=\$REAL_PACKAGE_NAME cd \$REAL_PACKAGE_NAME diff --git a/jenkins-scripts/docker/multidistribution-ignition-debbuild.bash b/jenkins-scripts/docker/multidistribution-ignition-debbuild.bash index 776610ae6..a553f4b85 100644 --- a/jenkins-scripts/docker/multidistribution-ignition-debbuild.bash +++ b/jenkins-scripts/docker/multidistribution-ignition-debbuild.bash @@ -6,6 +6,5 @@ SCRIPT_DIR="${SCRIPT_DIR%/*}" export RELEASE_REPO_DIRECTORY=${DISTRO} export ENABLE_ROS=false -export GITHUB_ORG=ignitionrobotics . ${SCRIPT_DIR}/lib/debbuild-base.bash From a7b464d0dd6042c5433df0eaf51169ceccb33e63 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 20 May 2022 02:11:46 +0200 Subject: [PATCH 152/540] Add support to Jammy for one-liner installation of Gazebo Classic (#729) --- one-line-installations/gazebo.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/one-line-installations/gazebo.sh b/one-line-installations/gazebo.sh index 35ac89f08..ed4b6252e 100644 --- a/one-line-installations/gazebo.sh +++ b/one-line-installations/gazebo.sh @@ -153,6 +153,11 @@ do_install() { dist_version="$(. /etc/lsb-release && echo "$DISTRIB_CODENAME")" fi case "$dist_version" in + jammy) + # Packages for Jammy come directly from Ubuntu repositories, unversioned + # No released packages in packages.o.o + GZ_VER= + ;; xenial) GZ_VER=10 ;; From 80b6d0503e5d9d2797ab36aafebfe0794a8ebaa5 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 20 May 2022 17:00:11 +0200 Subject: [PATCH 153/540] Generate ogre-2.3 debbuilder (#731) --- jenkins-scripts/dsl/extra.dsl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/extra.dsl b/jenkins-scripts/dsl/extra.dsl index 2a395aceb..1b8dc3f23 100644 --- a/jenkins-scripts/dsl/extra.dsl +++ b/jenkins-scripts/dsl/extra.dsl @@ -11,7 +11,8 @@ release_repo_debbuilds = [ 'opensplice' ] // package) method used by debian gbp_repo_debbuilds = [ 'lark-parser', 'ogre-2.1', - 'ogre-2.2' ] + 'ogre-2.2', + 'ogre-2.3'] release_repo_debbuilds.each { software -> // -------------------------------------------------------------- From b12d40189ec6ba780bb92d52ab3ad0f4b2bb59fa Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Sat, 21 May 2022 02:04:41 +0200 Subject: [PATCH 154/540] Check for transitional packages and remove debian buster --- .../rename_ignition_to_gazebo.bash | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/release-repo-scripts/rename_ignition_to_gazebo.bash b/release-repo-scripts/rename_ignition_to_gazebo.bash index d517caf27..8e3790a83 100755 --- a/release-repo-scripts/rename_ignition_to_gazebo.bash +++ b/release-repo-scripts/rename_ignition_to_gazebo.bash @@ -2,6 +2,20 @@ set -e +# 0. Pre checks +find . -name control -type f | while IFS= read -r f; do + if grep "transitional package" "${f}" > /dev/null; then + echo "There are transitional packages declared in ${f}" + echo "Please remove them, probably not needed." + exit 1 + fi +done + +# we don't support buster anymore +if [[ -d debian/buster ]]; then + git rm -fr debian/buster +fi + # 1. Block: change in the control file find . -name control -type f | while IFS= read -r f; do if [[ $f == ./ubuntu/debian/tests/control ]]; then From 1767bdfd9a2b869c3c481bd7bbaff64e1f050685 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 1 Jun 2022 17:52:28 +0200 Subject: [PATCH 155/540] Remove obsolete GITHUB_ORG and use new github orgs in debbuild (#732) * Remove obsolete GITHUB_ORG and use new github orgs * Restore support for osrf/gazebo Signed-off-by: Steve Peters Co-authored-by: Steve Peters --- jenkins-scripts/docker/lib/boilerplate_prepare.sh | 2 +- jenkins-scripts/docker/lib/debbuild-base.bash | 6 +++--- .../docker/multidistribution-ignition-debbuild.bash | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/jenkins-scripts/docker/lib/boilerplate_prepare.sh b/jenkins-scripts/docker/lib/boilerplate_prepare.sh index e04759b9b..6a67fdce1 100644 --- a/jenkins-scripts/docker/lib/boilerplate_prepare.sh +++ b/jenkins-scripts/docker/lib/boilerplate_prepare.sh @@ -101,7 +101,7 @@ if [ -z ${ENABLE_REAPER} ]; then ENABLE_REAPER=true fi -# We use ignitionsrobotics or osrf. osrf by default +# We use gazebosim or osrf. osrf by default if [ -z ${GITHUB_ORG} ]; then GITHUB_ORG="osrf" fi diff --git a/jenkins-scripts/docker/lib/debbuild-base.bash b/jenkins-scripts/docker/lib/debbuild-base.bash index 64339a073..e1b6dcce4 100644 --- a/jenkins-scripts/docker/lib/debbuild-base.bash +++ b/jenkins-scripts/docker/lib/debbuild-base.bash @@ -83,7 +83,7 @@ fi # Step 4: add debian/ subdirectory with necessary metadata files to unpacked source tarball rm -rf /tmp/$PACKAGE-release -git clone https://github.com/ignition-release/$PACKAGE-release -b $RELEASE_REPO_BRANCH /tmp/$PACKAGE-release +git clone https://github.com/gazebo-release/$PACKAGE-release -b $RELEASE_REPO_BRANCH /tmp/$PACKAGE-release cd /tmp/$PACKAGE-release # In nightly get the default latest version from default changelog if $NIGHTLY_MODE; then @@ -109,7 +109,7 @@ fi case \${BUILD_METHOD} in "OVERWRITE_BASE") # 1. Clone the base branch - git clone https://github.com/ignition-release/\${PACKAGE}-release \\ + git clone https://github.com/gazebo-release/\${PACKAGE}-release \\ -b \${RELEASE_BASE_BRANCH} \\ /tmp/base_$PACKAGE-release # 2. Overwrite the information @@ -190,7 +190,7 @@ fi timeout=0 # Help to debug race conditions in nightly generation or other problems with versions if ${NIGHTLY_MODE}; then - apt-cache show *ignition* | ( grep 'Package\\|Version' || true) + apt-cache show *gz-* | ( grep 'Package\\|Version' || true) # 5 minutes to give time to the uploader timeout=300 fi diff --git a/jenkins-scripts/docker/multidistribution-ignition-debbuild.bash b/jenkins-scripts/docker/multidistribution-ignition-debbuild.bash index 776610ae6..41cd95757 100644 --- a/jenkins-scripts/docker/multidistribution-ignition-debbuild.bash +++ b/jenkins-scripts/docker/multidistribution-ignition-debbuild.bash @@ -6,6 +6,6 @@ SCRIPT_DIR="${SCRIPT_DIR%/*}" export RELEASE_REPO_DIRECTORY=${DISTRO} export ENABLE_ROS=false -export GITHUB_ORG=ignitionrobotics +export GITHUB_ORG=gazebosim . ${SCRIPT_DIR}/lib/debbuild-base.bash From 72777a6c3ace6438c57ab9ae63ecfba3cfe978e6 Mon Sep 17 00:00:00 2001 From: Louise Poubel Date: Wed, 15 Jun 2022 10:25:40 -0700 Subject: [PATCH 156/540] Fix release table for transport11 (#740) Signed-off-by: Louise Poubel --- terminal-dashboard/table.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terminal-dashboard/table.bash b/terminal-dashboard/table.bash index 2fecd02a9..06f5399e3 100755 --- a/terminal-dashboard/table.bash +++ b/terminal-dashboard/table.bash @@ -73,7 +73,7 @@ for LIB in $(get_libraries_by_collection "${COLLECTION}" ); do # The Source field is not mandatory and it is probably not present when # the binary package has the same name than the source package PKG_VERSION=$(wget -qO- http://packages.osrfoundation.org/gazebo/${DISTRO}-${PACKAGE_REPO}/dists/${VER}/main/binary-${ARCH}/Packages | \ - grep -1 -m 1 -e "Source: ${LIB}" -e "Package: ${LIB}" | \ + grep -2 -m 1 -e "Source: ${LIB}" -e "Package: ${LIB}" | \ sed -n 's/^Version: \(.*\)/\1/p' | uniq) fi From 266d10818231b5edefd6d789f794cccdb78cabd8 Mon Sep 17 00:00:00 2001 From: Louise Poubel Date: Fri, 17 Jun 2022 09:17:16 -0700 Subject: [PATCH 157/540] Support gz- repos on source_changelog script (#741) Signed-off-by: Louise Poubel --- source-repo-scripts/source_changelog.bash | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/source-repo-scripts/source_changelog.bash b/source-repo-scripts/source_changelog.bash index 1f5c4a801..4a7ae07ee 100755 --- a/source-repo-scripts/source_changelog.bash +++ b/source-repo-scripts/source_changelog.bash @@ -1,15 +1,23 @@ #!/bin/bash +# bash source_changelog.bash PREV_TAG=$1 git fetch --tags REPO=$(basename `git rev-parse --show-toplevel`) -REPO_FULL="${REPO/ign-/ignition-}" +REPO="${REPO/ign-/gz-}" MAJOR=${PREV_TAG%.*.*} BRANCH=${REPO/sdformat/sdf}${MAJOR} +TAG="${REPO}${MAJOR}" -COMMITS=$(git log ${BRANCH}...${REPO_FULL}${MAJOR}_${PREV_TAG} --pretty=format:"%h") +# TODO(chapulina) Support Garden tags and branches, which will start with gz- +TAG="${TAG/gz-/ignition-}" +TAG="${TAG/sim/gazebo}" +BRANCH="${BRANCH/gz-/ign-}" +BRANCH="${BRANCH/sim/gazebo}" + +COMMITS=$(git log ${BRANCH}...${TAG}_${PREV_TAG} --pretty=format:"%h") for COMMIT in $COMMITS do @@ -19,6 +27,6 @@ do PR=${PR%)} echo "1. $TITLE" - echo " * [Pull request #$PR](https://github.com/ignitionrobotics/$REPO/pull/$PR)" + echo " * [Pull request #$PR](https://github.com/gazebosim/$REPO/pull/$PR)" echo "" done From 6d489919863ca1884d8e216a44e17dce2aa2ebc3 Mon Sep 17 00:00:00 2001 From: Louise Poubel Date: Tue, 21 Jun 2022 16:19:55 -0700 Subject: [PATCH 158/540] Add GZDEV_PROJECT_NAME to gz-tools (#749) Signed-off-by: Louise Poubel --- jenkins-scripts/docker/ign_tools-compilation.bash | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/jenkins-scripts/docker/ign_tools-compilation.bash b/jenkins-scripts/docker/ign_tools-compilation.bash index af5196f02..caceddc42 100644 --- a/jenkins-scripts/docker/ign_tools-compilation.bash +++ b/jenkins-scripts/docker/ign_tools-compilation.bash @@ -18,4 +18,16 @@ export BUILDING_SOFTWARE_DIRECTORY="ign-tools" export BUILDING_JOB_REPOSITORIES="stable" export BUILDING_DEPENDENCIES="ruby" +GZ_TOOLS_MAJOR_VERSION=$(\ + python3 ${SCRIPT_DIR}/../tools/detect_cmake_major_version.py \ + ${WORKSPACE}/ign-tools/CMakeLists.txt) + +# Check GZ_TOOLS version is integer +if ! [[ ${GZ_TOOLS_MAJOR_VERSION} =~ ^-?[0-9]+$ ]]; then + echo "Error! GZ_TOOLS_MAJOR_VERSION is not an integer, check the detection" + exit -1 +fi + +export GZDEV_PROJECT_NAME="gz-tools${GZ_TOOLS_MAJOR_VERSION}" + . ${SCRIPT_DIR}/lib/generic-building-base.bash From f936000e829d2a4d5fb019a1d7523a46624f45db Mon Sep 17 00:00:00 2001 From: Louise Poubel Date: Wed, 22 Jun 2022 20:22:39 -0700 Subject: [PATCH 159/540] Explicit head branch when opening forward-port PR (#747) Signed-off-by: Louise Poubel --- source-repo-scripts/merge_forward_pull_request.bash | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/source-repo-scripts/merge_forward_pull_request.bash b/source-repo-scripts/merge_forward_pull_request.bash index 8e1b8328b..befbadf28 100755 --- a/source-repo-scripts/merge_forward_pull_request.bash +++ b/source-repo-scripts/merge_forward_pull_request.bash @@ -35,6 +35,8 @@ fi set -e +CURRENT_BRANCH=$(git branch | sed -n -e 's/^\* \(.*\)/\1/p') + ORIGIN_URL=$(git remote get-url origin) ORIGIN_ORG_REPO=$(echo ${ORIGIN_URL} | sed -e 's@.*github\.com.@@') @@ -52,4 +54,5 @@ gh pr create \ --title "$TITLE" \ --repo "$ORIGIN_ORG_REPO" \ --base "$TO_BRANCH" \ - --body "$BODY" + --body "$BODY" \ + --head "$CURRENT_BRANCH" From 6a21f9eafeb873f49982aee7417db4ee060b9bc3 Mon Sep 17 00:00:00 2001 From: Louise Poubel Date: Thu, 23 Jun 2022 17:41:01 -0700 Subject: [PATCH 160/540] Update release summaries for gz / Gazebo (#742) Signed-off-by: Louise Poubel --- source-repo-scripts/release_summary.bash | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/source-repo-scripts/release_summary.bash b/source-repo-scripts/release_summary.bash index c717c73be..ebad1e35d 100644 --- a/source-repo-scripts/release_summary.bash +++ b/source-repo-scripts/release_summary.bash @@ -19,8 +19,8 @@ if [[ ! -f CMakeLists.txt ]]; then exit 1 fi -# Parse CMakeLists project () for sdformat and ignitions -LIB=$(sed -n 's/^project[[:space:]]*(\(ignition-\)\?\([a-Z]*\)[0-9]*.*)/\2/p' CMakeLists.txt) +# Get lib name from CMakeLists project () +LIB=$(sed -n 's/^project[[:space:]]*(\(ignition-\)\?\([a-Z|_]*\)[0-9]*.*)/\2/p' CMakeLists.txt) if [ -z "${LIB}" ]; then echo "Parsing of CMakeLists.txt project tag failed" @@ -29,20 +29,28 @@ if [ -z "${LIB}" ]; then fi echo "---------------------------------" +# TODO(chapulina) Support gz tags NAME_FOR_TAGS=ignition-${LIB} +NAME_FOR_TAGS="${NAME_FOR_TAGS/_/-}" NAME_FOR_TAGS="${NAME_FOR_TAGS/ignition-sdformat/sdformat}" echo "NAME_FOR_TAGS: $NAME_FOR_TAGS" -NAME_FOR_REPO=ign-${LIB} -NAME_FOR_REPO="${NAME_FOR_REPO/ign-sdformat/sdformat}" +NAME_FOR_REPO=gz-${LIB} +NAME_FOR_REPO="${NAME_FOR_REPO/_/-}" +NAME_FOR_REPO="${NAME_FOR_REPO/gz-sdformat/sdformat}" +NAME_FOR_REPO="${NAME_FOR_REPO/gz-gazebo/gz-sim}" echo "NAME_FOR_REPO: $NAME_FOR_REPO" +# TODO(chapulina) Support gz branches NAME_FOR_BRANCH=ign-${LIB} +NAME_FOR_BRANCH="${NAME_FOR_BRANCH/_/-}" NAME_FOR_BRANCH="${NAME_FOR_BRANCH/ign-sdformat/sdf}" echo "NAME_FOR_BRANCH: $NAME_FOR_BRANCH" -NAME_FOR_TITLE="Ignition ${LIB^}" -NAME_FOR_TITLE="${NAME_FOR_TITLE/Ignition Sdformat/SDFormat}" +NAME_FOR_TITLE="Gazebo ${LIB^}" +NAME_FOR_TITLE="${NAME_FOR_TITLE/Fuel_tools/Fuel Tools}" +NAME_FOR_TITLE="${NAME_FOR_TITLE/Gazebo Sdformat/SDFormat}" +NAME_FOR_TITLE="${NAME_FOR_TITLE/Gazebo Gazebo/Gazebo Sim}" echo "NAME_FOR_TITLE: $NAME_FOR_TITLE" echo "Previous version: ${PREV}" @@ -68,7 +76,7 @@ echo "## $NAME_FOR_TITLE $NEW" echo "" echo "## Changelog" echo "" -echo "[Full changelog](https://github.com/ignitionrobotics/${NAME_FOR_REPO}/blob/${NAME_FOR_BRANCH}${MAJOR}/Changelog.md)" +echo "[Full changelog](https://github.com/gazebosim/${NAME_FOR_REPO}/blob/${NAME_FOR_BRANCH}${MAJOR}/Changelog.md)" echo "" #awk '/${NEW}/{ f = 1; next } /${PREV}/{ f = 0 } f' Changelog.md sed -n "/${NEW}/, /${PREV}/{ /${NEW}/! { /${PREV}/! p } }" Changelog.md From 127137e924eb21ec16ec303d9da0da464420de2a Mon Sep 17 00:00:00 2001 From: methylDragon Date: Fri, 24 Jun 2022 08:29:53 -0700 Subject: [PATCH 161/540] ign -> gz : Release Tools Migrate Links and Lib Names (#751) Where appropriate, and where the links don't break... ignitionrobotics -> gazebosim Ignition -> Gazebo ignition-release -> gazebo-release (with ignition-release/ign- -> gazebo-release/gz- for currently UNARCHIVED, migrated release repos) Signed-off-by: methylDragon Co-authored-by: Louise Poubel --- .github/ISSUE_TEMPLATE/bump_libraries.md | 8 ++-- .github/ISSUE_TEMPLATE/eol_collection.md | 8 ++-- .github/ISSUE_TEMPLATE/new_collection.md | 4 +- .github/ISSUE_TEMPLATE/release_collection.md | 48 +++++++++---------- README.md | 22 ++++----- bloom/release-bloom.py | 2 +- .../_generic_linux_compilation_build.sh.bash | 2 +- .../docker/lib/gazebo-base-default.bash | 6 +-- jenkins-scripts/dsl/ignition.dsl | 2 +- jenkins-scripts/dsl/ignition_collection.dsl | 2 +- jenkins-scripts/dsl/ros1_ign_bridge.dsl | 2 +- jenkins-scripts/lib/gazebo-base-windows.bat | 4 +- .../lib/gazebo9_10-base-windows.bat | 2 +- jenkins-scripts/lib/sdformat-base-windows.bat | 2 +- jenkins-scripts/lib/windows_library.bat | 4 +- release-repo-scripts/bump_major_version.bash | 4 +- .../new_ignition_release_repos.bash | 2 +- release.py | 6 +-- terminal-dashboard/unreleased.bash | 2 +- 19 files changed, 66 insertions(+), 66 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bump_libraries.md b/.github/ISSUE_TEMPLATE/bump_libraries.md index 29c40ba82..f721780e9 100644 --- a/.github/ISSUE_TEMPLATE/bump_libraries.md +++ b/.github/ISSUE_TEMPLATE/bump_libraries.md @@ -9,7 +9,7 @@ title: "Bump in [COLLECTION]: [LIBRARIES]" Use this template to keep track of everything that needs to be done to bump the major version of a library within a collection. -For example, Ignition-X is currently under development, and it's using +For example, Gazebo-X is currently under development, and it's using ign-math42. This checklist includes all that's needed to use ign-math43 instead. @@ -26,10 +26,10 @@ Libraries being bumped: * : - - [ ] Source code of all downstream libraries ([example](https://github.com/ignitionrobotics/ign-transport/pull/149)) - - [ ] Release repositories of all downstream libraries ([example](https://github.com/ignition-release/ign-fuel-tools7-release/pull/1)) + - [ ] Source code of all downstream libraries ([example](https://github.com/gazebosim/gz-transport/pull/149)) + - [ ] Release repositories of all downstream libraries ([example](https://github.com/gazebo-release/gz-fuel-tools7-release/pull/1)) - [ ] Use `main` branch on `gazebodistro` ([example](https://github.com/gazebo-tooling/gazebodistro/pull/42)) - [ ] Build nightlies from the `main` branch ([example](https://github.com/gazebo-tooling/release-tools/pull/437)) - [ ] homebrew-simulation: create formula and update dependencies ([example](https://github.com/osrf/homebrew-simulation/pull/14230)) - - [ ] docs (the collection’s page) ([example](https://github.com/ignitionrobotics/docs/pull/175)) + - [ ] docs (the collection’s page) ([example](https://github.com/gazebosim/docs/pull/175)) * diff --git a/.github/ISSUE_TEMPLATE/eol_collection.md b/.github/ISSUE_TEMPLATE/eol_collection.md index a2b9c27dd..a12bdf994 100644 --- a/.github/ISSUE_TEMPLATE/eol_collection.md +++ b/.github/ISSUE_TEMPLATE/eol_collection.md @@ -15,11 +15,11 @@ When opening PRs, add a link back to this issue for easier tracking. - [ ] Write a post to https://community.gazebosim.org/ announcing the community it has last X weeks for finishing their PRs. * -- [ ] Check library versions that can be EOL’d on the [versions table](https://github.com/ignitionrobotics/docs/blob/master/tools/versions.md) +- [ ] Check library versions that can be EOL’d on the [versions table](https://github.com/gazebosim/docs/blob/master/tools/versions.md) * - [ ] Make releases with outstanding changes in each library about to be EOL'ed * -- [ ] Archive release repos in [ignition-release](https://github.com/ignition-release) for all libraries after the last release has been made +- [ ] Archive release repos in [gazebo-release](https://github.com/gazebo-release) for all libraries after the last release has been made - [ ] Update `release-tools` after all releases have been made ([example](https://github.com/gazebo-tooling/release-tools/pull/369)) * - [ ] Deprecate homebrew bottles if they aren't deprecated yet (don't disable them) ([example](https://github.com/osrf/homebrew-simulation/pull/1785)) @@ -30,7 +30,7 @@ When opening PRs, add a link back to this issue for easier tracking. * - [ ] Remove from `pr-collection-labeler` ([example](https://github.com/gazebo-tooling/pr-collection-labeler/pull/13)) * -- [ ] Remove from `ros_ign` ([example](https://github.com/ignitionrobotics/ros_ign/pull/199)) +- [ ] Remove from `ros_ign` ([example](https://github.com/gazebosim/ros_ign/pull/199)) * - [ ] Sync latest binaries to http://packages.ros.org ([example](https://github.com/ros-infrastructure/reprepro-updater/pull/145)) * @@ -38,7 +38,7 @@ When opening PRs, add a link back to this issue for easier tracking. * - [ ] Close " support" ticket after final releases ([example](https://github.com/gazebo-tooling/release-tools/issues/297#issuecomment-1002232980)) * -- [ ] Update docs after EOL is complete ([example](https://github.com/ignitionrobotics/docs/pull/124)) +- [ ] Update docs after EOL is complete ([example](https://github.com/gazebosim/docs/pull/124)) * - [ ] Announce EOL on https://community.gazebosim.org/ ([example](https://community.gazebosim.org/t/ignition-blueprint-officially-end-of-life/764)) * diff --git a/.github/ISSUE_TEMPLATE/new_collection.md b/.github/ISSUE_TEMPLATE/new_collection.md index 88e3576d0..481d816da 100644 --- a/.github/ISSUE_TEMPLATE/new_collection.md +++ b/.github/ISSUE_TEMPLATE/new_collection.md @@ -14,11 +14,11 @@ When opening PRs, add a link back to this issue for easier tracking. --> - [ ] Create `gazebodistro` file for new collection ([example](https://github.com/gazebo-tooling/gazebodistro/pull/11)) -- [ ] Create `ign-` and `ign--release` repositories ([example source](https://github.com/ignitionrobotics/ign-edifice), [example release](https://github.com/ignition-release/ign-edifice-release)) +- [ ] Create `ign-` and `ign--release` repositories ([example source](https://github.com/gazebosim/gz-edifice), [example release](https://github.com/gazebo-release/ign-edifice-release)) - [ ] Create homebrew-simulation formula for new collection ([example](https://github.com/osrf/homebrew-simulation/pull/1173)) - [ ] Create collection view on release-tools and build nightlies ([example](https://github.com/gazebo-tooling/release-tools/pull/422/files)) - [ ] Add collection to buildfarmer dashboard ([example](https://github.com/osrf/buildfarmer/pull/183)) / scripts ([example](https://github.com/osrf/buildfarmer/pull/184)) -- [ ] Add docs for new collection ([example](https://github.com/ignitionrobotics/docs/pull/95)) +- [ ] Add docs for new collection ([example](https://github.com/gazebosim/docs/pull/95)) - [ ] Add new collection label: - [ ] Add to pr-collection-labeler ([example](https://github.com/gazebo-tooling/pr-collection-labeler/pull/11)) - [ ] Add to all libraries ([handy script](https://github.com/chapulina/github-label-maker/tree/chapulina/ign#ignition)) diff --git a/.github/ISSUE_TEMPLATE/release_collection.md b/.github/ISSUE_TEMPLATE/release_collection.md index 483af6d24..403ed392d 100644 --- a/.github/ISSUE_TEMPLATE/release_collection.md +++ b/.github/ISSUE_TEMPLATE/release_collection.md @@ -23,7 +23,7 @@ When opening PRs, add a link back to this issue for easier tracking. - [ ] Choose name for next release. - [ ] Merge each library forward from previous versions * -- [ ] Label pull requests being considered to enter release with [beta](https://github.com/search?q=org%3Aignitionrobotics+label%3Abeta&state=open) +- [ ] Label pull requests being considered to enter release with [beta](https://github.com/search?q=org%3Agazebosim+label%3Abeta&state=open) Pull requests opened against the new collection after then: - Bug fixes: will be considered for the initial release and labeled `beta` - New features: will not be considered for the initial release. @@ -31,7 +31,7 @@ When opening PRs, add a link back to this issue for easier tracking. * - [ ] Make collection pre-release after all libraries are pre-released. * -- [ ] Update public documentation to install from pre-releases instead of nightlies ([example](https://github.com/ignitionrobotics/docs/pull/196/files#diff-ebeee4adce7cb444f663b59020fb9f43f6f9adf36f63657b8afdeeea7a8530d1)). +- [ ] Update public documentation to install from pre-releases instead of nightlies ([example](https://github.com/gazebosim/docs/pull/196/files#diff-ebeee4adce7cb444f663b59020fb9f43f6f9adf36f63657b8afdeeea7a8530d1)). * - [ ] Change nightlies to pre-release for release branches on `gzdev` ([example](https://github.com/gazebo-tooling/gzdev/pull/35)) * @@ -54,8 +54,8 @@ When opening PRs, add a link back to this issue for easier tracking. * - [ ] Update `gazebodistro` ([example](https://github.com/gazebo-tooling/gazebodistro/pull/31)) * -- [ ] Make GitHub releases for all the X.0.0 versions ([example](https://github.com/ignitionrobotics/ign-transport/releases/tag/ignition-transport9_9.0.0)) -- [ ] Update docs to say release is stable ([example](https://github.com/ignitionrobotics/docs/pull/171)) +- [ ] Make GitHub releases for all the X.0.0 versions ([example](https://github.com/gazebosim/gz-transport/releases/tag/ignition-transport9_9.0.0)) +- [ ] Update docs to say release is stable ([example](https://github.com/gazebosim/docs/pull/171)) * - [ ] Make announcement * @@ -67,7 +67,7 @@ When opening PRs, add a link back to this issue for easier tracking. - [ ] Remove pre-release for release branches on `gzdev` ([example](https://github.com/gazebo-tooling/gzdev/pull/36)) * - [ ] Bump all `main` branches to the next major versioni (`X.0.0~pre1`) - - [ ] Source code ([example](https://github.com/ignitionrobotics/ign-common/pull/193)) + - [ ] Source code ([example](https://github.com/gazebosim/gz-common/pull/193)) * - [ ] Add files to `gazebodistro` ([example](https://github.com/gazebo-tooling/gazebodistro/pull/12)) * @@ -75,7 +75,7 @@ When opening PRs, add a link back to this issue for easier tracking. * - [ ] Create new `-release` repositories (use [this script](https://github.com/gazebo-tooling/release-tools/blob/master/release-repo-scripts/new_ignition_release_repos.bash)) - [ ] Enable nightlies for all `main` branches on `gzdev` ([example](https://github.com/gazebo-tooling/gzdev/pull/50)) - - [ ] Execute the tick-tock's "tock" for deprecations ([example](https://github.com/ignitionrobotics/ign-gazebo/pull/875)) + - [ ] Execute the tick-tock's "tock" for deprecations ([example](https://github.com/gazebosim/gz-sim/pull/875)) * - [ ] Update all repositories to default to the new stable branches. @@ -90,24 +90,24 @@ If the collection will be officially paired with a ROS 2 distro: Repo | Version | Release Branch | Feature frozen | Synced with previous versions? | Prerelease? | Code Frozen | Stable release? | Open PRs | Pending dep release -- | -- | -- | -- | -- | -- | -- | -- | -- | -- -[ign-tools](https://github.com/ignitionrobotics/ign-tools) | | ign-toolsN | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | [PRs](https://github.com/ignitionrobotics/ign-tools/pulls/?q=is%3Apr+is%3Aopen+label%3A%22beta%22) | -[ign-cmake](https://github.com/ignitionrobotics/ign-cmake) | | ign-cmakeN | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | [PRs](https://github.com/ignitionrobotics/ign-cmake/pulls/?q=is%3Apr+is%3Aopen+label%3A%22beta%22) | -[ign-math](https://github.com/ignitionrobotics/ign-math) | | ign-mathN | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | [PRs](https://github.com/ignitionrobotics/ign-math/pulls/?q=is%3Apr+is%3Aopen+label%3A%22beta%22) | -[ign-plugin](https://github.com/ignitionrobotics/ign-plugin) | | ign-pluginN | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | [PRs](https://github.com/ignitionrobotics/ign-plugin/pulls/?q=is%3Apr+is%3Aopen+label%3A%22beta%22) | -[ign-utils](https://github.com/ignitionrobotics/ign-utils) | | ign-utilsN | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | [PRs](https://github.com/ignitionrobotics/ign-utils/pulls/?q=is%3Apr+is%3Aopen+label%3A%22beta%22) | -[ign-common](https://github.com/ignitionrobotics/ign-common) | | ign-commonN | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | [PRs](https://github.com/ignitionrobotics/ign-common/pulls/?q=is%3Apr+is%3Aopen+label%3A%22beta%22) | -[ign-msgs](https://github.com/ignitionrobotics/ign-msgs) | | ign-msgsN | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | [PRs](https://github.com/ignitionrobotics/ign-msgs/pulls/?q=is%3Apr+is%3Aopen+label%3A%22beta%22) | -[ign-rendering](https://github.com/ignitionrobotics/ign-rendering) | | ign-renderingN | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | [PRs](https://github.com/ignitionrobotics/ign-rendering/pulls/?q=is%3Apr+is%3Aopen+label%3A%22beta%22) | -[sdformat](https://github.com/ignitionrobotics/sdformat) | | sdfN | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | [PRs](https://github.com/ignitionrobotics/sdformat/pulls/?q=is%3Apr+is%3Aopen+label%3A%22beta%22) | -[ign-fuel-tools](https://github.com/ignitionrobotics/ign-fuel-tools) | | ign-fuel-toolsN | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | [PRs](https://github.com/ignitionrobotics/ign-fuel-tools/pulls/?q=is%3Apr+is%3Aopen+label%3A%22beta%22) | -[ign-transport](https://github.com/ignitionrobotics/ign-transport) | | ign-transportN | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | [PRs](https://github.com/ignitionrobotics/ign-transport/pulls/?q=is%3Apr+is%3Aopen+label%3A%22beta%22) | -[ign-gui](https://github.com/ignitionrobotics/ign-gui) | | ign-guiN | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | [PRs](https://github.com/ignitionrobotics/ign-gui/pulls/?q=is%3Apr+is%3Aopen+label%3A%22beta%22) | -[ign-sensors](https://github.com/ignitionrobotics/ign-sensors) | | ign-sensorsN | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | [PRs](https://github.com/ignitionrobotics/ign-sensors/pulls/?q=is%3Apr+is%3Aopen+label%3A%22beta%22) | -[ign-physics](https://github.com/ignitionrobotics/ign-physics) | | ign-physicsN | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | [PRs](https://github.com/ignitionrobotics/ign-physics/pulls/?q=is%3Apr+is%3Aopen+label%3A%22beta%22) | -[ign-gazebo](https://github.com/ignitionrobotics/ign-gazebo) | | ign-gazeboN | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | [PRs](https://github.com/ignitionrobotics/ign-gazebo/pulls/?q=is%3Apr+is%3Aopen+label%3A%22beta%22) | -[ign-launch](https://github.com/ignitionrobotics/ign-launch) | | ign-launchN | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | [PRs](https://github.com/ignitionrobotics/ign-launch/pulls/?q=is%3Apr+is%3Aopen+label%3A%22beta%22) | -[ign-](https://github.com/ignitionrobotics/ign-) | | main | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | [PRs](https://github.com/ignitionrobotics/ign-/pulls/?q=is%3Apr+is%3Aopen+label%3A%22beta%22) | -[ros_ign](https://github.com/ignitionrobotics/ros_ign) | | | ✔️ | ✔️ | ✔️ | | | [PRs](https://github.com/ignitionrobotics/ros_ign/pulls/?q=is%3Apr+is%3Aopen+label%3A%22beta%22) | +[gz-tools](https://github.com/gazebosim/gz-tools) | | gz-toolsN | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | [PRs](https://github.com/gazebosim/gz-tools/pulls/?q=is%3Apr+is%3Aopen+label%3A%22beta%22) | +[gz-cmake](https://github.com/gazebosim/gz-cmake) | | gz-cmakeN | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | [PRs](https://github.com/gazebosim/gz-cmake/pulls/?q=is%3Apr+is%3Aopen+label%3A%22beta%22) | +[gz-math](https://github.com/gazebosim/gz-math) | | gz-mathN | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | [PRs](https://github.com/gazebosim/gz-math/pulls/?q=is%3Apr+is%3Aopen+label%3A%22beta%22) | +[gz-plugin](https://github.com/gazebosim/gz-plugin) | | gz-pluginN | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | [PRs](https://github.com/gazebosim/gz-plugin/pulls/?q=is%3Apr+is%3Aopen+label%3A%22beta%22) | +[gz-utils](https://github.com/gazebosim/gz-utils) | | gz-utilsN | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | [PRs](https://github.com/gazebosim/gz-utils/pulls/?q=is%3Apr+is%3Aopen+label%3A%22beta%22) | +[gz-common](https://github.com/gazebosim/gz-common) | | gz-commonN | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | [PRs](https://github.com/gazebosim/gz-common/pulls/?q=is%3Apr+is%3Aopen+label%3A%22beta%22) | +[gz-msgs](https://github.com/gazebosim/gz-msgs) | | gz-msgsN | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | [PRs](https://github.com/gazebosim/gz-msgs/pulls/?q=is%3Apr+is%3Aopen+label%3A%22beta%22) | +[gz-rendering](https://github.com/gazebosim/gz-rendering) | | gz-renderingN | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | [PRs](https://github.com/gazebosim/gz-rendering/pulls/?q=is%3Apr+is%3Aopen+label%3A%22beta%22) | +[sdformat](https://github.com/gazebosim/sdformat) | | sdfN | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | [PRs](https://github.com/gazebosim/sdformat/pulls/?q=is%3Apr+is%3Aopen+label%3A%22beta%22) | +[gz-fuel-tools](https://github.com/gazebosim/gz-fuel-tools) | | gz-fuel-toolsN | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | [PRs](https://github.com/gazebosim/gz-fuel-tools/pulls/?q=is%3Apr+is%3Aopen+label%3A%22beta%22) | +[gz-transport](https://github.com/gazebosim/gz-transport) | | gz-transportN | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | [PRs](https://github.com/gazebosim/gz-transport/pulls/?q=is%3Apr+is%3Aopen+label%3A%22beta%22) | +[gz-gui](https://github.com/gazebosim/gz-gui) | | gz-guiN | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | [PRs](https://github.com/gazebosim/gz-gui/pulls/?q=is%3Apr+is%3Aopen+label%3A%22beta%22) | +[gz-sensors](https://github.com/gazebosim/gz-sensors) | | gz-sensorsN | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | [PRs](https://github.com/gazebosim/gz-sensors/pulls/?q=is%3Apr+is%3Aopen+label%3A%22beta%22) | +[gz-physics](https://github.com/gazebosim/gz-physics) | | gz-physicsN | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | [PRs](https://github.com/gazebosim/gz-physics/pulls/?q=is%3Apr+is%3Aopen+label%3A%22beta%22) | +[gz-sim](https://github.com/gazebosim/gz-sim) | | gz-simN | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | [PRs](https://github.com/gazebosim/gz-sim/pulls/?q=is%3Apr+is%3Aopen+label%3A%22beta%22) | +[gz-launch](https://github.com/gazebosim/gz-launch) | | gz-launchN | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | [PRs](https://github.com/gazebosim/gz-launch/pulls/?q=is%3Apr+is%3Aopen+label%3A%22beta%22) | +[gz-](https://github.com/gazebosim/gz-) | | main | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | [PRs](https://github.com/gazebosim/gz-/pulls/?q=is%3Apr+is%3Aopen+label%3A%22beta%22) | +[ros_gz](https://github.com/gazebosim/ros_gz) | | | ✔️ | ✔️ | ✔️ | | | [PRs](https://github.com/gazebosim/ros_gz/pulls/?q=is%3Apr+is%3Aopen+label%3A%22beta%22) | Track stable release of metapackage `ignition-`. diff --git a/README.md b/README.md index 45f27cbbb..1591da239 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # Release Tools This repository holds scripts and tools that are used for testing and releasing -Gazebo Classic and Ignition software. +Gazebo software. ## Scripts @@ -33,10 +33,10 @@ Gazebo Classic and Ignition software. * Bump minor if there are new backwards-compatible features, like new APIs. * Bump patch if there are only bug fixes that won't affect the API. 1. Open pull request updating the version on the source code - * [Example pull request.](https://github.com/ignitionrobotics/ign-physics/pull/132) + * [Example pull request.](https://github.com/gazebosim/gz-physics/pull/132) * Update the version on `CMakeLists.txt`. * Include a link comparing the current branch to the latest release. - 1. [Example branch comparison.](https://github.com/ignitionrobotics/ign-gazebo/compare/ignition-gazebo3_3.5.0...ign-gazebo3) + 1. [Example branch comparison.](https://github.com/gazebosim/gz-sim/compare/ignition-gazebo3_3.5.0...ign-gazebo3) 1. Substitute the package version and name that are relevant to your release. * Update the changelog. 1. Use the [source_changelog.bash](source_changelog.bash) script to generate the Changelog.md entries newer than the most recent tag. @@ -46,10 +46,10 @@ Gazebo Classic and Ignition software. 1. Verify the Changelog.md entries using the GitHub branch comparison. * Update the migration guide as needed. * Wait for this pull request to be merged before proceeding. -1. Clone the appropriate release repository from https://github.com/ignition-release. - * For example, `git clone https://github.com/ignition-release/ign-physics3-release` +1. Clone the appropriate release repository from https://github.com/gazebo-release. + * For example, `git clone https://github.com/gazebo-release/ign-physics3-release` 1. If the versions of any dependencies have increased in the source code, open a pull request updating them on the release repo - * [Example pull request.](https://github.com/ignition-release/ign-gazebo3-release/pull/4) + * [Example pull request.](https://github.com/gazebo-release/gz-sim3-release/pull/4) 1. Wait for all pull requests opened above to be approved and merged. 1. Clone `release-tools` * `git clone https://github.com/gazebo-tooling/release-tools` @@ -62,7 +62,7 @@ Gazebo Classic and Ignition software. 1. `cd ` 1. Make sure you're in the branch to be released, after the pull request bumping the version has been merged. 1. Dry-run `release.py` with the appropriate arguments. Some examples: - 1. New Ignition minor release and Gazebo classic release: + 1. New Gazebo minor release and Gazebo classic release: 1. `/release.py -r 1 ign-physics3 3.1.0 secrettoken --dry-run` 1. `/release.py -r 1 gazebo11 11.2.0 secrettoken --dry-run` 1. Pre-release: `/release.py -r 1 ign-physics3 3.0.0~pre1 --upload-to-repo=prerelease secrettoken --dry-run` @@ -71,20 +71,20 @@ Gazebo Classic and Ignition software. * Several `-debbuilder` jobs have been queued in https://build.osrfoundation.org/ and watch those jobs to see if any of them fail. While it would be easier to track the status of these jobs if there was a [dashboard](https://github.com/gazebo-tooling/release-tools/issues/295), you can watch the page for a specific debbuild, such as https://build.osrfoundation.org/job/ign-gui3-debbuilder/. - 1. If you observe a failure of an Ignition Dome package, check the list of supported + 1. If you observe a failure of an Gazebo Dome package, check the list of supported architectures in [issue #297](https://github.com/gazebo-tooling/release-tools/issues/297); for example Bionic/armhf and Focal/armhf are not supported for Dome. To disable builds for a specific architecture, add a file to the `-release` repository that starts with `.releasepy_NO_ARCH_` and append the name of the architecture to be excluded. The file can be added to the - root of the `-release` repository (like [.releasepy_NO_ARCH_ARM64 in ign-gazebo2-release](https://github.com/ignition-release/ign-gazebo2-release/blob/master/.releasepy_NO_ARCH_ARM64)) - or in a distro-specific sub-folder (like [bionic/.releasepy_NO_ARCH_armhf in ign-launch2-release](https://github.com/ignition-release/ign-launch2-release/blob/master/bionic/.releasepy_NO_ARCH_armhf)). + root of the `-release` repository (like [.releasepy_NO_ARCH_ARM64 in ign-gazebo2-release](https://github.com/gazebo-release/ign-gazebo2-release/blob/master/.releasepy_NO_ARCH_ARM64)) + or in a distro-specific sub-folder (like [bionic/.releasepy_NO_ARCH_armhf in ign-launch2-release](https://github.com/gazebo-release/ign-launch2-release/blob/master/bionic/.releasepy_NO_ARCH_armhf)). The architecture suffix is not case-sensitive. 1. To check if a debbuild has previously succeeded for a given architecture, you can look at packages.osrfoundation.org to see the most recent successful builds. For example, you can see the most recent [Ubuntu builds of ignition-gazebo3](https://packages.osrfoundation.org/gazebo/ubuntu-stable/pool/main/i/ignition-gazebo3/) or the [Debian builds of ignition-gazebo3](https://packages.osrfoundation.org/gazebo/debian-stable/pool/main/i/ignition-gazebo3/). 1. If the failure is on a supported architecture, check the source repository for an existing report of this failure and if none - exists, please report the failure (see [ignitionrobotics/ign-math#161](https://github.com/ignitionrobotics/ign-math/issues/161) + exists, please report the failure (see [gazebosim/gz-math#161](https://github.com/gazebosim/gz-math/issues/161) for an example). * A pull request was opened to https://github.com/osrf/homebrew-simulation 1. This pull request may take a minute or two to open. diff --git a/bloom/release-bloom.py b/bloom/release-bloom.py index b808b17d3..f5afce442 100755 --- a/bloom/release-bloom.py +++ b/bloom/release-bloom.py @@ -36,7 +36,7 @@ def parse_args(argv): parser.add_argument('--upload-to-repo', dest='upload_to_repository', default="stable", help='OSRF repo to upload: stable | prerelease | nightly') parser.add_argument('--ignition-version', action='store', default=None, - help='Ignition version to use in the binary packages') + help='Gazebo version to use in the binary packages') args = parser.parse_args() DRY_RUN = args.dry_run diff --git a/jenkins-scripts/docker/lib/_generic_linux_compilation_build.sh.bash b/jenkins-scripts/docker/lib/_generic_linux_compilation_build.sh.bash index 511807060..8576dcb68 100644 --- a/jenkins-scripts/docker/lib/_generic_linux_compilation_build.sh.bash +++ b/jenkins-scripts/docker/lib/_generic_linux_compilation_build.sh.bash @@ -55,7 +55,7 @@ cat >> build.sh << DELIM_BUILD_DEPS dependency_repo="osrf/${dep}" else # need to replace _ by - - dependency_repo="ignitionrobotics/${dep//_/-}" + dependency_repo="gazebosim/${dep//_/-}" fi git clone http://github.com/\$dependency_repo -b ${dep_branch} \ diff --git a/jenkins-scripts/docker/lib/gazebo-base-default.bash b/jenkins-scripts/docker/lib/gazebo-base-default.bash index e404348a3..afbfd8b28 100644 --- a/jenkins-scripts/docker/lib/gazebo-base-default.bash +++ b/jenkins-scripts/docker/lib/gazebo-base-default.bash @@ -67,8 +67,8 @@ if ${COVERAGE_ENABLED} ; then # Download and install Bullseyes cd $WORKSPACE rm -fr $WORKSPACE/Bulls* - - # Look for current version. NOT IN USE since we lost the maintenance support on 2014 + + # Look for current version. NOT IN USE since we lost the maintenance support on 2014 # reenable if the support is back. # wget http://www.bullseye.com/download/ -O bull_index.html # BULL_TAR=\$( grep -R BullseyeCoverage-.*-Linux-x64.tar bull_index.html | head -n 1 | sed 's/.*">//' | sed 's/<.*//' ) @@ -134,7 +134,7 @@ cat >> build.sh << DELIM_BUILD_DEPS dependency_repo="osrf/${dep}" else # need to replace _ by - - dependency_repo="ignitionrobotics/${dep/_/-}" + dependency_repo="gazebosim/${dep/_/-}" fi git clone http://github.com/\$dependency_repo -b ${dep_branch} \ diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index a5abc360f..ab23bf69d 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -502,7 +502,7 @@ all_debbuilders().each { debbuilder_name -> if (debbuilder_name.contains("gazebo") || debbuilder_name == "transport7") extra_str="export NEED_C17_COMPILER=true" - // Ignition physics consumes huge amount of memory making arm node to FAIL + // Gazebo physics consumes huge amount of memory making arm node to FAIL // Force here to use one compilation thread if (debbuilder_name.contains("-physics")) extra_str += '\nif [ $(uname -m) = "aarch64" ]; then export MAKE_JOBS=1; fi' diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index a574ce5a3..1bea5cdab 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -408,7 +408,7 @@ ignition_collections.each { ign_collection -> } } - // Ignition dashboards + // Gazebo dashboards dashboardView("ign-${ign_collection_name}") { jobs { diff --git a/jenkins-scripts/dsl/ros1_ign_bridge.dsl b/jenkins-scripts/dsl/ros1_ign_bridge.dsl index 114ed3eeb..894d8f8de 100644 --- a/jenkins-scripts/dsl/ros1_ign_bridge.dsl +++ b/jenkins-scripts/dsl/ros1_ign_bridge.dsl @@ -34,7 +34,7 @@ bridge_packages.each { pkg -> parameters { stringParam("PACKAGE","$pkg_dashed","Package name to be built") - stringParam("IGNITION_VERSION", '', 'Ignition release supported in the binaries') + stringParam("IGNITION_VERSION", '', 'Gazebo release supported in the binaries') stringParam("VERSION",null,"Packages version to be built") stringParam("RELEASE_VERSION", null, "Packages release version") stringParam("LINUX_DISTRO", 'ubuntu', "Linux distribution to build packages for") diff --git a/jenkins-scripts/lib/gazebo-base-windows.bat b/jenkins-scripts/lib/gazebo-base-windows.bat index 81bee5b53..58cddc40d 100644 --- a/jenkins-scripts/lib/gazebo-base-windows.bat +++ b/jenkins-scripts/lib/gazebo-base-windows.bat @@ -64,7 +64,7 @@ echo # END SECTION echo # BEGIN SECTION: compile and install ign-fuel-tools set IGN_TRANSPORT_DIR=%WORKSPACE%\ign-fuel-tools if EXIST %IGN_TRANSPORT_DIR% ( rmdir /s /q %IGN_TRANSPORT_DIR% ) -git clone https://github.com/ignitionrobotics/ign-fuel-tools %IGN_TRANSPORT_DIR% -b ign-fuel-tools4 +git clone https://github.com/gazebosim/gz-fuel-tools %IGN_TRANSPORT_DIR% -b ign-fuel-tools4 set VCS_DIRECTORY=ign-fuel-tools set KEEP_WORKSPACE=TRUE set ENABLE_TESTS=FALSE @@ -75,7 +75,7 @@ echo # END SECTION echo # BEGIN SECTION: compile and install ign-transport set IGN_TRANSPORT_DIR=%WORKSPACE%\ign-transport if EXIST %IGN_TRANSPORT_DIR% ( rmdir /s /q %IGN_TRANSPORT_DIR% ) -git clone https://github.com/ignitionrobotics/ign-transport %IGN_TRANSPORT_DIR% -b ign-transport8 +git clone https://github.com/gazebosim/gz-transport %IGN_TRANSPORT_DIR% -b ign-transport8 set VCS_DIRECTORY=ign-transport set KEEP_WORKSPACE=TRUE set ENABLE_TESTS=FALSE diff --git a/jenkins-scripts/lib/gazebo9_10-base-windows.bat b/jenkins-scripts/lib/gazebo9_10-base-windows.bat index fcfb820db..7f1550c1d 100644 --- a/jenkins-scripts/lib/gazebo9_10-base-windows.bat +++ b/jenkins-scripts/lib/gazebo9_10-base-windows.bat @@ -55,7 +55,7 @@ echo # END SECTION echo # BEGIN SECTION: compile and install ign-transport set IGN_TRANSPORT_DIR=%WORKSPACE%\ign-transport if EXIST %IGN_TRANSPORT_DIR% ( rmdir /s /q %IGN_TRANSPORT_DIR% ) -git clone https://github.com/ignitionrobotics/ign-transport %IGN_TRANSPORT_DIR% -b ign-transport4 +git clone https://github.com/gazebosim/gz-transport %IGN_TRANSPORT_DIR% -b ign-transport4 set VCS_DIRECTORY=ign-transport set KEEP_WORKSPACE=TRUE set ENABLE_TESTS=FALSE diff --git a/jenkins-scripts/lib/sdformat-base-windows.bat b/jenkins-scripts/lib/sdformat-base-windows.bat index 1595a130b..9ea8ba71b 100644 --- a/jenkins-scripts/lib/sdformat-base-windows.bat +++ b/jenkins-scripts/lib/sdformat-base-windows.bat @@ -35,7 +35,7 @@ echo # END SECTION IF %USE_IGNITION_ZIP% == FALSE ( echo # BEGIN SECTION: compile and install ign-math IF exist %WORKSPACE%\ign-math ( rmdir /s /q %WORKSPACE%\ign-math ) || goto :error - git clone https://github.com/ignitionrobotics/ign-math %WORKSPACE%\ign-math -u %IGNMATH_BRANCH% || goto :error + git clone https://github.com/gazebosim/gz-math %WORKSPACE%\ign-math -u %IGNMATH_BRANCH% || goto :error set VCS_DIRECTORY=ign-math set KEEP_WORKSPACE=TRUE call "%SCRIPT_DIR%\lib\project-default-devel-windows.bat" diff --git a/jenkins-scripts/lib/windows_library.bat b/jenkins-scripts/lib/windows_library.bat index f5a8e2527..24fc1fcda 100644 --- a/jenkins-scripts/lib/windows_library.bat +++ b/jenkins-scripts/lib/windows_library.bat @@ -123,10 +123,10 @@ set IGN_PROJECT_DEPENDENCY_DIR=%LOCAL_WS%\%1 if exist %IGN_PROJECT_DEPENDENCY_DIR% ( rmdir /s /q %IGN_PROJECT_DEPENDENCY_DIR% ) if "%2"=="" ( echo Installing master branch of %1 - git clone https://github.com/ignitionrobotics/%1 %IGN_PROJECT_DEPENDENCY_DIR% -b master + git clone https://github.com/gazebosim/%1 %IGN_PROJECT_DEPENDENCY_DIR% -b master ) else ( echo Installing branch %2 of %1 - git clone https://github.com/ignitionrobotics/%1 %IGN_PROJECT_DEPENDENCY_DIR% -b %2 + git clone https://github.com/gazebosim/%1 %IGN_PROJECT_DEPENDENCY_DIR% -b %2 ) cd /d %IGN_PROJECT_DEPENDENCY_DIR% call .\configure.bat diff --git a/release-repo-scripts/bump_major_version.bash b/release-repo-scripts/bump_major_version.bash index 45421ee02..c961ea111 100755 --- a/release-repo-scripts/bump_major_version.bash +++ b/release-repo-scripts/bump_major_version.bash @@ -119,8 +119,8 @@ for f in $(find . -name control -type f); do # Change short version of ign-too sed -i -e "s:${old_software_name/ignition/ign}:${new_software_name/ignition/ign}:g" "${f}" # Change bitbucket for github - sed -i -e "s:https\://bitbucket.org/ignitionrobotics/:https\://github.com/ignition-release/:g" "${f}" - sed -i -e "s:https\://bitbucket.org/osrf/:https\://github.com/ignition-release/:g" "${f}" + sed -i -e "s:https\://bitbucket.org/ignitionrobotics/:https\://github.com/gazebo-release/:g" "${f}" + sed -i -e "s:https\://bitbucket.org/osrf/:https\://github.com/gazebo-release/:g" "${f}" echo " * Safety checks in control" # 1. Check for custom version modifiers in control file # 2. Breaks or replaces clause? diff --git a/release-repo-scripts/new_ignition_release_repos.bash b/release-repo-scripts/new_ignition_release_repos.bash index 3d7f151ed..4bb5b6639 100755 --- a/release-repo-scripts/new_ignition_release_repos.bash +++ b/release-repo-scripts/new_ignition_release_repos.bash @@ -42,7 +42,7 @@ empty_directory() for repo_name in ${NEW_REPOS}; do echo "Procesing ${repo_name}" repo_fname="${repo_name}-release" - repo_github="ignition-release/${repo_fname}" + repo_github="gazebo-release/${repo_fname}" echo " + creating the repository " # check destination directory diff --git a/release.py b/release.py index df0412a6d..268b0bc3d 100755 --- a/release.py +++ b/release.py @@ -166,12 +166,12 @@ def get_release_repository_info(package): # Do not use git@github method since it fails in non existant repositories # asking for stdin user/pass. Same happen if no user/pass is provided # using the fake foo:foo here seems to work - github_test_url = "https://foo:foo@github.com/ignition-release/" + package + "-release" + github_test_url = "https://foo:foo@github.com/gazebo-release/" + package + "-release" if (github_repo_exists(github_test_url)): - github_url = "https://github.com/ignition-release/" + package + "-release" + github_url = "https://github.com/gazebo-release/" + package + "-release" return 'git', github_url - error("release repository not found in github.com/ignition-release") + error("release repository not found in github.com/gazebo-release") def download_release_repository(package, release_branch): vcs, url = get_release_repository_info(package) diff --git a/terminal-dashboard/unreleased.bash b/terminal-dashboard/unreleased.bash index 92cc1f5ae..8a5d5efb0 100755 --- a/terminal-dashboard/unreleased.bash +++ b/terminal-dashboard/unreleased.bash @@ -40,7 +40,7 @@ for LIB in $(get_libraries_by_collection "${COLLECTION}" ); do COMPACT_VERSION=${PKG_VERSION#* } COMPACT_VERSION=${COMPACT_VERSION%%-*} - echo https://github.com/ignitionrobotics/${LIB_NAME}/compare/${LIB}_${COMPACT_VERSION}...${LIB_SHORT} + echo https://github.com/gazebosim/${LIB_NAME}/compare/${LIB}_${COMPACT_VERSION}...${LIB_SHORT} done From 7f052cf06d645d2a2291aee5a955c8d1c66618d7 Mon Sep 17 00:00:00 2001 From: Louise Poubel Date: Sat, 25 Jun 2022 10:31:25 -0700 Subject: [PATCH 162/540] Windows colcon: Rename ignition package to gz if not in ws (#754) * Windows colcon: Attempt gz package if ignition not in ws Signed-off-by: Louise Poubel * debug Signed-off-by: Louise Poubel * echo list Signed-off-by: Louise Poubel * findstr Signed-off-by: Louise Poubel * findstr Signed-off-by: Louise Poubel * find Signed-off-by: Louise Poubel * Debug output, separate section Signed-off-by: Louise Poubel * cleanup Signed-off-by: Louise Poubel * Move msg Signed-off-by: Louise Poubel --- jenkins-scripts/lib/windows_library.bat | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/jenkins-scripts/lib/windows_library.bat b/jenkins-scripts/lib/windows_library.bat index 24fc1fcda..f42189a8d 100644 --- a/jenkins-scripts/lib/windows_library.bat +++ b/jenkins-scripts/lib/windows_library.bat @@ -210,6 +210,23 @@ goto :EOF set COLCON_PACKAGE=%1 +:: Check if package is in colcon workspace +echo # BEGIN SECTION: Update package %COLCON_PACKAGE% from ignition to gz if needed +echo Packages in workspace: +colcon list --names-only + +colcon list --names-only | find "%COLCON_PACKAGE%" +if errorlevel 1 ( + set COLCON_PACKAGE=%COLCON_PACKAGE:ignition=gz% +) +colcon list --names-only | find "%COLCON_PACKAGE%" +if errorlevel 1 ( + echo Failed to find package %COLCON_PACKAGE% in workspace. + goto :error +) +echo Using package name %COLCON_PACKAGE% +echo # END SECTION + :: two runs to get the dependencies built with testing and the package under :: test build with tests echo # BEGIN SECTION: colcon compilation without test for dependencies of %COLCON_PACKAGE% From cf6e4c68fe90cf5651c919a0802f53894b74a21a Mon Sep 17 00:00:00 2001 From: methylDragon Date: Sat, 25 Jun 2022 11:06:05 -0700 Subject: [PATCH 163/540] Windows colcon: Rename ignition package to gz if not in ws (2) (#755) --- .../lib/colcon-default-devel-windows.bat | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/jenkins-scripts/lib/colcon-default-devel-windows.bat b/jenkins-scripts/lib/colcon-default-devel-windows.bat index dfd3c408d..5fe37b4c8 100644 --- a/jenkins-scripts/lib/colcon-default-devel-windows.bat +++ b/jenkins-scripts/lib/colcon-default-devel-windows.bat @@ -35,6 +35,23 @@ if "%COLCON_AUTO_MAJOR_VERSION%" == "true" ( echo "MAJOR_VERSION detected: !PKG_MAJOR_VERSION!" ) +:: Check if package is in colcon workspace +echo # BEGIN SECTION: Update package %COLCON_PACKAGE% from ignition to gz if needed +echo Packages in workspace: +colcon list --names-only + +colcon list --names-only | find "%COLCON_PACKAGE%" +if errorlevel 1 ( + set COLCON_PACKAGE=%COLCON_PACKAGE:ignition=gz% +) +colcon list --names-only | find "%COLCON_PACKAGE%" +if errorlevel 1 ( + echo Failed to find package %COLCON_PACKAGE% in workspace. + goto :error +) +echo Using package name %COLCON_PACKAGE% +echo # END SECTION + set TEST_RESULT_PATH=%WORKSPACE%\ws\build\%COLCON_PACKAGE%\test_results setlocal ENABLEDELAYEDEXPANSION From cf125a11ca723f6876d9e5b74ca2b9d6def6ed4a Mon Sep 17 00:00:00 2001 From: methylDragon Date: Sat, 25 Jun 2022 14:59:58 -0700 Subject: [PATCH 164/540] ign -> gz : Release Tools Migrate Project Names (#752) --- jenkins-scripts/docker/ign_cmake-compilation.bash | 2 +- jenkins-scripts/docker/ign_common-compilation.bash | 2 +- jenkins-scripts/docker/ign_fuel-tools-compilation.bash | 2 +- jenkins-scripts/docker/ign_gazebo-compilation.bash | 2 +- jenkins-scripts/docker/ign_gui-compilation.bash | 2 +- jenkins-scripts/docker/ign_launch-compilation.bash | 2 +- jenkins-scripts/docker/ign_math-compilation.bash | 2 +- jenkins-scripts/docker/ign_msgs-compilation.bash | 2 +- jenkins-scripts/docker/ign_physics-compilation.bash | 2 +- jenkins-scripts/docker/ign_plugin-compilation.bash | 2 +- jenkins-scripts/docker/ign_rendering-compilation.bash | 2 +- jenkins-scripts/docker/ign_sensors-compilation.bash | 2 +- jenkins-scripts/docker/ign_transport-compilation.bash | 2 +- jenkins-scripts/docker/ign_utils-compilation.bash | 2 +- jenkins-scripts/ign_common-default-devel-windows-amd64.bat | 2 +- jenkins-scripts/ign_fuel-tools-default-devel-windows-amd64.bat | 2 +- jenkins-scripts/ign_gazebo-default-devel-windows-amd64.bat | 2 +- jenkins-scripts/ign_gui-default-devel-windows-amd64.bat | 2 +- jenkins-scripts/ign_launch-default-devel-windows-amd64.bat | 2 +- jenkins-scripts/ign_math-default-devel-windows-amd64.bat | 2 +- jenkins-scripts/ign_msgs-default-devel-windows-amd64.bat | 2 +- jenkins-scripts/ign_physics-default-devel-windows-amd64.bat | 2 +- jenkins-scripts/ign_plugin-default-devel-windows-amd64.bat | 2 +- jenkins-scripts/ign_rendering-default-devel-windows-amd64.bat | 2 +- jenkins-scripts/ign_sensors-default-devel-windows-amd64.bat | 2 +- jenkins-scripts/ign_tools-default-devel-windows-amd64.bat | 2 +- jenkins-scripts/ign_transport-default-devel-windows-amd64.bat | 2 +- jenkins-scripts/ign_utils-default-devel-windows-amd64.bat | 2 +- 28 files changed, 28 insertions(+), 28 deletions(-) diff --git a/jenkins-scripts/docker/ign_cmake-compilation.bash b/jenkins-scripts/docker/ign_cmake-compilation.bash index d5b396489..732e98c52 100644 --- a/jenkins-scripts/docker/ign_cmake-compilation.bash +++ b/jenkins-scripts/docker/ign_cmake-compilation.bash @@ -31,6 +31,6 @@ if ! [[ ${IGN_CMAKE_MAJOR_VERSION} =~ ^-?[0-9]+$ ]]; then exit -1 fi -export GZDEV_PROJECT_NAME="ignition-cmake${IGN_CMAKE_MAJOR_VERSION}" +export GZDEV_PROJECT_NAME="gz-cmake${IGN_CMAKE_MAJOR_VERSION}" . ${SCRIPT_DIR}/lib/generic-building-base.bash diff --git a/jenkins-scripts/docker/ign_common-compilation.bash b/jenkins-scripts/docker/ign_common-compilation.bash index adfc18df9..c70ddb0d8 100644 --- a/jenkins-scripts/docker/ign_common-compilation.bash +++ b/jenkins-scripts/docker/ign_common-compilation.bash @@ -32,6 +32,6 @@ if [[ ${IGN_COMMON_MAJOR_VERSION} -ge 3 ]]; then export NEED_C17_COMPILER=true fi -export GZDEV_PROJECT_NAME="ignition-common${IGN_COMMON_MAJOR_VERSION}" +export GZDEV_PROJECT_NAME="gz-common${IGN_COMMON_MAJOR_VERSION}" . ${SCRIPT_DIR}/lib/generic-building-base.bash diff --git a/jenkins-scripts/docker/ign_fuel-tools-compilation.bash b/jenkins-scripts/docker/ign_fuel-tools-compilation.bash index 34fbcf2b7..aafe604d0 100644 --- a/jenkins-scripts/docker/ign_fuel-tools-compilation.bash +++ b/jenkins-scripts/docker/ign_fuel-tools-compilation.bash @@ -29,6 +29,6 @@ if ! [[ ${IGN_FUEL_TOOLS_MAJOR_VERSION} =~ ^-?[0-9]+$ ]]; then exit -1 fi -export GZDEV_PROJECT_NAME="ignition-fuel-tools${IGN_FUEL_TOOLS_MAJOR_VERSION}" +export GZDEV_PROJECT_NAME="gz-fuel-tools${IGN_FUEL_TOOLS_MAJOR_VERSION}" . ${SCRIPT_DIR}/lib/generic-building-base.bash diff --git a/jenkins-scripts/docker/ign_gazebo-compilation.bash b/jenkins-scripts/docker/ign_gazebo-compilation.bash index 9f5f56e87..b7f81a6ae 100644 --- a/jenkins-scripts/docker/ign_gazebo-compilation.bash +++ b/jenkins-scripts/docker/ign_gazebo-compilation.bash @@ -31,6 +31,6 @@ fi export NEED_C17_COMPILER=true export GPU_SUPPORT_NEEDED=true -export GZDEV_PROJECT_NAME="ignition-gazebo${IGN_GAZEBO_MAJOR_VERSION}" +export GZDEV_PROJECT_NAME="gz-sim${IGN_GAZEBO_MAJOR_VERSION}" . ${SCRIPT_DIR}/lib/generic-building-base.bash diff --git a/jenkins-scripts/docker/ign_gui-compilation.bash b/jenkins-scripts/docker/ign_gui-compilation.bash index df7e90850..9081132ba 100644 --- a/jenkins-scripts/docker/ign_gui-compilation.bash +++ b/jenkins-scripts/docker/ign_gui-compilation.bash @@ -33,6 +33,6 @@ if [[ ${IGN_GUI_MAJOR_VERSION} -ge 1 ]]; then fi export GPU_SUPPORT_NEEDED=true -export GZDEV_PROJECT_NAME="ignition-gui${IGN_GUI_MAJOR_VERSION}" +export GZDEV_PROJECT_NAME="gz-gui${IGN_GUI_MAJOR_VERSION}" . ${SCRIPT_DIR}/lib/generic-building-base.bash diff --git a/jenkins-scripts/docker/ign_launch-compilation.bash b/jenkins-scripts/docker/ign_launch-compilation.bash index de389df11..691b21cee 100644 --- a/jenkins-scripts/docker/ign_launch-compilation.bash +++ b/jenkins-scripts/docker/ign_launch-compilation.bash @@ -29,6 +29,6 @@ if ! [[ ${IGN_LAUNCH_MAJOR_VERSION} =~ ^-?[0-9]+$ ]]; then fi export NEED_C17_COMPILER=true -export GZDEV_PROJECT_NAME="ignition-launch${IGN_LAUNCH_MAJOR_VERSION}" +export GZDEV_PROJECT_NAME="gz-launch${IGN_LAUNCH_MAJOR_VERSION}" . ${SCRIPT_DIR}/lib/generic-building-base.bash diff --git a/jenkins-scripts/docker/ign_math-compilation.bash b/jenkins-scripts/docker/ign_math-compilation.bash index d6fe005c8..528e9fae3 100644 --- a/jenkins-scripts/docker/ign_math-compilation.bash +++ b/jenkins-scripts/docker/ign_math-compilation.bash @@ -33,6 +33,6 @@ if [[ ${IGN_MATH_MAJOR_VERSION} -ge 6 ]]; then export NEED_C17_COMPILER=true fi -export GZDEV_PROJECT_NAME="ignition-math${IGN_MATH_MAJOR_VERSION}" +export GZDEV_PROJECT_NAME="gz-math${IGN_MATH_MAJOR_VERSION}" . "${SCRIPT_DIR}/lib/generic-building-base.bash" diff --git a/jenkins-scripts/docker/ign_msgs-compilation.bash b/jenkins-scripts/docker/ign_msgs-compilation.bash index 7a47506aa..ab10e9e7b 100644 --- a/jenkins-scripts/docker/ign_msgs-compilation.bash +++ b/jenkins-scripts/docker/ign_msgs-compilation.bash @@ -32,6 +32,6 @@ if [[ ${IGN_MSGS_MAJOR_VERSION} -ge 3 ]]; then export NEED_C17_COMPILER=true fi -export GZDEV_PROJECT_NAME="ignition-msgs${IGN_MSGS_MAJOR_VERSION}" +export GZDEV_PROJECT_NAME="gz-msgs${IGN_MSGS_MAJOR_VERSION}" . ${SCRIPT_DIR}/lib/generic-building-base.bash diff --git a/jenkins-scripts/docker/ign_physics-compilation.bash b/jenkins-scripts/docker/ign_physics-compilation.bash index 3516e14b4..257e21548 100644 --- a/jenkins-scripts/docker/ign_physics-compilation.bash +++ b/jenkins-scripts/docker/ign_physics-compilation.bash @@ -29,6 +29,6 @@ if ! [[ ${IGN_PHYSICS_MAJOR_VERSION} =~ ^-?[0-9]+$ ]]; then fi export NEED_C17_COMPILER=true -export GZDEV_PROJECT_NAME="ignition-physics${IGN_PHYSICS_MAJOR_VERSION}" +export GZDEV_PROJECT_NAME="gz-physics${IGN_PHYSICS_MAJOR_VERSION}" . ${SCRIPT_DIR}/lib/generic-building-base.bash diff --git a/jenkins-scripts/docker/ign_plugin-compilation.bash b/jenkins-scripts/docker/ign_plugin-compilation.bash index 245e17ce3..5c722e1a7 100644 --- a/jenkins-scripts/docker/ign_plugin-compilation.bash +++ b/jenkins-scripts/docker/ign_plugin-compilation.bash @@ -32,6 +32,6 @@ if [[ ${IGN_PLUGIN_MAJOR_VERSION} -ge 6 ]]; then export NEED_C17_COMPILER=true fi -export GZDEV_PROJECT_NAME="ignition-plugin${IGN_PLUGIN_MAJOR_VERSION}" +export GZDEV_PROJECT_NAME="gz-plugin${IGN_PLUGIN_MAJOR_VERSION}" . ${SCRIPT_DIR}/lib/generic-building-base.bash diff --git a/jenkins-scripts/docker/ign_rendering-compilation.bash b/jenkins-scripts/docker/ign_rendering-compilation.bash index d9607edb3..bf5cde07f 100644 --- a/jenkins-scripts/docker/ign_rendering-compilation.bash +++ b/jenkins-scripts/docker/ign_rendering-compilation.bash @@ -33,7 +33,7 @@ if [[ ${IGN_RENDERING_MAJOR_VERSION} -ge 1 ]]; then fi export GPU_SUPPORT_NEEDED=true -export GZDEV_PROJECT_NAME="ignition-rendering${IGN_RENDERING_MAJOR_VERSION}" +export GZDEV_PROJECT_NAME="gz-rendering${IGN_RENDERING_MAJOR_VERSION}" export BUILDING_EXTRA_CMAKE_PARAMS+=" -DSKIP_optix=true" diff --git a/jenkins-scripts/docker/ign_sensors-compilation.bash b/jenkins-scripts/docker/ign_sensors-compilation.bash index cbdbcd07d..14623fd11 100644 --- a/jenkins-scripts/docker/ign_sensors-compilation.bash +++ b/jenkins-scripts/docker/ign_sensors-compilation.bash @@ -31,6 +31,6 @@ fi export NEED_C17_COMPILER=true export GPU_SUPPORT_NEEDED=true -export GZDEV_PROJECT_NAME="ignition-sensors${IGN_SENSORS_MAJOR_VERSION}" +export GZDEV_PROJECT_NAME="gz-sensors${IGN_SENSORS_MAJOR_VERSION}" . ${SCRIPT_DIR}/lib/generic-building-base.bash diff --git a/jenkins-scripts/docker/ign_transport-compilation.bash b/jenkins-scripts/docker/ign_transport-compilation.bash index 26a77e524..c7e393f1d 100644 --- a/jenkins-scripts/docker/ign_transport-compilation.bash +++ b/jenkins-scripts/docker/ign_transport-compilation.bash @@ -32,6 +32,6 @@ if [[ ${IGN_TRANSPORT_MAJOR_VERSION} -ge 6 ]]; then export NEED_C17_COMPILER=true fi -export GZDEV_PROJECT_NAME="ignition-transport${IGN_TRANSPORT_MAJOR_VERSION}" +export GZDEV_PROJECT_NAME="gz-transport${IGN_TRANSPORT_MAJOR_VERSION}" . "${SCRIPT_DIR}/lib/generic-building-base.bash" diff --git a/jenkins-scripts/docker/ign_utils-compilation.bash b/jenkins-scripts/docker/ign_utils-compilation.bash index a91d40638..b35263a63 100644 --- a/jenkins-scripts/docker/ign_utils-compilation.bash +++ b/jenkins-scripts/docker/ign_utils-compilation.bash @@ -30,6 +30,6 @@ fi export NEED_C17_COMPILER=true -export GZDEV_PROJECT_NAME="ignition-utils${IGN_UTILS_MAJOR_VERSION}" +export GZDEV_PROJECT_NAME="gz-utils${IGN_UTILS_MAJOR_VERSION}" . ${SCRIPT_DIR}/lib/generic-building-base.bash diff --git a/jenkins-scripts/ign_common-default-devel-windows-amd64.bat b/jenkins-scripts/ign_common-default-devel-windows-amd64.bat index 8f42e7dd5..27cf5c9d4 100644 --- a/jenkins-scripts/ign_common-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_common-default-devel-windows-amd64.bat @@ -10,7 +10,7 @@ if %IGN_MAJOR_VERSION% GEQ 5 ( set DEPEN_PKGS=%DEPEN_PKGS% gdal ) -set COLCON_PACKAGE=ignition-common +set COLCON_PACKAGE=gz-common set COLCON_AUTO_MAJOR_VERSION=true call "%SCRIPT_DIR%\lib\colcon-default-devel-windows.bat" diff --git a/jenkins-scripts/ign_fuel-tools-default-devel-windows-amd64.bat b/jenkins-scripts/ign_fuel-tools-default-devel-windows-amd64.bat index 3753288db..665379493 100644 --- a/jenkins-scripts/ign_fuel-tools-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_fuel-tools-default-devel-windows-amd64.bat @@ -7,7 +7,7 @@ set IGN_CLEAN_WORKSPACE=true :: tinyxml2 from msgs set DEPEN_PKGS=libyaml libzip tinyxml2 openssl protobuf curl -set COLCON_PACKAGE=ignition-fuel_tools +set COLCON_PACKAGE=gz-fuel_tools set COLCON_AUTO_MAJOR_VERSION=true call "%SCRIPT_DIR%\lib\colcon-default-devel-windows.bat" \ No newline at end of file diff --git a/jenkins-scripts/ign_gazebo-default-devel-windows-amd64.bat b/jenkins-scripts/ign_gazebo-default-devel-windows-amd64.bat index 6bf2ee19a..da356e6d8 100644 --- a/jenkins-scripts/ign_gazebo-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_gazebo-default-devel-windows-amd64.bat @@ -11,7 +11,7 @@ if %IGN_MAJOR_VERSION% GEQ 7 ( set DEPEN_PKGS=%DEPEN_PKGS% gdal ) :: This needs to be migrated to DSL to get multi-major versions correctly -set COLCON_PACKAGE=ignition-gazebo +set COLCON_PACKAGE=gz-sim set COLCON_AUTO_MAJOR_VERSION=true call "%SCRIPT_DIR%\lib\colcon-default-devel-windows.bat" diff --git a/jenkins-scripts/ign_gui-default-devel-windows-amd64.bat b/jenkins-scripts/ign_gui-default-devel-windows-amd64.bat index c724e5254..2fc6c885b 100644 --- a/jenkins-scripts/ign_gui-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_gui-default-devel-windows-amd64.bat @@ -12,7 +12,7 @@ if %IGN_MAJOR_VERSION% GEQ 7 ( set DEPEN_PKGS=%DEPEN_PKGS% gdal ) :: This needs to be migrated to DSL to get multi-major versions correctly -set COLCON_PACKAGE=ignition-gui +set COLCON_PACKAGE=gz-gui set COLCON_AUTO_MAJOR_VERSION=true call "%SCRIPT_DIR%\lib\colcon-default-devel-windows.bat" diff --git a/jenkins-scripts/ign_launch-default-devel-windows-amd64.bat b/jenkins-scripts/ign_launch-default-devel-windows-amd64.bat index 4e054415b..79e45168a 100644 --- a/jenkins-scripts/ign_launch-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_launch-default-devel-windows-amd64.bat @@ -9,7 +9,7 @@ for /f %%i in ('python "%SCRIPT_DIR%\tools\detect_cmake_major_version.py" "%WORK if %IGN_MAJOR_VERSION% GEQ 6 ( set DEPEN_PKGS=%DEPEN_PKGS% gdal ) -set COLCON_PACKAGE=ignition-launch +set COLCON_PACKAGE=gz-launch set COLCON_AUTO_MAJOR_VERSION=true call "%SCRIPT_DIR%\lib\colcon-default-devel-windows.bat" diff --git a/jenkins-scripts/ign_math-default-devel-windows-amd64.bat b/jenkins-scripts/ign_math-default-devel-windows-amd64.bat index e5c6acf92..e841c138d 100644 --- a/jenkins-scripts/ign_math-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_math-default-devel-windows-amd64.bat @@ -3,7 +3,7 @@ set SCRIPT_DIR=%~dp0 set VCS_DIRECTORY=ign-math set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true -set COLCON_PACKAGE=ignition-math +set COLCON_PACKAGE=gz-math set COLCON_AUTO_MAJOR_VERSION=true call "%SCRIPT_DIR%\lib\colcon-default-devel-windows.bat" diff --git a/jenkins-scripts/ign_msgs-default-devel-windows-amd64.bat b/jenkins-scripts/ign_msgs-default-devel-windows-amd64.bat index 702dcb618..3c6677854 100644 --- a/jenkins-scripts/ign_msgs-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_msgs-default-devel-windows-amd64.bat @@ -5,7 +5,7 @@ set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true set DEPEN_PKGS=protobuf tinyxml2 -set COLCON_PACKAGE=ignition-msgs +set COLCON_PACKAGE=gz-msgs set COLCON_AUTO_MAJOR_VERSION=true call "%SCRIPT_DIR%\lib\colcon-default-devel-windows.bat" diff --git a/jenkins-scripts/ign_physics-default-devel-windows-amd64.bat b/jenkins-scripts/ign_physics-default-devel-windows-amd64.bat index 7eecb22bd..48b3417f2 100644 --- a/jenkins-scripts/ign_physics-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_physics-default-devel-windows-amd64.bat @@ -10,7 +10,7 @@ if %IGN_MAJOR_VERSION% GEQ 6 ( set DEPEN_PKGS=%DEPEN_PKGS% gdal ) :: This needs to be migrated to DSL to get multi-major versions correctly -set COLCON_PACKAGE=ignition-physics +set COLCON_PACKAGE=gz-physics set COLCON_AUTO_MAJOR_VERSION=true call "%SCRIPT_DIR%/lib/colcon-default-devel-windows.bat" diff --git a/jenkins-scripts/ign_plugin-default-devel-windows-amd64.bat b/jenkins-scripts/ign_plugin-default-devel-windows-amd64.bat index 0ccc66b6a..988b63e30 100644 --- a/jenkins-scripts/ign_plugin-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_plugin-default-devel-windows-amd64.bat @@ -6,7 +6,7 @@ set IGN_CLEAN_WORKSPACE=true set DEPEN_PKGS="dlfcn-win32" -set COLCON_PACKAGE=ignition-plugin +set COLCON_PACKAGE=gz-plugin set COLCON_AUTO_MAJOR_VERSION=true call "%SCRIPT_DIR%/lib/colcon-default-devel-windows.bat" diff --git a/jenkins-scripts/ign_rendering-default-devel-windows-amd64.bat b/jenkins-scripts/ign_rendering-default-devel-windows-amd64.bat index a76a2837a..e13d7b96f 100644 --- a/jenkins-scripts/ign_rendering-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_rendering-default-devel-windows-amd64.bat @@ -15,7 +15,7 @@ if %IGN_MAJOR_VERSION% GEQ 7 ( ) :: This needs to be migrated to DSL to get multi-major versions correctly -set COLCON_PACKAGE=ignition-rendering +set COLCON_PACKAGE=gz-rendering set COLCON_AUTO_MAJOR_VERSION=true call "%SCRIPT_DIR%\lib\colcon-default-devel-windows.bat" diff --git a/jenkins-scripts/ign_sensors-default-devel-windows-amd64.bat b/jenkins-scripts/ign_sensors-default-devel-windows-amd64.bat index 7bb6fe561..38e9a5b1d 100644 --- a/jenkins-scripts/ign_sensors-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_sensors-default-devel-windows-amd64.bat @@ -11,7 +11,7 @@ if %IGN_MAJOR_VERSION% GEQ 7 ( set DEPEN_PKGS=%DEPEN_PKGS% gdal ) :: This needs to be migrated to DSL to get multi-major versions correctly -set COLCON_PACKAGE=ignition-sensors +set COLCON_PACKAGE=gz-sensors set COLCON_AUTO_MAJOR_VERSION=true call "%SCRIPT_DIR%\lib\colcon-default-devel-windows.bat" diff --git a/jenkins-scripts/ign_tools-default-devel-windows-amd64.bat b/jenkins-scripts/ign_tools-default-devel-windows-amd64.bat index afcb992a3..1fc03b98d 100644 --- a/jenkins-scripts/ign_tools-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_tools-default-devel-windows-amd64.bat @@ -4,7 +4,7 @@ set VCS_DIRECTORY=ign-tools set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true -set COLCON_PACKAGE=ignition-tools +set COLCON_PACKAGE=gz-tools :: override logic @ colcon-default-devel-windows.bat to handle ign-tools1 case on windows setlocal ENABLEDELAYEDEXPANSION diff --git a/jenkins-scripts/ign_transport-default-devel-windows-amd64.bat b/jenkins-scripts/ign_transport-default-devel-windows-amd64.bat index 20395af4f..abe3d7717 100644 --- a/jenkins-scripts/ign_transport-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_transport-default-devel-windows-amd64.bat @@ -7,7 +7,7 @@ set PLATFORM_TO_BUILD=amd64 set IGN_CLEAN_WORKSPACE=true set DEPEN_PKGS=cppzmq dlfcn-win32 protobuf sqlite3 tinyxml2 zeromq -set COLCON_PACKAGE=ignition-transport +set COLCON_PACKAGE=gz-transport set COLCON_AUTO_MAJOR_VERSION=true call "%SCRIPT_DIR%\lib\colcon-default-devel-windows.bat" diff --git a/jenkins-scripts/ign_utils-default-devel-windows-amd64.bat b/jenkins-scripts/ign_utils-default-devel-windows-amd64.bat index c22638f54..503e18525 100644 --- a/jenkins-scripts/ign_utils-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_utils-default-devel-windows-amd64.bat @@ -9,7 +9,7 @@ set IGN_CLEAN_WORKSPACE=true :: set DEPEN_PKGS= :: This needs to be migrated to DSL to get multi-major versions correctly -set COLCON_PACKAGE=ignition-utils +set COLCON_PACKAGE=gz-utils set COLCON_AUTO_MAJOR_VERSION=true call "%SCRIPT_DIR%\lib\colcon-default-devel-windows.bat" From ea9b1198f5b9749e8808dd63de774f62a2c60d00 Mon Sep 17 00:00:00 2001 From: Louise Poubel Date: Mon, 27 Jun 2022 08:25:02 -0700 Subject: [PATCH 165/540] Restore ignition COLCON_PACKAGE var (#756) Signed-off-by: Louise Poubel --- jenkins-scripts/ign_common-default-devel-windows-amd64.bat | 2 +- jenkins-scripts/ign_fuel-tools-default-devel-windows-amd64.bat | 2 +- jenkins-scripts/ign_gazebo-default-devel-windows-amd64.bat | 2 +- jenkins-scripts/ign_gui-default-devel-windows-amd64.bat | 2 +- jenkins-scripts/ign_launch-default-devel-windows-amd64.bat | 2 +- jenkins-scripts/ign_math-default-devel-windows-amd64.bat | 2 +- jenkins-scripts/ign_msgs-default-devel-windows-amd64.bat | 2 +- jenkins-scripts/ign_physics-default-devel-windows-amd64.bat | 2 +- jenkins-scripts/ign_plugin-default-devel-windows-amd64.bat | 2 +- jenkins-scripts/ign_rendering-default-devel-windows-amd64.bat | 2 +- jenkins-scripts/ign_sensors-default-devel-windows-amd64.bat | 2 +- jenkins-scripts/ign_tools-default-devel-windows-amd64.bat | 2 +- jenkins-scripts/ign_transport-default-devel-windows-amd64.bat | 2 +- jenkins-scripts/ign_utils-default-devel-windows-amd64.bat | 2 +- 14 files changed, 14 insertions(+), 14 deletions(-) diff --git a/jenkins-scripts/ign_common-default-devel-windows-amd64.bat b/jenkins-scripts/ign_common-default-devel-windows-amd64.bat index 27cf5c9d4..8f42e7dd5 100644 --- a/jenkins-scripts/ign_common-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_common-default-devel-windows-amd64.bat @@ -10,7 +10,7 @@ if %IGN_MAJOR_VERSION% GEQ 5 ( set DEPEN_PKGS=%DEPEN_PKGS% gdal ) -set COLCON_PACKAGE=gz-common +set COLCON_PACKAGE=ignition-common set COLCON_AUTO_MAJOR_VERSION=true call "%SCRIPT_DIR%\lib\colcon-default-devel-windows.bat" diff --git a/jenkins-scripts/ign_fuel-tools-default-devel-windows-amd64.bat b/jenkins-scripts/ign_fuel-tools-default-devel-windows-amd64.bat index 665379493..3753288db 100644 --- a/jenkins-scripts/ign_fuel-tools-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_fuel-tools-default-devel-windows-amd64.bat @@ -7,7 +7,7 @@ set IGN_CLEAN_WORKSPACE=true :: tinyxml2 from msgs set DEPEN_PKGS=libyaml libzip tinyxml2 openssl protobuf curl -set COLCON_PACKAGE=gz-fuel_tools +set COLCON_PACKAGE=ignition-fuel_tools set COLCON_AUTO_MAJOR_VERSION=true call "%SCRIPT_DIR%\lib\colcon-default-devel-windows.bat" \ No newline at end of file diff --git a/jenkins-scripts/ign_gazebo-default-devel-windows-amd64.bat b/jenkins-scripts/ign_gazebo-default-devel-windows-amd64.bat index da356e6d8..6bf2ee19a 100644 --- a/jenkins-scripts/ign_gazebo-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_gazebo-default-devel-windows-amd64.bat @@ -11,7 +11,7 @@ if %IGN_MAJOR_VERSION% GEQ 7 ( set DEPEN_PKGS=%DEPEN_PKGS% gdal ) :: This needs to be migrated to DSL to get multi-major versions correctly -set COLCON_PACKAGE=gz-sim +set COLCON_PACKAGE=ignition-gazebo set COLCON_AUTO_MAJOR_VERSION=true call "%SCRIPT_DIR%\lib\colcon-default-devel-windows.bat" diff --git a/jenkins-scripts/ign_gui-default-devel-windows-amd64.bat b/jenkins-scripts/ign_gui-default-devel-windows-amd64.bat index 2fc6c885b..c724e5254 100644 --- a/jenkins-scripts/ign_gui-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_gui-default-devel-windows-amd64.bat @@ -12,7 +12,7 @@ if %IGN_MAJOR_VERSION% GEQ 7 ( set DEPEN_PKGS=%DEPEN_PKGS% gdal ) :: This needs to be migrated to DSL to get multi-major versions correctly -set COLCON_PACKAGE=gz-gui +set COLCON_PACKAGE=ignition-gui set COLCON_AUTO_MAJOR_VERSION=true call "%SCRIPT_DIR%\lib\colcon-default-devel-windows.bat" diff --git a/jenkins-scripts/ign_launch-default-devel-windows-amd64.bat b/jenkins-scripts/ign_launch-default-devel-windows-amd64.bat index 79e45168a..4e054415b 100644 --- a/jenkins-scripts/ign_launch-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_launch-default-devel-windows-amd64.bat @@ -9,7 +9,7 @@ for /f %%i in ('python "%SCRIPT_DIR%\tools\detect_cmake_major_version.py" "%WORK if %IGN_MAJOR_VERSION% GEQ 6 ( set DEPEN_PKGS=%DEPEN_PKGS% gdal ) -set COLCON_PACKAGE=gz-launch +set COLCON_PACKAGE=ignition-launch set COLCON_AUTO_MAJOR_VERSION=true call "%SCRIPT_DIR%\lib\colcon-default-devel-windows.bat" diff --git a/jenkins-scripts/ign_math-default-devel-windows-amd64.bat b/jenkins-scripts/ign_math-default-devel-windows-amd64.bat index e841c138d..e5c6acf92 100644 --- a/jenkins-scripts/ign_math-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_math-default-devel-windows-amd64.bat @@ -3,7 +3,7 @@ set SCRIPT_DIR=%~dp0 set VCS_DIRECTORY=ign-math set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true -set COLCON_PACKAGE=gz-math +set COLCON_PACKAGE=ignition-math set COLCON_AUTO_MAJOR_VERSION=true call "%SCRIPT_DIR%\lib\colcon-default-devel-windows.bat" diff --git a/jenkins-scripts/ign_msgs-default-devel-windows-amd64.bat b/jenkins-scripts/ign_msgs-default-devel-windows-amd64.bat index 3c6677854..702dcb618 100644 --- a/jenkins-scripts/ign_msgs-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_msgs-default-devel-windows-amd64.bat @@ -5,7 +5,7 @@ set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true set DEPEN_PKGS=protobuf tinyxml2 -set COLCON_PACKAGE=gz-msgs +set COLCON_PACKAGE=ignition-msgs set COLCON_AUTO_MAJOR_VERSION=true call "%SCRIPT_DIR%\lib\colcon-default-devel-windows.bat" diff --git a/jenkins-scripts/ign_physics-default-devel-windows-amd64.bat b/jenkins-scripts/ign_physics-default-devel-windows-amd64.bat index 48b3417f2..7eecb22bd 100644 --- a/jenkins-scripts/ign_physics-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_physics-default-devel-windows-amd64.bat @@ -10,7 +10,7 @@ if %IGN_MAJOR_VERSION% GEQ 6 ( set DEPEN_PKGS=%DEPEN_PKGS% gdal ) :: This needs to be migrated to DSL to get multi-major versions correctly -set COLCON_PACKAGE=gz-physics +set COLCON_PACKAGE=ignition-physics set COLCON_AUTO_MAJOR_VERSION=true call "%SCRIPT_DIR%/lib/colcon-default-devel-windows.bat" diff --git a/jenkins-scripts/ign_plugin-default-devel-windows-amd64.bat b/jenkins-scripts/ign_plugin-default-devel-windows-amd64.bat index 988b63e30..0ccc66b6a 100644 --- a/jenkins-scripts/ign_plugin-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_plugin-default-devel-windows-amd64.bat @@ -6,7 +6,7 @@ set IGN_CLEAN_WORKSPACE=true set DEPEN_PKGS="dlfcn-win32" -set COLCON_PACKAGE=gz-plugin +set COLCON_PACKAGE=ignition-plugin set COLCON_AUTO_MAJOR_VERSION=true call "%SCRIPT_DIR%/lib/colcon-default-devel-windows.bat" diff --git a/jenkins-scripts/ign_rendering-default-devel-windows-amd64.bat b/jenkins-scripts/ign_rendering-default-devel-windows-amd64.bat index e13d7b96f..a76a2837a 100644 --- a/jenkins-scripts/ign_rendering-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_rendering-default-devel-windows-amd64.bat @@ -15,7 +15,7 @@ if %IGN_MAJOR_VERSION% GEQ 7 ( ) :: This needs to be migrated to DSL to get multi-major versions correctly -set COLCON_PACKAGE=gz-rendering +set COLCON_PACKAGE=ignition-rendering set COLCON_AUTO_MAJOR_VERSION=true call "%SCRIPT_DIR%\lib\colcon-default-devel-windows.bat" diff --git a/jenkins-scripts/ign_sensors-default-devel-windows-amd64.bat b/jenkins-scripts/ign_sensors-default-devel-windows-amd64.bat index 38e9a5b1d..7bb6fe561 100644 --- a/jenkins-scripts/ign_sensors-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_sensors-default-devel-windows-amd64.bat @@ -11,7 +11,7 @@ if %IGN_MAJOR_VERSION% GEQ 7 ( set DEPEN_PKGS=%DEPEN_PKGS% gdal ) :: This needs to be migrated to DSL to get multi-major versions correctly -set COLCON_PACKAGE=gz-sensors +set COLCON_PACKAGE=ignition-sensors set COLCON_AUTO_MAJOR_VERSION=true call "%SCRIPT_DIR%\lib\colcon-default-devel-windows.bat" diff --git a/jenkins-scripts/ign_tools-default-devel-windows-amd64.bat b/jenkins-scripts/ign_tools-default-devel-windows-amd64.bat index 1fc03b98d..afcb992a3 100644 --- a/jenkins-scripts/ign_tools-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_tools-default-devel-windows-amd64.bat @@ -4,7 +4,7 @@ set VCS_DIRECTORY=ign-tools set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true -set COLCON_PACKAGE=gz-tools +set COLCON_PACKAGE=ignition-tools :: override logic @ colcon-default-devel-windows.bat to handle ign-tools1 case on windows setlocal ENABLEDELAYEDEXPANSION diff --git a/jenkins-scripts/ign_transport-default-devel-windows-amd64.bat b/jenkins-scripts/ign_transport-default-devel-windows-amd64.bat index abe3d7717..20395af4f 100644 --- a/jenkins-scripts/ign_transport-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_transport-default-devel-windows-amd64.bat @@ -7,7 +7,7 @@ set PLATFORM_TO_BUILD=amd64 set IGN_CLEAN_WORKSPACE=true set DEPEN_PKGS=cppzmq dlfcn-win32 protobuf sqlite3 tinyxml2 zeromq -set COLCON_PACKAGE=gz-transport +set COLCON_PACKAGE=ignition-transport set COLCON_AUTO_MAJOR_VERSION=true call "%SCRIPT_DIR%\lib\colcon-default-devel-windows.bat" diff --git a/jenkins-scripts/ign_utils-default-devel-windows-amd64.bat b/jenkins-scripts/ign_utils-default-devel-windows-amd64.bat index 503e18525..c22638f54 100644 --- a/jenkins-scripts/ign_utils-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_utils-default-devel-windows-amd64.bat @@ -9,7 +9,7 @@ set IGN_CLEAN_WORKSPACE=true :: set DEPEN_PKGS= :: This needs to be migrated to DSL to get multi-major versions correctly -set COLCON_PACKAGE=gz-utils +set COLCON_PACKAGE=ignition-utils set COLCON_AUTO_MAJOR_VERSION=true call "%SCRIPT_DIR%\lib\colcon-default-devel-windows.bat" From 475a77e185289a8cad7bd902cb74a690ef314b71 Mon Sep 17 00:00:00 2001 From: Louise Poubel Date: Wed, 29 Jun 2022 16:48:54 -0700 Subject: [PATCH 166/540] Windows colcon: handle ignition-gazebo -> gz-sim case (#758) Signed-off-by: Louise Poubel --- jenkins-scripts/lib/colcon-default-devel-windows.bat | 1 + jenkins-scripts/lib/windows_library.bat | 1 + 2 files changed, 2 insertions(+) diff --git a/jenkins-scripts/lib/colcon-default-devel-windows.bat b/jenkins-scripts/lib/colcon-default-devel-windows.bat index 5fe37b4c8..fd0cc22ec 100644 --- a/jenkins-scripts/lib/colcon-default-devel-windows.bat +++ b/jenkins-scripts/lib/colcon-default-devel-windows.bat @@ -43,6 +43,7 @@ colcon list --names-only colcon list --names-only | find "%COLCON_PACKAGE%" if errorlevel 1 ( set COLCON_PACKAGE=%COLCON_PACKAGE:ignition=gz% + set COLCON_PACKAGE=%COLCON_PACKAGE:gazebo=sim% ) colcon list --names-only | find "%COLCON_PACKAGE%" if errorlevel 1 ( diff --git a/jenkins-scripts/lib/windows_library.bat b/jenkins-scripts/lib/windows_library.bat index f42189a8d..31f3c10e2 100644 --- a/jenkins-scripts/lib/windows_library.bat +++ b/jenkins-scripts/lib/windows_library.bat @@ -218,6 +218,7 @@ colcon list --names-only colcon list --names-only | find "%COLCON_PACKAGE%" if errorlevel 1 ( set COLCON_PACKAGE=%COLCON_PACKAGE:ignition=gz% + set COLCON_PACKAGE=%COLCON_PACKAGE:gazebo=sim% ) colcon list --names-only | find "%COLCON_PACKAGE%" if errorlevel 1 ( From 3b78ac84dc96f93d0fee62452030dd355eac3586 Mon Sep 17 00:00:00 2001 From: methylDragon Date: Wed, 29 Jun 2022 17:56:41 -0700 Subject: [PATCH 167/540] Use appropriate substitution invocation for COLCON_PACKAGE (#760) --- .../ign_tools-default-devel-windows-amd64.bat | 18 +++++++++++++++ .../lib/colcon-default-devel-windows.bat | 22 +++++++++---------- jenkins-scripts/lib/windows_library.bat | 22 +++++++++---------- 3 files changed, 40 insertions(+), 22 deletions(-) diff --git a/jenkins-scripts/ign_tools-default-devel-windows-amd64.bat b/jenkins-scripts/ign_tools-default-devel-windows-amd64.bat index afcb992a3..236a050f7 100644 --- a/jenkins-scripts/ign_tools-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_tools-default-devel-windows-amd64.bat @@ -16,6 +16,24 @@ if "!PKG_MAJOR_VERSION!" == "1" ( set COLCON_PACKAGE=%COLCON_PACKAGE%!PKG_MAJOR_VERSION! ) +echo # BEGIN SECTION: Update package !COLCON_PACKAGE! from ignition to gz if needed +echo Packages in workspace: +colcon list --names-only + +colcon list --names-only | find "!COLCON_PACKAGE!" +if errorlevel 1 ( + set COLCON_PACKAGE=%COLCON_PACKAGE:ignition=gz% + set COLCON_PACKAGE=!COLCON_PACKAGE:gazebo=sim! +) +colcon list --names-only | find "!COLCON_PACKAGE!" +if errorlevel 1 ( + echo Failed to find package !COLCON_PACKAGE! in workspace. + goto :error +) +echo Using package name !COLCON_PACKAGE! +echo # END SECTION + + set COLCON_AUTO_MAJOR_VERSION=false call "%SCRIPT_DIR%\lib\colcon-default-devel-windows.bat" diff --git a/jenkins-scripts/lib/colcon-default-devel-windows.bat b/jenkins-scripts/lib/colcon-default-devel-windows.bat index fd0cc22ec..7dbdfeb8a 100644 --- a/jenkins-scripts/lib/colcon-default-devel-windows.bat +++ b/jenkins-scripts/lib/colcon-default-devel-windows.bat @@ -36,24 +36,24 @@ if "%COLCON_AUTO_MAJOR_VERSION%" == "true" ( ) :: Check if package is in colcon workspace -echo # BEGIN SECTION: Update package %COLCON_PACKAGE% from ignition to gz if needed +echo # BEGIN SECTION: Update package !COLCON_PACKAGE! from ignition to gz if needed echo Packages in workspace: colcon list --names-only -colcon list --names-only | find "%COLCON_PACKAGE%" +colcon list --names-only | find "!COLCON_PACKAGE!" if errorlevel 1 ( - set COLCON_PACKAGE=%COLCON_PACKAGE:ignition=gz% - set COLCON_PACKAGE=%COLCON_PACKAGE:gazebo=sim% + set COLCON_PACKAGE=!COLCON_PACKAGE:ignition=gz! + set COLCON_PACKAGE=!COLCON_PACKAGE:gazebo=sim! ) -colcon list --names-only | find "%COLCON_PACKAGE%" +colcon list --names-only | find "!COLCON_PACKAGE!" if errorlevel 1 ( - echo Failed to find package %COLCON_PACKAGE% in workspace. + echo Failed to find package !COLCON_PACKAGE! in workspace. goto :error ) -echo Using package name %COLCON_PACKAGE% +echo Using package name !COLCON_PACKAGE! echo # END SECTION -set TEST_RESULT_PATH=%WORKSPACE%\ws\build\%COLCON_PACKAGE%\test_results +set TEST_RESULT_PATH=%WORKSPACE%\ws\build\!COLCON_PACKAGE!\test_results setlocal ENABLEDELAYEDEXPANSION if not defined GAZEBODISTRO_FILE ( @@ -123,12 +123,12 @@ if exist %LOCAL_WS_SOFTWARE_DIR%\configure.bat ( echo # BEGIN SECTION: compiling %VCS_DIRECTORY% cd %LOCAL_WS% -call %win_lib% :build_workspace %COLCON_PACKAGE% || goto :error +call %win_lib% :build_workspace !COLCON_PACKAGE! || goto :error echo # END SECTION if "%ENABLE_TESTS%" == "TRUE" ( - echo # BEGIN SECTION: running tests for %COLCON_PACKAGE% - call %win_lib% :tests_in_workspace %COLCON_PACKAGE% + echo # BEGIN SECTION: running tests for !COLCON_PACKAGE! + call %win_lib% :tests_in_workspace !COLCON_PACKAGE! echo # END SECTION echo # BEGIN SECTION: export testing results diff --git a/jenkins-scripts/lib/windows_library.bat b/jenkins-scripts/lib/windows_library.bat index 31f3c10e2..857ba9314 100644 --- a/jenkins-scripts/lib/windows_library.bat +++ b/jenkins-scripts/lib/windows_library.bat @@ -217,24 +217,24 @@ colcon list --names-only colcon list --names-only | find "%COLCON_PACKAGE%" if errorlevel 1 ( - set COLCON_PACKAGE=%COLCON_PACKAGE:ignition=gz% - set COLCON_PACKAGE=%COLCON_PACKAGE:gazebo=sim% + set COLCON_PACKAGE=!COLCON_PACKAGE:ignition=gz! + set COLCON_PACKAGE=!COLCON_PACKAGE:gazebo=sim! ) -colcon list --names-only | find "%COLCON_PACKAGE%" +colcon list --names-only | find "!COLCON_PACKAGE!" if errorlevel 1 ( - echo Failed to find package %COLCON_PACKAGE% in workspace. + echo Failed to find package !COLCON_PACKAGE! in workspace. goto :error ) -echo Using package name %COLCON_PACKAGE% +echo Using package name !COLCON_PACKAGE! echo # END SECTION :: two runs to get the dependencies built with testing and the package under :: test build with tests -echo # BEGIN SECTION: colcon compilation without test for dependencies of %COLCON_PACKAGE% -call :_colcon_build_cmd --packages-skip %COLCON_PACKAGE% "-DBUILD_TESTING=0" "-DCMAKE_CXX_FLAGS=-w" +echo # BEGIN SECTION: colcon compilation without test for dependencies of !COLCON_PACKAGE! +call :_colcon_build_cmd --packages-skip !COLCON_PACKAGE! "-DBUILD_TESTING=0" "-DCMAKE_CXX_FLAGS=-w" echo # END SECTION -echo # BEGIN SECTION: colcon compilation with tests for %COLCON_PACKAGE% -call :_colcon_build_cmd --packages-select %COLCON_PACKAGE% " -DBUILD_TESTING=1" +echo # BEGIN SECTION: colcon compilation with tests for !COLCON_PACKAGE! +call :_colcon_build_cmd --packages-select !COLCON_PACKAGE! " -DBUILD_TESTING=1" echo # END SECTION goto :EOF @@ -248,9 +248,9 @@ goto :EOF :: arg1: package whitelist to test set COLCON_PACKAGE=%1 -echo # BEGIN SECTION: colcon test for %COLCON_PACKAGE% +echo # BEGIN SECTION: colcon test for !COLCON_PACKAGE! colcon test --install-base "install"^ - --packages-select %COLCON_PACKAGE%^ + --packages-select !COLCON_PACKAGE!^ --executor sequential^ --event-handler console_direct+ echo # END SECTION From 80662db64845aa01eca03def0c553d80e8234c74 Mon Sep 17 00:00:00 2001 From: Louise Poubel Date: Fri, 1 Jul 2022 12:36:21 -0700 Subject: [PATCH 168/540] Support releasing gz packages (#762) * Support releasing gz packages Signed-off-by: Louise Poubel * sim Signed-off-by: Louise Poubel --- release.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/release.py b/release.py index 268b0bc3d..0ccdfd000 100755 --- a/release.py +++ b/release.py @@ -206,13 +206,17 @@ def sanity_package_name(repo_dir, package, package_alias): if package_alias: expected_name = package_alias + # Use igntiion for Citadel and Fortress, gz for Garden and beyond + gz_name = expected_name.replace("ignition", "gz"); + gz_name = gz_name.replace("gazebo", "sim"); + cmd = ["find", repo_dir, "-name", "changelog","-exec","head","-n","1","{}",";"] out, err = check_call(cmd, IGNORE_DRY_RUN) for line in out.decode().split('\n'): if not line: continue # Check that first word is the package alias or name - if line.partition(' ')[0] != expected_name: + if line.partition(' ')[0] != expected_name and line.partition(' ')[0] != gz_name: error("Error in changelog package name or alias: " + line) cmd = ["find", repo_dir, "-name", "control","-exec","grep","-H","Source:","{}",";"] @@ -221,8 +225,8 @@ def sanity_package_name(repo_dir, package, package_alias): if not line: continue # Check that first word is the package alias or name - if line.partition(' ')[2] != expected_name: - error("Error in source package. File: " + line.partition(' ')[1] + ". Got " + line.partition(' ')[2] + " expected " + expected_name) + if line.partition(' ')[2] != expected_name and line.partition(' ')[2] != gz_name: + error("Error in source package. File: " + line.partition(' ')[1] + ". Got " + line.partition(' ')[2] + " expected " + expected_name + " or " + gz_name) print_success("Package names in changelog and control") From c719ea69d036158f1d82424680f218332728cf44 Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Fri, 8 Jul 2022 13:22:53 -0700 Subject: [PATCH 169/540] windows: add boost-core vcpkg dep to ign-gazebo, launch (#763) * Alphabetize DEPEN_PKGS variables Signed-off-by: Steve Peters --- jenkins-scripts/ign_gazebo-default-devel-windows-amd64.bat | 2 +- jenkins-scripts/ign_launch-default-devel-windows-amd64.bat | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/jenkins-scripts/ign_gazebo-default-devel-windows-amd64.bat b/jenkins-scripts/ign_gazebo-default-devel-windows-amd64.bat index 6bf2ee19a..bb7aa86b4 100644 --- a/jenkins-scripts/ign_gazebo-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_gazebo-default-devel-windows-amd64.bat @@ -5,7 +5,7 @@ set VCS_DIRECTORY=ign-gazebo set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true :: dlfcn -set DEPEN_PKGS=dlfcn-win32 cuda cppzmq curl openssl jsoncpp ffmpeg freeimage ogre ogre2 ogre22 qt5 qwt gts glib fcl eigen3 ccd assimp libyaml libzip gflags protobuf tinyxml2 zeromq +set DEPEN_PKGS=assimp boost-core ccd cppzmq cuda curl dlfcn-win32 eigen3 fcl ffmpeg freeimage gflags glib gts jsoncpp libyaml libzip ogre ogre2 ogre22 openssl protobuf qt5 qwt tinyxml2 zeromq for /f %%i in ('python "%SCRIPT_DIR%\tools\detect_cmake_major_version.py" "%WORKSPACE%\%VCS_DIRECTORY%\CMakeLists.txt"') do set IGN_MAJOR_VERSION=%%i if %IGN_MAJOR_VERSION% GEQ 7 ( set DEPEN_PKGS=%DEPEN_PKGS% gdal diff --git a/jenkins-scripts/ign_launch-default-devel-windows-amd64.bat b/jenkins-scripts/ign_launch-default-devel-windows-amd64.bat index 4e054415b..74b5cc273 100644 --- a/jenkins-scripts/ign_launch-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_launch-default-devel-windows-amd64.bat @@ -4,7 +4,7 @@ set VCS_DIRECTORY=ign-launch set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true -set DEPEN_PKGS=dlfcn-win32 cuda cppzmq curl openssl jsoncpp ffmpeg freeimage ogre ogre2 ogre22 qt5 qwt gts glib fcl eigen3 ccd assimp libyaml libzip gflags protobuf tinyxml2 zeromq +set DEPEN_PKGS=assimp boost-core ccd cppzmq cuda curl dlfcn-win32 eigen3 fcl ffmpeg freeimage gflags glib gts jsoncpp libyaml libzip ogre ogre2 ogre22 openssl protobuf qt5 qwt tinyxml2 zeromq for /f %%i in ('python "%SCRIPT_DIR%\tools\detect_cmake_major_version.py" "%WORKSPACE%\%VCS_DIRECTORY%\CMakeLists.txt"') do set IGN_MAJOR_VERSION=%%i if %IGN_MAJOR_VERSION% GEQ 6 ( set DEPEN_PKGS=%DEPEN_PKGS% gdal From c2d8abc77236c568a956cf9e2fd559d263588a08 Mon Sep 17 00:00:00 2001 From: methylDragon Date: Mon, 11 Jul 2022 09:51:10 -0700 Subject: [PATCH 170/540] Migrate ros_gz release script (#764) Signed-off-by: methylDragon --- ...ign_bridge-release.py.bash => ros_gz_bridge-release.py.bash} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename bloom/{ros_ign_bridge-release.py.bash => ros_gz_bridge-release.py.bash} (90%) diff --git a/bloom/ros_ign_bridge-release.py.bash b/bloom/ros_gz_bridge-release.py.bash similarity index 90% rename from bloom/ros_ign_bridge-release.py.bash rename to bloom/ros_gz_bridge-release.py.bash index e2b76e172..0e5534821 100755 --- a/bloom/ros_ign_bridge-release.py.bash +++ b/bloom/ros_gz_bridge-release.py.bash @@ -1,6 +1,6 @@ #!/bin/bash # Launch all the suite of ros_gazebo_pkgs: -# Usage: ros1_ign_bridge-release.py.bash +# Usage: ros_gz_bridge-release.py.bash # if [[ $# -lt 2 ]]; then From c8f710b0004e65da0a737515fef993090816597b Mon Sep 17 00:00:00 2001 From: Jorge Perez Date: Fri, 15 Jul 2022 16:15:02 -0500 Subject: [PATCH 171/540] Print vcs export information on Windows (#766) Signed-off-by: Jorge Perez --- jenkins-scripts/lib/windows_library.bat | 1 + 1 file changed, 1 insertion(+) diff --git a/jenkins-scripts/lib/windows_library.bat b/jenkins-scripts/lib/windows_library.bat index 857ba9314..7f5f54d79 100644 --- a/jenkins-scripts/lib/windows_library.bat +++ b/jenkins-scripts/lib/windows_library.bat @@ -241,6 +241,7 @@ goto :EOF :: ################################## :list_workspace_pkgs colcon list -t || goto :error +vcs export --exact || goto :error goto :EOF :: ################################## From c16a319ed4eb8d0641562589ec6b0b8f2ff90ecd Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Mon, 18 Jul 2022 16:38:20 -0700 Subject: [PATCH 172/540] Release checklist: check for unstable debbuilds (#768) Signed-off-by: Steve Peters --- README.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 1591da239..817f2abbe 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,7 @@ Gazebo software. 1. Pre-release: `/release.py -r 1 ign-physics3 3.0.0~pre1 --upload-to-repo=prerelease secrettoken --dry-run` 1. If the dry run succeeds, run the same command again, now without `--dry-run`. 1. Check that: - * Several `-debbuilder` jobs have been queued in https://build.osrfoundation.org/ and watch those jobs to see if any of them fail. + * Several `-debbuilder` jobs have been queued in https://build.osrfoundation.org/ and watch those jobs to see if any of them fail or are marked unstable. While it would be easier to track the status of these jobs if there was a [dashboard](https://github.com/gazebo-tooling/release-tools/issues/295), you can watch the page for a specific debbuild, such as https://build.osrfoundation.org/job/ign-gui3-debbuilder/. 1. If you observe a failure of an Gazebo Dome package, check the list of supported @@ -86,6 +86,12 @@ Gazebo software. 1. If the failure is on a supported architecture, check the source repository for an existing report of this failure and if none exists, please report the failure (see [gazebosim/gz-math#161](https://github.com/gazebosim/gz-math/issues/161) for an example). + 1. If a build is unstable, check if it was unstable before this release and if it has already been reported. + A common cause of unstable debbuilds is newly installed files that are not captured by the pattenrs in the `.install` + files in the `-release` repository. This can be checked by searching for `dh_missing` in the console log of the + unstable build and looking for a list of uninstalled files. Example pull requests that fix problems with `dh_missing` + are [gazebo-release/gz-transport11-release#4](https://github.com/gazebo-release/gz-transport11-release/pull/4) + and [gazebo-release/gz-tools-release#4](https://github.com/gazebo-release/gz-tools-release/pull/4). * A pull request was opened to https://github.com/osrf/homebrew-simulation 1. This pull request may take a minute or two to open. 1. Once it is open, make a comment containing the text "build bottle". From 7cc4571c5a9c0ab30491c9e217fdd3ff91728bf2 Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Thu, 21 Jul 2022 10:22:35 -0700 Subject: [PATCH 173/540] extra.dsl: debbuild priority 100 (#772) Match the priority used in OSRFLinuxBuildPkg Signed-off-by: Steve Peters --- jenkins-scripts/dsl/extra.dsl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/jenkins-scripts/dsl/extra.dsl b/jenkins-scripts/dsl/extra.dsl index 1b8dc3f23..0b95f6a33 100644 --- a/jenkins-scripts/dsl/extra.dsl +++ b/jenkins-scripts/dsl/extra.dsl @@ -59,6 +59,10 @@ gbp_repo_debbuilds.each { software -> 'OSRF repo name to upload the package to') } + properties { + priority 100 + } + scm { git { remote { From 103968d1689825a1635422804974546af4759fee Mon Sep 17 00:00:00 2001 From: Louise Poubel Date: Mon, 25 Jul 2022 09:54:58 -0700 Subject: [PATCH 174/540] Forward-port PR: handle .git URLs (#770) Signed-off-by: Louise Poubel --- source-repo-scripts/merge_forward_pull_request.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source-repo-scripts/merge_forward_pull_request.bash b/source-repo-scripts/merge_forward_pull_request.bash index befbadf28..95cf912d0 100755 --- a/source-repo-scripts/merge_forward_pull_request.bash +++ b/source-repo-scripts/merge_forward_pull_request.bash @@ -38,7 +38,7 @@ set -e CURRENT_BRANCH=$(git branch | sed -n -e 's/^\* \(.*\)/\1/p') ORIGIN_URL=$(git remote get-url origin) -ORIGIN_ORG_REPO=$(echo ${ORIGIN_URL} | sed -e 's@.*github\.com.@@') +ORIGIN_ORG_REPO=$(echo ${ORIGIN_URL} | sed -e 's@.*github\.com.@@' | sed -e 's/\.git//g') TITLE="Merge ${FROM_BRANCH} ➡️ ${TO_BRANCH}" From f7959b4fdf58188129112d67e956f57dc52f0ef3 Mon Sep 17 00:00:00 2001 From: Louise Poubel Date: Mon, 25 Jul 2022 13:17:33 -0700 Subject: [PATCH 175/540] Move pre-release tasks (#769) Signed-off-by: Louise Poubel --- .github/ISSUE_TEMPLATE/release_collection.md | 38 +++++++++++--------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/release_collection.md b/.github/ISSUE_TEMPLATE/release_collection.md index 403ed392d..7465d0152 100644 --- a/.github/ISSUE_TEMPLATE/release_collection.md +++ b/.github/ISSUE_TEMPLATE/release_collection.md @@ -23,10 +23,28 @@ When opening PRs, add a link back to this issue for easier tracking. - [ ] Choose name for next release. - [ ] Merge each library forward from previous versions * -- [ ] Label pull requests being considered to enter release with [beta](https://github.com/search?q=org%3Agazebosim+label%3Abeta&state=open) - Pull requests opened against the new collection after then: - - Bug fixes: will be considered for the initial release and labeled `beta` - - New features: will not be considered for the initial release. +- [ ] Create stable branches off of `main` +- [ ] Pull requests previously targeted at `main`: + - Bug fixes: + - Retarget to the new stable branch. + - Label with [beta](https://github.com/search?q=org%3Agazebosim+label%3Abeta&state=open) + - They can be merged until code freeze. + - Backwards-compatible features, either: + - Retarget them to the new stable branch (but they can only be merged after the stable release), or + - Keep targeting `main` and backport after the stable release. + - Breaking features: + - Keep targeting `main` (i.e. they will be delayed until the next major release) +- [ ] Bump all `main` branches to the next major version (`X.0.0~pre1`) + - [ ] Source code ([example](https://github.com/gazebosim/gz-common/pull/193)) + * + - [ ] Add files to `gazebodistro` ([example](https://github.com/gazebo-tooling/gazebodistro/pull/12)) + * + - [ ] Add aliases to `homebrew-simulation` ([example](https://github.com/osrf/homebrew-simulation/commit/1f8602af6f52e06e0542eebfbdbe97f5f6cf950c)) + * + - [ ] Create new `-release` repositories (use [this script](https://github.com/gazebo-tooling/release-tools/blob/master/release-repo-scripts/new_ignition_release_repos.bash)) + - [ ] Enable nightlies for all `main` branches on `gzdev` ([example](https://github.com/gazebo-tooling/gzdev/pull/50)) + - [ ] Execute the tick-tock's "tock" for deprecations ([example](https://github.com/gazebosim/gz-sim/pull/875)) + * - [ ] Pre-release libraries as all `beta` labels are merged. * - [ ] Make collection pre-release after all libraries are pre-released. @@ -44,7 +62,6 @@ When opening PRs, add a link back to this issue for easier tracking. * - [ ] PRs fixing documentation and critical bugs can be merged and pre-released - As libraries have all their documentation reviewed: - - [ ] Create stable branches off of main - [ ] Make stable releases * @@ -66,17 +83,6 @@ When opening PRs, add a link back to this issue for easier tracking. * - [ ] Remove pre-release for release branches on `gzdev` ([example](https://github.com/gazebo-tooling/gzdev/pull/36)) * -- [ ] Bump all `main` branches to the next major versioni (`X.0.0~pre1`) - - [ ] Source code ([example](https://github.com/gazebosim/gz-common/pull/193)) - * - - [ ] Add files to `gazebodistro` ([example](https://github.com/gazebo-tooling/gazebodistro/pull/12)) - * - - [ ] Add aliases to `homebrew-simulation` ([example](https://github.com/osrf/homebrew-simulation/commit/1f8602af6f52e06e0542eebfbdbe97f5f6cf950c)) - * - - [ ] Create new `-release` repositories (use [this script](https://github.com/gazebo-tooling/release-tools/blob/master/release-repo-scripts/new_ignition_release_repos.bash)) - - [ ] Enable nightlies for all `main` branches on `gzdev` ([example](https://github.com/gazebo-tooling/gzdev/pull/50)) - - [ ] Execute the tick-tock's "tock" for deprecations ([example](https://github.com/gazebosim/gz-sim/pull/875)) - * - [ ] Update all repositories to default to the new stable branches. If the collection will be officially paired with a ROS 2 distro: From 19b5209e2b3c97931d6c521636d6ae1992e99ad7 Mon Sep 17 00:00:00 2001 From: Louise Poubel Date: Thu, 28 Jul 2022 08:23:03 -0700 Subject: [PATCH 176/540] Install assimp for gz-common on Windows (#774) Signed-off-by: Louise Poubel --- jenkins-scripts/ign_common-default-devel-windows-amd64.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/ign_common-default-devel-windows-amd64.bat b/jenkins-scripts/ign_common-default-devel-windows-amd64.bat index 8f42e7dd5..5a76d1300 100644 --- a/jenkins-scripts/ign_common-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_common-default-devel-windows-amd64.bat @@ -4,7 +4,7 @@ set VCS_DIRECTORY=ign-common set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true -set DEPEN_PKGS=ffmpeg freeimage gts tinyxml2 +set DEPEN_PKGS=assimp ffmpeg freeimage gts tinyxml2 for /f %%i in ('python "%SCRIPT_DIR%\tools\detect_cmake_major_version.py" "%WORKSPACE%\%VCS_DIRECTORY%\CMakeLists.txt"') do set IGN_MAJOR_VERSION=%%i if %IGN_MAJOR_VERSION% GEQ 5 ( set DEPEN_PKGS=%DEPEN_PKGS% gdal From 167a533c9214ca38fa25e1348ce2f7323488ac3e Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Thu, 28 Jul 2022 16:06:28 -0700 Subject: [PATCH 177/540] merge forward script: ensure different args (#775) Signed-off-by: Steve Peters --- source-repo-scripts/merge_forward_pull_request.bash | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source-repo-scripts/merge_forward_pull_request.bash b/source-repo-scripts/merge_forward_pull_request.bash index 95cf912d0..a785a4d04 100755 --- a/source-repo-scripts/merge_forward_pull_request.bash +++ b/source-repo-scripts/merge_forward_pull_request.bash @@ -31,6 +31,9 @@ TO_BRANCH=${2} if [[ $# -ne 2 ]]; then echo "./merge_forward_pull_request.bash " exit 1 +elif [[ "$FROM_BRANCH" == "$TO_BRANCH" ]]; then + echo "Arguments ${FROM_BRANCH} and ${TO_BRANCH} must not be identical" + exit 1 fi set -e From e63a542d759e06a5f689e962f6e6fe2840909ca8 Mon Sep 17 00:00:00 2001 From: Louise Poubel Date: Fri, 29 Jul 2022 19:35:49 -0700 Subject: [PATCH 178/540] Remove SKIP_usd (#773) Signed-off-by: Louise Poubel --- jenkins-scripts/docker/sdformat-compilation.bash | 4 ---- 1 file changed, 4 deletions(-) diff --git a/jenkins-scripts/docker/sdformat-compilation.bash b/jenkins-scripts/docker/sdformat-compilation.bash index 21d03de1f..8cfb50a1f 100644 --- a/jenkins-scripts/docker/sdformat-compilation.bash +++ b/jenkins-scripts/docker/sdformat-compilation.bash @@ -24,10 +24,6 @@ fi export GZDEV_PROJECT_NAME="sdformat${SDFORMAT_MAJOR_VERSION}" -if [[ ${SDFORMAT_MAJOR_VERSION} -ge 12 ]]; then - export BUILDING_EXTRA_CMAKE_PARAMS+=" -DSKIP_usd=true" -fi - # master and major branches compilations export BUILDING_PKG_DEPENDENCIES_VAR_NAME="SDFORMAT_BASE_DEPENDENCIES" From 43243dbbf543a4fbdbf75362760c0e18d50ed9dc Mon Sep 17 00:00:00 2001 From: Louise Poubel Date: Wed, 3 Aug 2022 10:59:05 -0700 Subject: [PATCH 179/540] =?UTF-8?q?[ign=20=E2=9E=A1=EF=B8=8F=20=20gz]=20Up?= =?UTF-8?q?date=20VCS=20directory=20for=20Windows=20CI=20(#738)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Louise Poubel Co-authored-by: Jose Luis Rivero --- jenkins-scripts/ign_cmake-default-devel-windows-amd64.bat | 2 +- jenkins-scripts/ign_common-default-devel-windows-amd64.bat | 2 +- jenkins-scripts/ign_fuel-tools-default-devel-windows-amd64.bat | 2 +- jenkins-scripts/ign_gazebo-default-devel-windows-amd64.bat | 2 +- jenkins-scripts/ign_gui-default-devel-windows-amd64.bat | 2 +- jenkins-scripts/ign_launch-default-devel-windows-amd64.bat | 2 +- jenkins-scripts/ign_math-default-devel-windows-amd64.bat | 2 +- jenkins-scripts/ign_msgs-default-devel-windows-amd64.bat | 2 +- jenkins-scripts/ign_physics-default-devel-windows-amd64.bat | 2 +- jenkins-scripts/ign_plugin-default-devel-windows-amd64.bat | 2 +- jenkins-scripts/ign_rendering-default-devel-windows-amd64.bat | 2 +- jenkins-scripts/ign_sensors-default-devel-windows-amd64.bat | 2 +- jenkins-scripts/ign_tools-default-devel-windows-amd64.bat | 2 +- jenkins-scripts/ign_transport-default-devel-windows-amd64.bat | 2 +- jenkins-scripts/ign_utils-default-devel-windows-amd64.bat | 2 +- 15 files changed, 15 insertions(+), 15 deletions(-) diff --git a/jenkins-scripts/ign_cmake-default-devel-windows-amd64.bat b/jenkins-scripts/ign_cmake-default-devel-windows-amd64.bat index d833a53bf..b8ca4929c 100644 --- a/jenkins-scripts/ign_cmake-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_cmake-default-devel-windows-amd64.bat @@ -1,6 +1,6 @@ set SCRIPT_DIR=%~dp0 -set VCS_DIRECTORY=ign-cmake +set VCS_DIRECTORY=gz-cmake set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true diff --git a/jenkins-scripts/ign_common-default-devel-windows-amd64.bat b/jenkins-scripts/ign_common-default-devel-windows-amd64.bat index 5a76d1300..c6f6b31aa 100644 --- a/jenkins-scripts/ign_common-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_common-default-devel-windows-amd64.bat @@ -1,6 +1,6 @@ set SCRIPT_DIR=%~dp0 -set VCS_DIRECTORY=ign-common +set VCS_DIRECTORY=gz-common set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true diff --git a/jenkins-scripts/ign_fuel-tools-default-devel-windows-amd64.bat b/jenkins-scripts/ign_fuel-tools-default-devel-windows-amd64.bat index 3753288db..0021cc5b7 100644 --- a/jenkins-scripts/ign_fuel-tools-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_fuel-tools-default-devel-windows-amd64.bat @@ -1,6 +1,6 @@ set SCRIPT_DIR=%~dp0 -set VCS_DIRECTORY=ign-fuel-tools +set VCS_DIRECTORY=gz-fuel-tools set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true diff --git a/jenkins-scripts/ign_gazebo-default-devel-windows-amd64.bat b/jenkins-scripts/ign_gazebo-default-devel-windows-amd64.bat index bb7aa86b4..210551130 100644 --- a/jenkins-scripts/ign_gazebo-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_gazebo-default-devel-windows-amd64.bat @@ -1,7 +1,7 @@ @echo on set SCRIPT_DIR=%~dp0 -set VCS_DIRECTORY=ign-gazebo +set VCS_DIRECTORY=gz-sim set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true :: dlfcn diff --git a/jenkins-scripts/ign_gui-default-devel-windows-amd64.bat b/jenkins-scripts/ign_gui-default-devel-windows-amd64.bat index c724e5254..3ab72a3bb 100644 --- a/jenkins-scripts/ign_gui-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_gui-default-devel-windows-amd64.bat @@ -1,7 +1,7 @@ @echo on set SCRIPT_DIR=%~dp0 -set VCS_DIRECTORY=ign-gui +set VCS_DIRECTORY=gz-gui set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true diff --git a/jenkins-scripts/ign_launch-default-devel-windows-amd64.bat b/jenkins-scripts/ign_launch-default-devel-windows-amd64.bat index 74b5cc273..823e281fb 100644 --- a/jenkins-scripts/ign_launch-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_launch-default-devel-windows-amd64.bat @@ -1,6 +1,6 @@ set SCRIPT_DIR=%~dp0 -set VCS_DIRECTORY=ign-launch +set VCS_DIRECTORY=gz-launch set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true diff --git a/jenkins-scripts/ign_math-default-devel-windows-amd64.bat b/jenkins-scripts/ign_math-default-devel-windows-amd64.bat index e5c6acf92..91b5315c7 100644 --- a/jenkins-scripts/ign_math-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_math-default-devel-windows-amd64.bat @@ -1,6 +1,6 @@ set SCRIPT_DIR=%~dp0 -set VCS_DIRECTORY=ign-math +set VCS_DIRECTORY=gz-math set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true set COLCON_PACKAGE=ignition-math diff --git a/jenkins-scripts/ign_msgs-default-devel-windows-amd64.bat b/jenkins-scripts/ign_msgs-default-devel-windows-amd64.bat index 702dcb618..9fbdedce2 100644 --- a/jenkins-scripts/ign_msgs-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_msgs-default-devel-windows-amd64.bat @@ -1,6 +1,6 @@ set SCRIPT_DIR=%~dp0 -set VCS_DIRECTORY=ign-msgs +set VCS_DIRECTORY=gz-msgs set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true diff --git a/jenkins-scripts/ign_physics-default-devel-windows-amd64.bat b/jenkins-scripts/ign_physics-default-devel-windows-amd64.bat index 7eecb22bd..c1e9ec62e 100644 --- a/jenkins-scripts/ign_physics-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_physics-default-devel-windows-amd64.bat @@ -1,6 +1,6 @@ set SCRIPT_DIR=%~dp0 -set VCS_DIRECTORY=ign-physics +set VCS_DIRECTORY=gz-physics set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true diff --git a/jenkins-scripts/ign_plugin-default-devel-windows-amd64.bat b/jenkins-scripts/ign_plugin-default-devel-windows-amd64.bat index 0ccc66b6a..f90eac5c2 100644 --- a/jenkins-scripts/ign_plugin-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_plugin-default-devel-windows-amd64.bat @@ -1,6 +1,6 @@ set SCRIPT_DIR=%~dp0 -set VCS_DIRECTORY=ign-plugin +set VCS_DIRECTORY=gz-plugin set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true diff --git a/jenkins-scripts/ign_rendering-default-devel-windows-amd64.bat b/jenkins-scripts/ign_rendering-default-devel-windows-amd64.bat index a76a2837a..bcf0a19aa 100644 --- a/jenkins-scripts/ign_rendering-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_rendering-default-devel-windows-amd64.bat @@ -1,7 +1,7 @@ @echo on set SCRIPT_DIR=%~dp0 -set VCS_DIRECTORY=ign-rendering +set VCS_DIRECTORY=gz-rendering set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true diff --git a/jenkins-scripts/ign_sensors-default-devel-windows-amd64.bat b/jenkins-scripts/ign_sensors-default-devel-windows-amd64.bat index 7bb6fe561..a4b5bede6 100644 --- a/jenkins-scripts/ign_sensors-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_sensors-default-devel-windows-amd64.bat @@ -1,7 +1,7 @@ @echo on set SCRIPT_DIR=%~dp0 -set VCS_DIRECTORY=ign-sensors +set VCS_DIRECTORY=gz-sensors set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true diff --git a/jenkins-scripts/ign_tools-default-devel-windows-amd64.bat b/jenkins-scripts/ign_tools-default-devel-windows-amd64.bat index 236a050f7..d9b607aeb 100644 --- a/jenkins-scripts/ign_tools-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_tools-default-devel-windows-amd64.bat @@ -1,6 +1,6 @@ set SCRIPT_DIR=%~dp0 -set VCS_DIRECTORY=ign-tools +set VCS_DIRECTORY=gz-tools set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true diff --git a/jenkins-scripts/ign_transport-default-devel-windows-amd64.bat b/jenkins-scripts/ign_transport-default-devel-windows-amd64.bat index 20395af4f..a1cc75ed6 100644 --- a/jenkins-scripts/ign_transport-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_transport-default-devel-windows-amd64.bat @@ -2,7 +2,7 @@ set SCRIPT_DIR=%~dp0 -set VCS_DIRECTORY=ign-transport +set VCS_DIRECTORY=gz-transport set PLATFORM_TO_BUILD=amd64 set IGN_CLEAN_WORKSPACE=true diff --git a/jenkins-scripts/ign_utils-default-devel-windows-amd64.bat b/jenkins-scripts/ign_utils-default-devel-windows-amd64.bat index c22638f54..6a7acb02e 100644 --- a/jenkins-scripts/ign_utils-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_utils-default-devel-windows-amd64.bat @@ -1,7 +1,7 @@ @echo on set SCRIPT_DIR=%~dp0 -set VCS_DIRECTORY=ign-utils +set VCS_DIRECTORY=gz-utils set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true From 817fbdfb53f6416d239cbc19f1cf61c5ae28b2b2 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 3 Aug 2022 20:48:39 +0200 Subject: [PATCH 180/540] Use default for RUN_AUTOPKGTEST in _gazebo_utils.sh (#733) * Use default for RUN_AUTOPKGTEST in _gazebo_utils.sh --- jenkins-scripts/docker/lib/_gazebo_utils.sh | 4 ++++ jenkins-scripts/docker/lib/debbuild-base.bash | 4 ---- jenkins-scripts/docker/lib/debian-git-repo-base.bash | 2 -- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/jenkins-scripts/docker/lib/_gazebo_utils.sh b/jenkins-scripts/docker/lib/_gazebo_utils.sh index b7cdc8631..1f5059c26 100644 --- a/jenkins-scripts/docker/lib/_gazebo_utils.sh +++ b/jenkins-scripts/docker/lib/_gazebo_utils.sh @@ -37,6 +37,10 @@ fi echo '# END SECTION' """ +# autopkgtest is a mechanism to test the installation of the generated packages +# at the end of the package production. +RUN_AUTOPKGTEST=${RUN_AUTOPKGTEST:-true} + DEBBUILD_AUTOPKGTEST=""" if $RUN_AUTOPKGTEST; then echo '# BEGIN SECTION: run autopkgtest' diff --git a/jenkins-scripts/docker/lib/debbuild-base.bash b/jenkins-scripts/docker/lib/debbuild-base.bash index e1b6dcce4..2dce40924 100644 --- a/jenkins-scripts/docker/lib/debbuild-base.bash +++ b/jenkins-scripts/docker/lib/debbuild-base.bash @@ -19,10 +19,6 @@ fi # testing jobs and seems to be slow at the end of jenkins jobs export ENABLE_REAPER=false -# autopkgtest is a mechanism to test the installation of the generated packages -# at the end of the package production. -RUN_AUTOPKGTEST=${RUN_AUTOPKGTEST:-true} - . ${SCRIPT_DIR}/lib/boilerplate_prepare.sh . ${SCRIPT_DIR}/lib/_gazebo_utils.sh diff --git a/jenkins-scripts/docker/lib/debian-git-repo-base.bash b/jenkins-scripts/docker/lib/debian-git-repo-base.bash index 48ad8121f..1578ebf2c 100644 --- a/jenkins-scripts/docker/lib/debian-git-repo-base.bash +++ b/jenkins-scripts/docker/lib/debian-git-repo-base.bash @@ -25,8 +25,6 @@ if [[ -z ${BRANCH} ]]; then export CLONE_NEEDED=false fi -RUN_AUTOPKGTEST=${RUN_AUTOPKGTEST:-true} - cat > build.sh << DELIM ################################################### # Make project-specific changes here From 180794ed805067794a34e78c4eea8fb293ffccb7 Mon Sep 17 00:00:00 2001 From: Louise Poubel Date: Wed, 3 Aug 2022 15:20:07 -0700 Subject: [PATCH 181/540] Use gz- projects on Homebrew CI (#778) * Use gz- projects on Homebrew CI Signed-off-by: Louise Poubel * More instances Signed-off-by: Louise Poubel * Don't be greedy, keep ign- path for now Signed-off-by: Louise Poubel Co-authored-by: Jose Luis Rivero --- jenkins-scripts/dsl/ignition.dsl | 4 ++-- jenkins-scripts/dsl/ignition_collection.dsl | 2 +- .../lib/project-default-devel-homebrew-amd64.bash | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index ab23bf69d..50951a1af 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -558,7 +558,7 @@ ignition_software.each { ign_sw -> then /bin/bash -xe "\$HOMEBREW_SCRIPT" else - /bin/bash -xe "./scripts/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash" "ignition-${ign_sw}" + /bin/bash -xe "./scripts/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash" "gz-${ign_sw}" fi """.stripIndent()) } @@ -591,7 +591,7 @@ ignition_software.each { ign_sw -> then /bin/bash -xe "\$HOMEBREW_SCRIPT" else - /bin/bash -xe "./scripts/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash" "ignition-${ign_sw}" + /bin/bash -xe "./scripts/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash" "gz-${ign_sw}" fi """.stripIndent()) } diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index 1bea5cdab..27e812731 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -355,7 +355,7 @@ ignition_collections.each { ign_collection -> shell("""\ #!/bin/bash -xe - /bin/bash -xe "./scripts/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash" "ignition-${ign_collection_name}" + /bin/bash -xe "./scripts/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash" "gz-${ign_collection_name}" """.stripIndent()) } } diff --git a/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash b/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash index f080fe263..1d09e0701 100644 --- a/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash +++ b/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash @@ -13,10 +13,10 @@ export HOMEBREW_MAKE_JOBS=${MAKE_JOBS} PROJECT=$1 # project will have the major version included (ex gazebo2) PROJECT_ARGS=${2} -# In ignition projects, the name of the repo and the formula does not match +# TODO(chapulina) Use gz path instead of legacy ign PROJECT_PATH=${PROJECT} -if [[ ${PROJECT/ignition} != ${PROJECT} ]]; then - PROJECT_PATH="ign${PROJECT/ignition}" +if [[ ${PROJECT/gz} != ${PROJECT} ]]; then + PROJECT_PATH="ign${PROJECT/gz}" PROJECT_PATH="${PROJECT_PATH/[0-9]*}" fi From aa17d02a69a2f180817d58dc06f5f85d3b1dac4d Mon Sep 17 00:00:00 2001 From: Louise Poubel Date: Wed, 3 Aug 2022 16:35:19 -0700 Subject: [PATCH 182/540] Fix gz-sim for Homebrew (#780) Signed-off-by: Louise Poubel --- jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash | 3 +++ 1 file changed, 3 insertions(+) diff --git a/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash b/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash index 1d09e0701..ebea2ca20 100644 --- a/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash +++ b/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash @@ -20,6 +20,9 @@ if [[ ${PROJECT/gz} != ${PROJECT} ]]; then PROJECT_PATH="${PROJECT_PATH/[0-9]*}" fi +# Temporary fix for gz-sim PROJECT_PATH is ign-gazebo, PROJECT is gz-sim +PROJECT=${PROJECT/gz-gazebo/gz-sim} + # Check for major version number # the PROJECT_FORMULA variable is only used for dependency resolution PROJECT_FORMULA=${PROJECT//[0-9]}$(\ From 8ee068d19d2f8e318289b1673d34ca423cc72357 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 5 Aug 2022 00:57:29 +0200 Subject: [PATCH 183/540] ign2gz: fixes for checkout directory names in Win/Brew (#779) * Modify checkout repository names to match software * Fixes for checkout directories in Win/Brew * Restrict changes in GenericAny to keep Linux files working Signed-off-by: Jose Luis Rivero --- .../dsl/_configs_/GenericAnyJobGitHub.groovy | 3 +++ jenkins-scripts/dsl/_configs_/Globals.groovy | 3 +++ jenkins-scripts/dsl/_configs_/OSRFGitHub.groovy | 4 ++-- .../_configs_/OSRFWinCompilationAnyGitHub.groovy | 4 +++- jenkins-scripts/dsl/ignition.dsl | 14 +++++++++----- 5 files changed, 20 insertions(+), 8 deletions(-) diff --git a/jenkins-scripts/dsl/_configs_/GenericAnyJobGitHub.groovy b/jenkins-scripts/dsl/_configs_/GenericAnyJobGitHub.groovy index ecc3cd7d6..e6d96d143 100644 --- a/jenkins-scripts/dsl/_configs_/GenericAnyJobGitHub.groovy +++ b/jenkins-scripts/dsl/_configs_/GenericAnyJobGitHub.groovy @@ -25,7 +25,10 @@ class GenericAnyJobGitHub // Get repo name for relativeTargetDirectory String github_repo_name = github_repo.substring(github_repo.lastIndexOf("/") + 1) + // Use new gz- repositories + // TODO(jrivero): move this before github_repo_name once -compilation + // scripts are prepared to work with gz- prefix github_repo = Globals.ign2gz(github_repo) job.with diff --git a/jenkins-scripts/dsl/_configs_/Globals.groovy b/jenkins-scripts/dsl/_configs_/Globals.groovy index ee82588aa..d7e2e2a3b 100644 --- a/jenkins-scripts/dsl/_configs_/Globals.groovy +++ b/jenkins-scripts/dsl/_configs_/Globals.groovy @@ -33,6 +33,9 @@ class Globals static String ign2gz(String str) { str = str.replaceAll("ignitionrobotics","gazebosim") str = str.replaceAll("ign-gazebo","gz-sim") + // This one is to workaround failures in main DSL files generating + // gz-gazebo instead of gz-sim + str = str.replaceAll("gz-gazebo","gz-sim") return str.replaceAll("ign-","gz-") } diff --git a/jenkins-scripts/dsl/_configs_/OSRFGitHub.groovy b/jenkins-scripts/dsl/_configs_/OSRFGitHub.groovy index 3ead839bb..df735c901 100644 --- a/jenkins-scripts/dsl/_configs_/OSRFGitHub.groovy +++ b/jenkins-scripts/dsl/_configs_/OSRFGitHub.groovy @@ -8,13 +8,13 @@ class OSRFGitHub String rev = 'master', String subdir = 'NOT-DEFINED-USE-DEFAULT') { + repo = Globals.ign2gz(repo) + String software_name = repo.tokenize('/').last() if (subdir == 'NOT-DEFINED-USE-DEFAULT') subdir = software_name - repo = Globals.ign2gz(repo) - job.with { scm { diff --git a/jenkins-scripts/dsl/_configs_/OSRFWinCompilationAnyGitHub.groovy b/jenkins-scripts/dsl/_configs_/OSRFWinCompilationAnyGitHub.groovy index 70d5a79bc..960d4f021 100644 --- a/jenkins-scripts/dsl/_configs_/OSRFWinCompilationAnyGitHub.groovy +++ b/jenkins-scripts/dsl/_configs_/OSRFWinCompilationAnyGitHub.groovy @@ -21,8 +21,10 @@ class OSRFWinCompilationAnyGitHub OSRFWinCompilation.create(job, enable_testing, enable_cmake_warnings) /* Properties from generic any */ + // TODO(jrivero): remove replace once the rest of the migraton code is + // deployed in the ignition.dsl file GenericAnyJobGitHub.create(job, - github_repo, + github_repo.replace('gz-gazebo','gz-sim'), supported_branches, enable_github_pr_integration) } diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index 50951a1af..c957b3399 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -558,7 +558,9 @@ ignition_software.each { ign_sw -> then /bin/bash -xe "\$HOMEBREW_SCRIPT" else - /bin/bash -xe "./scripts/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash" "gz-${ign_sw}" + software_name="gz-${ign_sw}" + [[ ${ign_sw} == 'gazebo' ]] && software_name="gz-sim" + /bin/bash -xe "./scripts/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash" "\${software_name}" fi """.stripIndent()) } @@ -591,7 +593,9 @@ ignition_software.each { ign_sw -> then /bin/bash -xe "\$HOMEBREW_SCRIPT" else - /bin/bash -xe "./scripts/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash" "gz-${ign_sw}" + software_name="gz-${ign_sw}" + [[ ${ign_sw} == 'gazebo' ]] && software_name="gz-sim" + /bin/bash -xe "./scripts/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash" "\${software_name}" fi """.stripIndent()) } @@ -665,7 +669,7 @@ ignition_software.each { ign_sw -> def ignition_win_ci_any_job = job(ignition_win_ci_any_job_name) OSRFWinCompilationAnyGitHub.create(ignition_win_ci_any_job, - "gazebosim/ign-${ign_sw}", + "gazebosim/gz-${ign_sw}", enable_testing(ign_sw), supported_branches, ENABLE_GITHUB_PR_INTEGRATION, @@ -701,8 +705,8 @@ ignition_software.each { ign_sw -> enable_testing(ign_sw), enable_cmake_warnings(ign_sw)) OSRFGitHub.create(ignition_win_ci_job, - "gazebosim/ign-${ign_sw}", - "${branch}", "ign-${ign_sw}") + "gazebosim/gz-${ign_sw}", + "${branch}") ignition_win_ci_job.with { From 66333973103d8e5e002d5b032cc31ec0cd824fdd Mon Sep 17 00:00:00 2001 From: Louise Poubel Date: Thu, 4 Aug 2022 18:48:26 -0700 Subject: [PATCH 184/540] =?UTF-8?q?Homebrew:=20more=20gz-sim=20band-aids?= =?UTF-8?q?=20=F0=9F=A9=B9=20=20(#782)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * More homebrew fixes Signed-off-by: Louise Poubel * ign-gazebo? Signed-off-by: Louise Poubel --- jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash b/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash index ebea2ca20..6e22201d4 100644 --- a/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash +++ b/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash @@ -20,8 +20,9 @@ if [[ ${PROJECT/gz} != ${PROJECT} ]]; then PROJECT_PATH="${PROJECT_PATH/[0-9]*}" fi -# Temporary fix for gz-sim PROJECT_PATH is ign-gazebo, PROJECT is gz-sim +# Temporary fix for gz-sim PROJECT=${PROJECT/gz-gazebo/gz-sim} +PROJECT_PATH=${PROJECT_PATH/ign-sim/ign-gazebo} # Check for major version number # the PROJECT_FORMULA variable is only used for dependency resolution From 89658305da041c95a11d93ccdd3a9668e0fc79f8 Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Tue, 9 Aug 2022 12:46:48 -0700 Subject: [PATCH 185/540] nightlies: build from release branches (#785) Signed-off-by: Steve Peters --- jenkins-scripts/dsl/ignition_collection.dsl | 30 ++++++++++----------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index 27e812731..15f638148 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -18,22 +18,22 @@ ignition_collections = [ // These are the branches currently targeted at the upcoming collection // They're in topological order nightly_jobs: [ - 'tools' : [ debbuild: 'ign-tools2' , branch: 'main' ], - 'cmake' : [ debbuild: 'ign-cmake3' , branch: 'main' ], - 'utils' : [ debbuild: 'ign-utils2' , branch: 'main' ], - 'math' : [ debbuild: 'ign-math7' , branch: 'main' ], - 'plugin' : [ debbuild: 'ign-plugin2' , branch: 'main' ], - 'common' : [ debbuild: 'ign-common5' , branch: 'main' ], - 'msgs' : [ debbuild: 'ign-msgs9' , branch: 'main' ], - 'rendering' : [ debbuild: 'ign-rendering7' , branch: 'main' ], - 'sdformat' : [ debbuild: 'sdformat13' , branch: 'main' ], - 'fuel-tools': [ debbuild: 'ign-fuel-tools8' , branch: 'main' ], - 'transport' : [ debbuild: 'ign-transport12' , branch: 'main' ], - 'gui' : [ debbuild: 'ign-gui7' , branch: 'main' ], - 'sensors' : [ debbuild: 'ign-sensors7' , branch: 'main' ], - 'physics' : [ debbuild: 'ign-physics6' , branch: 'main' ], + 'cmake' : [ debbuild: 'ign-cmake3' , branch: 'gz-cmake3' ], + 'tools' : [ debbuild: 'ign-tools2' , branch: 'gz-tools2' ], + 'utils' : [ debbuild: 'ign-utils2' , branch: 'gz-utils2' ], + 'math' : [ debbuild: 'ign-math7' , branch: 'gz-math7' ], + 'plugin' : [ debbuild: 'ign-plugin2' , branch: 'gz-plugin2' ], + 'common' : [ debbuild: 'ign-common5' , branch: 'gz-common5' ], + 'msgs' : [ debbuild: 'ign-msgs9' , branch: 'gz-msgs9' ], + 'rendering' : [ debbuild: 'ign-rendering7' , branch: 'gz-rendering7' ], + 'sdformat' : [ debbuild: 'sdformat13' , branch: 'sdf13' ], + 'fuel-tools': [ debbuild: 'ign-fuel-tools8' , branch: 'gz-fuel-tools8' ], + 'transport' : [ debbuild: 'ign-transport12' , branch: 'gz-transport12' ], + 'gui' : [ debbuild: 'ign-gui7' , branch: 'gz-gui7' ], + 'sensors' : [ debbuild: 'ign-sensors7' , branch: 'gz-sensors7' ], + 'physics' : [ debbuild: 'ign-physics6' , branch: 'gz-physics6' ], 'gazebo' : [ debbuild: 'ign-gazebo7' , branch: 'main' ], - 'launch' : [ debbuild: 'ign-launch6' , branch: 'main' ], + 'launch' : [ debbuild: 'ign-launch6' , branch: 'gz-launch6' ], 'garden' : [ debbuild: 'ign-garden' , branch: 'main' ], ], ], From b99228b18289bef6db2ef9c6068f422abfde0a53 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 10 Aug 2022 04:19:37 +0200 Subject: [PATCH 186/540] Add a github action CI to check correctness of DSL files (#783) * Add github action to check DSL files integrity * Fix DSL code in GenericToken --- .github/workflows/ci.yaml | 52 +++++++++++++++++++ .../dsl/_configs_/GenericRemoteToken.groovy | 7 ++- 2 files changed, 58 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/ci.yaml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 000000000..c927566ce --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,52 @@ +name: CI + +# base_ref / head_reaf are only available in PRs +on: [pull_request] + +jobs: + dsl_ci: + runs-on: ubuntu-latest + name: Diff for DSL code + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Idenfify files changed in this PR + id: files + run: | + git diff --name-only origin/${{ github.base_ref }}...origin/${{ github.head_ref }} + echo "::set-output name=changed-files::$(git diff --name-only origin/${{ github.base_ref }}...origin/${{ github.head_ref }}| tr '\n' ' ')" + - name: Run testing on changed config files + id: dsl_check + run: | + for changed_file in ${{ steps.files.outputs.changed-files }}; do + if [[ ${changed_file} != ${changed_file/dsl\/*} ]]; then + echo "+ Detected at leat one config file: ${changed_file}." + echo "::set-output name=run_job::true" + break + else + echo "::set-output name=run_job::false" + fi + done + - name: Checkout + if: steps.dsl_check.outputs.run_job == 'true' + uses: actions/checkout@v2 + with: + fetch-depth: 2 + - uses: actions/setup-java@v3 + if: steps.dsl_check.outputs.run_job == 'true' + with: + distribution: 'temurin' + java-version: '11' + - name: Download job dsl jar + if: steps.dsl_check.outputs.run_job == 'true' + run: curl https://repo.jenkins-ci.org/public/org/jenkins-ci/plugins/job-dsl-core/1.77/job-dsl-core-1.77-standalone.jar -o jobdsl.jar + - name: Generate all DSL files + if: steps.dsl_check.outputs.run_job == 'true' + run: | + # simulate token for brew_release + sudo mkdir -p /var/lib/jenkins/ && sudo touch /var/lib/jenkins/remote_token + sudo chown -R ${USER} /var/lib/jenkins + cd jenkins-scripts/dsl + java -jar ../../jobdsl.jar *.dsl diff --git a/jenkins-scripts/dsl/_configs_/GenericRemoteToken.groovy b/jenkins-scripts/dsl/_configs_/GenericRemoteToken.groovy index dd754900b..07e29d7c2 100644 --- a/jenkins-scripts/dsl/_configs_/GenericRemoteToken.groovy +++ b/jenkins-scripts/dsl/_configs_/GenericRemoteToken.groovy @@ -15,7 +15,12 @@ class GenericRemoteToken println("check your filesystem in the jenkins node for: ") println(token_file) // We can not use exit here, DSL job hangs - buildUnstable() + job.with + { + steps { + setBuildResult('UNSTABLE') + } + } } job.with From 6549ce15f66eb34865f4af49bb27b15e8d0398c4 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 10 Aug 2022 06:25:33 +0200 Subject: [PATCH 187/540] Be sure of update gzdev if docker cache is invalidated (#786) * Be sure of update gzdev if docker cache is invalidated If the docker command just before invalidates the cache, a new install of gzdev is needed to update to possible recent changes in configuration and/or code and not being used since the docker cache did not get them. Signed-off-by: Jose Luis Rivero * bash function need to go outside redirection --- jenkins-scripts/docker/lib/docker_generate_dockerfile.bash | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash b/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash index 1e2613061..dda26bab6 100644 --- a/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash +++ b/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash @@ -280,6 +280,11 @@ cat >> Dockerfile << DELIM_DOCKER3 RUN echo "${MONTH_YEAR_STR}" DELIM_DOCKER3 +# If the previous command invalidated the cache, a new install of gzdev is +# needed to update to possible recent changes in configuration and/or code and +# not being used since the docker cache did not get them. +dockerfile_install_gzdev_repos + cat >> Dockerfile << DELIM_DOCKER3_2 RUN sed -i -e 's:13\.56\.139\.45:packages.osrfoundation.org:g' /etc/apt/sources.list.d/* || true \ && (apt-get update || (rm -rf /var/lib/apt/lists/* && apt-get ${APT_PARAMS} update)) \ From 368ae14498ee765705066cdf4549cdbdab50f937 Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Wed, 10 Aug 2022 15:09:06 -0700 Subject: [PATCH 188/540] sdformat.dsl: rename internal variables (#789) There are several *_branches variables that are actually sdformat package names (sdformat6, sdformat9, etc.) or main. This renames the variables to *_versions to try to clarify matters. Signed-off-by: Steve Peters --- jenkins-scripts/dsl/sdformat.dsl | 64 ++++++++++++++++---------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/jenkins-scripts/dsl/sdformat.dsl b/jenkins-scripts/dsl/sdformat.dsl index de9e37c15..ba2cd0ca8 100644 --- a/jenkins-scripts/dsl/sdformat.dsl +++ b/jenkins-scripts/dsl/sdformat.dsl @@ -1,8 +1,8 @@ import _configs_.* import javaposse.jobdsl.dsl.Job -def sdformat_supported_branches = [ 'sdformat6' , 'sdformat9', 'sdformat12' ] -def sdformat_gz11_branches = [ 'sdformat9', 'sdformat10', 'sdformat12', 'main' ] +def sdformat_supported_versions = [ 'sdformat6' , 'sdformat9', 'sdformat12' ] +def sdformat_gz11_versions = [ 'sdformat9', 'sdformat10', 'sdformat12', 'main' ] // nightly and prereleases def extra_sdformat_debbuilder = [ 'sdformat13' ] @@ -32,7 +32,7 @@ String get_sdformat_branch_name(String full_branch_name) // ABI Checker job // Need to be the before ci-pr_any so the abi job name is defined -abi_branches = sdformat_supported_branches.collect { it -> get_sdformat_branch_name(it) } +abi_branches = sdformat_supported_versions.collect { it -> get_sdformat_branch_name(it) } abi_distro.each { distro -> supported_arches.each { arch -> abi_job_name = "sdformat-abichecker-any_to_any-ubuntu_auto-${arch}" @@ -136,19 +136,19 @@ other_supported_distros.each { distro -> } // end of arch } // end of distro -void generate_asan_ci_job(sdformat_ci_job, branch, distro, arch) +void generate_asan_ci_job(sdformat_ci_job, version, distro, arch) { - generate_ci_job(sdformat_ci_job, branch, distro, arch, + generate_ci_job(sdformat_ci_job, version, distro, arch, '-DIGN_SANITIZER=Address', Globals.MAKETEST_SKIP_IGN) } -void generate_ci_job(sdformat_ci_job, branch, distro, arch, +void generate_ci_job(sdformat_ci_job, version, distro, arch, extra_cmake = '', extra_test = '') { OSRFLinuxCompilation.create(sdformat_ci_job) OSRFGitHub.create(sdformat_ci_job, "gazebosim/sdformat", - get_sdformat_branch_name(branch)) + get_sdformat_branch_name(version)) sdformat_ci_job.with { steps { @@ -166,12 +166,12 @@ void generate_ci_job(sdformat_ci_job, branch, distro, arch, } // BRANCHES CI JOB @ SCM -sdformat_supported_branches.each { branch -> +sdformat_supported_versions.each { version -> ci_distro.each { distro -> supported_arches.each { arch -> // ci job for the rest of arches / scm@daily - def sdformat_ci_job = job("sdformat-ci-${branch}-${distro}-${arch}") - generate_ci_job(sdformat_ci_job, branch, distro, arch) + def sdformat_ci_job = job("sdformat-ci-${version}-${distro}-${arch}") + generate_ci_job(sdformat_ci_job, version, distro, arch) sdformat_ci_job.with { triggers { @@ -179,8 +179,8 @@ sdformat_supported_branches.each { branch -> } } // ci_asan job for the rest of arches / scm@weekend - def sdformat_ci_asan_job = job("sdformat-ci_asan-${branch}-${distro}-${arch}") - generate_asan_ci_job(sdformat_ci_asan_job, branch, distro, arch) + def sdformat_ci_asan_job = job("sdformat-ci_asan-${version}-${distro}-${arch}") + generate_asan_ci_job(sdformat_ci_asan_job, version, distro, arch) sdformat_ci_asan_job.with { triggers { @@ -218,11 +218,11 @@ ci_distro.each { distro -> } // INSTALL LINUX -DEV PACKAGES ALL PLATFORMS @ CRON/DAILY -sdformat_supported_branches.each { branch -> +sdformat_supported_versions.each { version -> ci_distro.each { distro -> supported_arches.each { arch -> // -------------------------------------------------------------- - def install_default_job = job("sdformat-install-${branch}_pkg-${distro}-${arch}") + def install_default_job = job("sdformat-install-${version}_pkg-${distro}-${arch}") OSRFLinuxInstall.create(install_default_job) install_default_job.with { @@ -230,7 +230,7 @@ sdformat_supported_branches.each { branch -> cron(Globals.CRON_EVERY_THREE_DAYS) } - def dev_package = "lib${branch}-dev" + def dev_package = "lib${version}-dev" steps { shell("""\ @@ -239,25 +239,25 @@ sdformat_supported_branches.each { branch -> export DISTRO=${distro} export ARCH=${arch} export INSTALL_JOB_PKG=${dev_package} - export GZDEV_PROJECT_NAME="${branch}" + export GZDEV_PROJECT_NAME="${version}" /bin/bash -x ./scripts/jenkins-scripts/docker/generic-install-test-job.bash """.stripIndent()) } } // end of with } // end of arch } // end of distro -} // end of branch +} // end of version // -------------------------------------------------------------- // DEBBUILD: linux package builder -all_debbuild_branches = sdformat_supported_branches + extra_sdformat_debbuilder -all_debbuild_branches.each { branch -> - def build_pkg_job = job("${branch}-debbuilder") +all_debbuild_versions = sdformat_supported_versions + extra_sdformat_debbuilder +all_debbuild_versions.each { version -> + def build_pkg_job = job("${version}-debbuilder") OSRFLinuxBuildPkg.create(build_pkg_job) extra_cmd_str = "" - if (branch in sdformat_gz11_branches) + if (version in sdformat_gz11_versions) extra_cmd_str = "export NEED_C17_COMPILER=true" build_pkg_job.with @@ -294,12 +294,12 @@ sdformat_brew_ci_any_job.with } // 2. main in all branches @SCM/daily -all_branches = sdformat_supported_branches + 'main' -all_branches.each { branch -> - def sdformat_brew_ci_job = job("sdformat-ci-${branch}-homebrew-amd64") +all_versions = sdformat_supported_versions + 'main' +all_versions.each { version -> + def sdformat_brew_ci_job = job("sdformat-ci-${version}-homebrew-amd64") OSRFBrewCompilation.create(sdformat_brew_ci_job) OSRFGitHub.create(sdformat_brew_ci_job, "gazebosim/sdformat", - get_sdformat_branch_name(branch)) + get_sdformat_branch_name(version)) sdformat_brew_ci_job.with { @@ -318,8 +318,8 @@ all_branches.each { branch -> } // 3. install jobs to test bottles -sdformat_supported_branches.each { branch -> - def install_default_job = job("${branch}-install_bottle-homebrew-amd64") +sdformat_supported_versions.each { version -> + def install_default_job = job("${version}-install_bottle-homebrew-amd64") OSRFBrewInstall.create(install_default_job) install_default_job.with @@ -332,7 +332,7 @@ sdformat_supported_branches.each { branch -> shell("""\ #!/bin/bash -xe - /bin/bash -x ./scripts/jenkins-scripts/lib/project-install-homebrew.bash ${branch} + /bin/bash -x ./scripts/jenkins-scripts/lib/project-install-homebrew.bash ${version} """.stripIndent()) } @@ -367,12 +367,12 @@ sdformat_supported_branches.each { branch -> } // 2. main / @ SCM/Daily -all_branches = sdformat_supported_branches + 'main' -all_branches.each { branch -> - def sdformat_win_ci_job = job("sdformat-ci-${branch}-windows7-amd64") +all_versions = sdformat_supported_versions + 'main' +all_versions.each { version -> + def sdformat_win_ci_job = job("sdformat-ci-${version}-windows7-amd64") OSRFWinCompilation.create(sdformat_win_ci_job) OSRFGitHub.create(sdformat_win_ci_job, "gazebosim/sdformat", - get_sdformat_branch_name(branch)) + get_sdformat_branch_name(version)) sdformat_win_ci_job.with { triggers { From 1bedca210bac9b1c841fa5abe026491046e7d1d3 Mon Sep 17 00:00:00 2001 From: Louise Poubel Date: Thu, 11 Aug 2022 08:47:30 -0700 Subject: [PATCH 189/540] Build nightlies from gz-sim7 branch (#790) Signed-off-by: Louise Poubel Signed-off-by: Louise Poubel --- jenkins-scripts/dsl/ignition_collection.dsl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index 15f638148..93fa2eabb 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -32,7 +32,7 @@ ignition_collections = [ 'gui' : [ debbuild: 'ign-gui7' , branch: 'gz-gui7' ], 'sensors' : [ debbuild: 'ign-sensors7' , branch: 'gz-sensors7' ], 'physics' : [ debbuild: 'ign-physics6' , branch: 'gz-physics6' ], - 'gazebo' : [ debbuild: 'ign-gazebo7' , branch: 'main' ], + 'gazebo' : [ debbuild: 'ign-gazebo7' , branch: 'gz-sim7' ], 'launch' : [ debbuild: 'ign-launch6' , branch: 'gz-launch6' ], 'garden' : [ debbuild: 'ign-garden' , branch: 'main' ], ], From b81cdf7f18f1a6a1ed8ec2d3a4f12585ff88aebe Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Thu, 11 Aug 2022 17:13:27 -0700 Subject: [PATCH 190/540] Add CI jobs for garden branches (#787) * ignition.dsl: rename supported_branches method to supported_ign_branches in preparation for supported_gz_branches * Add supported_gz_branches method * sdformat.dsl: add sdformat13 to supported version list * Add todo comment for install jobs * Update special branch names for windows Signed-off-by: Steve Peters Co-authored-by: Louise Poubel --- jenkins-scripts/dsl/ignition.dsl | 78 +++++++++++++++++++++----------- jenkins-scripts/dsl/sdformat.dsl | 6 +-- 2 files changed, 55 insertions(+), 29 deletions(-) diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index c957b3399..2c8fc63e5 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -58,6 +58,21 @@ ignition_branches = [ 'cmake' : [ '2' ], 'tools' : [ '1' ], 'transport' : [ '4', '8', '11' ], 'utils' : [ '1' ]] +gz_branches = [ 'cmake' : [ '3' ], + 'common' : [ '5' ], + 'fuel-tools' : [ '8' ], + 'gui' : [ '7' ], + 'launch' : [ '6' ], + 'math' : [ '7' ], + 'msgs' : [ '9' ], + 'physics' : [ '6' ], + 'plugin' : [ '2' ], + 'rendering' : [ '7' ], + 'sensors' : [ '7' ], + 'sim' : [ '7' ], + 'tools' : [ '2' ], + 'transport' : [ '12' ], + 'utils' : [ '2' ]] // DESC: prerelease branches are managed as any other supported branches for // special cases different to major branches: get compilation CI on the branch // physics/sensors don't need to be included since they use main for gz11 @@ -65,22 +80,8 @@ ignition_prerelease_branches = [] // DESC: versioned names to generate debbuild jobs for special cases that // don't appear in ignition_branches (like nightly builders or 0-debbuild // jobs for the special cases of foo0 packages) -ignition_extra_debbuild = [ 'cmake3', - 'common5', - 'fuel-tools8', - 'gazebo7', - 'gui7', - 'launch6', - 'math7', - 'msgs9', - 'physics6', - 'plugin2', - 'rendering7', - 'sensors7', - 'tools2', - 'transport12', - 'utils1', // see comment https://github.com/gazebo-tooling/release-tools/pull/431#issuecomment-815099918 - 'utils2'] +ignition_extra_debbuild = [ 'gazebo7', + 'utils1'] // see comment https://github.com/gazebo-tooling/release-tools/pull/431#issuecomment-815099918 // DESC: exclude ignition from generate any install testing job ignition_no_pkg_yet = [ ] // DESC: major versions that has a package in the prerelease repo. Should @@ -156,7 +157,7 @@ Map merge_maps(Map[] sources) { // return major versions supported or empty if just 0,1 series under // -dev package. -ArrayList supported_branches(String ign_software) +ArrayList supported_ign_branches(String ign_software) { major_versions_registered = ignition_branches["${ign_software}"] @@ -165,6 +166,15 @@ ArrayList supported_branches(String ign_software) return major_versions_registered } +ArrayList supported_gz_branches(String gz_software) +{ + major_versions_registered = gz_branches["${gz_software}"] + + if (major_versions_registered == null) + return [ '' ] + + return major_versions_registered +} // return prerelease branch names ArrayList prerelease_branches(String ign_software) @@ -181,11 +191,16 @@ ArrayList prerelease_branches(String ign_software) ArrayList all_branches(String ign_software) { List branches = new ArrayList(); - supported_branches("${ign_software}").each { major_version -> + supported_ign_branches("${ign_software}").each { major_version -> if ("${major_version}") { branches.add("ign-${ign_software}${major_version}") } } + supported_gz_branches("${ign_software}").each { major_version -> + if ("${major_version}") { + branches.add("gz-${ign_software}${major_version}") + } + } branches.add('main') prerelease_branches("${ign_software}").each { branch -> if ("${branch}") { @@ -201,12 +216,22 @@ ArrayList all_debbuilders() List branches = new ArrayList(); // add all supported branches ignition_software.each { ign_software -> - supported_branches("${ign_software}").each { major_version -> + supported_ign_branches("${ign_software}").each { major_version -> if (major_version) { // No 1-debbuild versions, they use the unversioned job if ("${major_version}" == "0" || "${major_version}" == "1" ) major_version = "" - branches.add("ign-${ign_software}${major_version}") + + branches.add("ign-${ign_software}${major_version}") + } + } + supported_gz_branches("${ign_software}").each { major_version -> + if (major_version) { + // No 1-debbuild versions, they use the unversioned job + if ("${major_version}" == "0" || "${major_version}" == "1" ) + major_version = "" + + branches.add("ign-${ign_software}${major_version}") } } } @@ -221,15 +246,16 @@ ArrayList all_debbuilders() // return all ci branch names // Map with the form of: major versions as keys. // Lists of distros supported as values +// @TODO[scpeters] update this to support gz_branches after garden release Map supported_install_pkg_branches(String ign_software) { major_versions_prerelease = ignition_prerelease_pkgs["${ign_software}"] - // construct a map of stable packages based on supported_branches and + // construct a map of stable packages based on supported_ign_branches and // all_supported_distros map_of_stable_versions = [:] map_of_stable_versions[ign_software] = [:] - supported_branches(ign_software).each { major_version -> + supported_ign_branches(ign_software).each { major_version -> new_relation = [:] new_relation[major_version] = all_supported_distros map_of_stable_versions[ign_software] << new_relation @@ -661,11 +687,11 @@ ignition_software.each { ign_sw -> // ign-gazebo only support Windows from ign-gazebo5 if (ign_sw == 'gazebo') - supported_branches = [ 'ign-gazebo6', 'main' ] + supported_branches = [ 'ign-gazebo6', 'gz-sim7', 'main' ] // ign-launch only support Windows from ign-launch5 if (ign_sw == 'launch') - supported_branches = [ 'ign-launch5', 'main' ] + supported_branches = [ 'ign-launch5', 'gz-launch6', 'main' ] def ignition_win_ci_any_job = job(ignition_win_ci_any_job_name) OSRFWinCompilationAnyGitHub.create(ignition_win_ci_any_job, @@ -711,11 +737,11 @@ ignition_software.each { ign_sw -> ignition_win_ci_job.with { // ign-gazebo only works on Windows from ign-gazebo5 - if (branch == 'ign-gazebo3' || branch == 'ign-gazebo4') + if (branch == 'ign-gazebo3') disabled() // ign-launch was not ported to windows until 5 - if (branch == 'ign-launch2' || branch == 'ign-launch3' || branch == 'ign-launch4') + if (branch == 'ign-launch2') disabled() triggers { diff --git a/jenkins-scripts/dsl/sdformat.dsl b/jenkins-scripts/dsl/sdformat.dsl index ba2cd0ca8..d6d975816 100644 --- a/jenkins-scripts/dsl/sdformat.dsl +++ b/jenkins-scripts/dsl/sdformat.dsl @@ -1,10 +1,10 @@ import _configs_.* import javaposse.jobdsl.dsl.Job -def sdformat_supported_versions = [ 'sdformat6' , 'sdformat9', 'sdformat12' ] -def sdformat_gz11_versions = [ 'sdformat9', 'sdformat10', 'sdformat12', 'main' ] +def sdformat_supported_versions = [ 'sdformat6' , 'sdformat9', 'sdformat12', 'sdformat13' ] +def sdformat_gz11_versions = [ 'sdformat9', 'sdformat12', 'sdformat13', 'main' ] // nightly and prereleases -def extra_sdformat_debbuilder = [ 'sdformat13' ] +def extra_sdformat_debbuilder = [ ] // Main platform using for quick CI def ci_distro = Globals.get_ci_distro() From fad1ea85eb2e6f4b9cc5a3e98a831599e99b78f4 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Mon, 15 Aug 2022 17:42:42 +0200 Subject: [PATCH 191/540] Disable apt phased updates in Dockerfile (#777) Signed-off-by: Jose Luis Rivero Signed-off-by: Jose Luis Rivero --- jenkins-scripts/docker/lib/docker_generate_dockerfile.bash | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash b/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash index dda26bab6..867ec1283 100644 --- a/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash +++ b/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash @@ -164,6 +164,11 @@ DELIM_DEBIAN_APT fi if [[ ${LINUX_DISTRO} == 'ubuntu' ]]; then +# Opt-out of phased updates, which can create inconsistencies between installed package versions as different containers end up on different phases. +# https://wiki.ubuntu.com/PhasedUpdates +cat >> Dockerfile << DELIM_PHASED +RUN echo 'APT::Get::Never-Include-Phased-Updates "true";' > /etc/apt/apt.conf.d/90-phased-updates +DELIM_PHASED if [[ ${ARCH} != 'armhf' && ${ARCH} != 'arm64' ]]; then cat >> Dockerfile << DELIM_DOCKER_ARCH RUN echo "deb ${SOURCE_LIST_URL} ${DISTRO}-security main restricted universe multiverse" && \\ From 92fb29d988685d080bc9861994db8f136e6bebbd Mon Sep 17 00:00:00 2001 From: Louise Poubel Date: Wed, 17 Aug 2022 11:35:07 -0700 Subject: [PATCH 192/540] Remove buster from release dashboard (#791) Signed-off-by: Louise Poubel Signed-off-by: Louise Poubel --- terminal-dashboard/table.bash | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/terminal-dashboard/table.bash b/terminal-dashboard/table.bash index 06f5399e3..bb319c340 100755 --- a/terminal-dashboard/table.bash +++ b/terminal-dashboard/table.bash @@ -32,7 +32,8 @@ ARCHS=( "amd64") DISTROS=( "ubuntu" ) if [[ $PACKAGE_REPO != "nightly" ]]; then ARCHS+=( "i386" "arm64" "armhf") - DISTROS+=( "debian" ) + # No debian version supported across the stack right now + # DISTROS+=( "debian" ) fi for LIB in $(get_libraries_by_collection "${COLLECTION}" ); do @@ -50,8 +51,6 @@ for LIB in $(get_libraries_by_collection "${COLLECTION}" ); do elif [[ $COLLECTION == "garden" ]]; then VERS=( "focal" "jammy" ) fi - else - VERS=( "buster" ) # "sid" fi for VER in "${VERS[@]}" From d2b22785257f741ccaf77d3786ed975e9342b575 Mon Sep 17 00:00:00 2001 From: methylDragon Date: Thu, 18 Aug 2022 10:57:12 -0700 Subject: [PATCH 193/540] ign -> gz : Release Tools Migrate Internal Vars (#753) * Migrate COLCON_PACKAGE=ignition * Migrate some (hopefully private/localized) bash variables * Migrate BUILD_IGN_XXX type variables Signed-off-by: methylDragon Co-authored-by: Jose Luis Rivero Co-authored-by: Steve Peters --- .../docker/ign_cmake-compilation.bash | 12 +- .../docker/ign_common-compilation.bash | 16 +-- .../docker/ign_fuel-tools-compilation.bash | 14 +- .../docker/ign_gazebo-compilation.bash | 12 +- .../docker/ign_gui-compilation.bash | 16 +-- .../docker/ign_launch-compilation.bash | 14 +- .../docker/ign_launch-install-test-job.bash | 2 +- .../docker/ign_math-compilation.bash | 16 +-- .../docker/ign_msgs-compilation.bash | 16 +-- .../docker/ign_physics-compilation.bash | 14 +- .../docker/ign_plugin-compilation.bash | 16 +-- .../docker/ign_rendering-compilation.bash | 16 +-- .../docker/ign_rndf-compilation.bash | 2 +- .../docker/ign_sensors-compilation.bash | 14 +- .../docker/ign_transport-compilation.bash | 16 +-- .../docker/ign_utils-compilation.bash | 14 +- .../docker/ignition-abichecker.bash | 40 +++--- jenkins-scripts/docker/lib/_gazebo_utils.sh | 2 +- .../_generic_linux_compilation_build.sh.bash | 4 +- .../docker/lib/gazebo-base-default.bash | 34 ++--- .../docker/lib/generic-building-base.bash | 72 +++++----- jenkins-scripts/dsl/gazebo.dsl | 4 +- jenkins-scripts/dsl/sdformat.dsl | 2 +- ...ign_common-default-devel-windows-amd64.bat | 6 +- ...fuel-tools-default-devel-windows-amd64.bat | 2 +- ...ign_gazebo-default-devel-windows-amd64.bat | 6 +- .../ign_gui-default-devel-windows-amd64.bat | 6 +- ...ign_launch-default-devel-windows-amd64.bat | 6 +- .../ign_math-default-devel-windows-amd64.bat | 2 +- .../ign_msgs-default-devel-windows-amd64.bat | 2 +- ...gn_physics-default-devel-windows-amd64.bat | 6 +- ...ign_plugin-default-devel-windows-amd64.bat | 2 +- ..._rendering-default-devel-windows-amd64.bat | 8 +- ...gn_sensors-default-devel-windows-amd64.bat | 6 +- .../ign_tools-default-devel-windows-amd64.bat | 8 +- ..._transport-default-devel-windows-amd64.bat | 2 +- .../ign_utils-default-devel-windows-amd64.bat | 2 +- .../lib/colcon-default-devel-windows.bat | 6 +- jenkins-scripts/lib/dependencies_archive.sh | 134 +++++++++--------- jenkins-scripts/lib/sdformat-base-windows.bat | 8 +- jenkins-scripts/lib/windows_library.bat | 15 +- 41 files changed, 291 insertions(+), 304 deletions(-) diff --git a/jenkins-scripts/docker/ign_cmake-compilation.bash b/jenkins-scripts/docker/ign_cmake-compilation.bash index 732e98c52..b956bfd5b 100644 --- a/jenkins-scripts/docker/ign_cmake-compilation.bash +++ b/jenkins-scripts/docker/ign_cmake-compilation.bash @@ -20,17 +20,17 @@ export BUILDING_DEPENDENCIES="pkg-config" # Enable long-running ign-cmake tests in CI. export BUILDING_EXTRA_CMAKE_PARAMS+=" -DBUILDSYSTEM_TESTING=True" -# Identify IGN_CMAKE_MAJOR_VERSION to help with dependency resolution -IGN_CMAKE_MAJOR_VERSION=$(\ +# Identify GZ_CMAKE_MAJOR_VERSION to help with dependency resolution +GZ_CMAKE_MAJOR_VERSION=$(\ python3 ${SCRIPT_DIR}/../tools/detect_cmake_major_version.py \ ${WORKSPACE}/ign-cmake/CMakeLists.txt) -# Check IGN_CMAKE version is integer -if ! [[ ${IGN_CMAKE_MAJOR_VERSION} =~ ^-?[0-9]+$ ]]; then - echo "Error! IGN_CMAKE_MAJOR_VERSION is not an integer, check the detection" +# Check GZ_CMAKE version is integer +if ! [[ ${GZ_CMAKE_MAJOR_VERSION} =~ ^-?[0-9]+$ ]]; then + echo "Error! GZ_CMAKE_MAJOR_VERSION is not an integer, check the detection" exit -1 fi -export GZDEV_PROJECT_NAME="gz-cmake${IGN_CMAKE_MAJOR_VERSION}" +export GZDEV_PROJECT_NAME="gz-cmake${GZ_CMAKE_MAJOR_VERSION}" . ${SCRIPT_DIR}/lib/generic-building-base.bash diff --git a/jenkins-scripts/docker/ign_common-compilation.bash b/jenkins-scripts/docker/ign_common-compilation.bash index c70ddb0d8..2f1ecd1f0 100644 --- a/jenkins-scripts/docker/ign_common-compilation.bash +++ b/jenkins-scripts/docker/ign_common-compilation.bash @@ -15,23 +15,23 @@ if [[ -z ${DISTRO} ]]; then fi export BUILDING_SOFTWARE_DIRECTORY="ign-common" -export BUILDING_PKG_DEPENDENCIES_VAR_NAME="IGN_COMMON_DEPENDENCIES" +export BUILDING_PKG_DEPENDENCIES_VAR_NAME="GZ_COMMON_DEPENDENCIES" -# Identify IGN_COMMON_MAJOR_VERSION to help with dependency resolution -IGN_COMMON_MAJOR_VERSION=$(\ +# Identify GZ_COMMON_MAJOR_VERSION to help with dependency resolution +GZ_COMMON_MAJOR_VERSION=$(\ python3 ${SCRIPT_DIR}/../tools/detect_cmake_major_version.py \ ${WORKSPACE}/ign-common/CMakeLists.txt) -# Check IGN_COMMON version is integer -if ! [[ ${IGN_COMMON_MAJOR_VERSION} =~ ^-?[0-9]+$ ]]; then - echo "Error! IGN_COMMON_MAJOR_VERSION is not an integer, check the detection" +# Check GZ_COMMON version is integer +if ! [[ ${GZ_COMMON_MAJOR_VERSION} =~ ^-?[0-9]+$ ]]; then + echo "Error! GZ_COMMON_MAJOR_VERSION is not an integer, check the detection" exit -1 fi -if [[ ${IGN_COMMON_MAJOR_VERSION} -ge 3 ]]; then +if [[ ${GZ_COMMON_MAJOR_VERSION} -ge 3 ]]; then export NEED_C17_COMPILER=true fi -export GZDEV_PROJECT_NAME="gz-common${IGN_COMMON_MAJOR_VERSION}" +export GZDEV_PROJECT_NAME="gz-common${GZ_COMMON_MAJOR_VERSION}" . ${SCRIPT_DIR}/lib/generic-building-base.bash diff --git a/jenkins-scripts/docker/ign_fuel-tools-compilation.bash b/jenkins-scripts/docker/ign_fuel-tools-compilation.bash index aafe604d0..7bed0ae12 100644 --- a/jenkins-scripts/docker/ign_fuel-tools-compilation.bash +++ b/jenkins-scripts/docker/ign_fuel-tools-compilation.bash @@ -15,20 +15,20 @@ if [[ -z ${DISTRO} ]]; then fi export BUILDING_SOFTWARE_DIRECTORY="ign-fuel-tools" -export BUILDING_PKG_DEPENDENCIES_VAR_NAME="IGN_FUEL_TOOLS_DEPENDENCIES" +export BUILDING_PKG_DEPENDENCIES_VAR_NAME="GZ_FUEL_TOOLS_DEPENDENCIES" export BUILDING_JOB_REPOSITORIES="stable" -# Identify IGN_FUEL_TOOLS_MAJOR_VERSION to help with dependency resolution -IGN_FUEL_TOOLS_MAJOR_VERSION=$(\ +# Identify GZ_FUEL_TOOLS_MAJOR_VERSION to help with dependency resolution +GZ_FUEL_TOOLS_MAJOR_VERSION=$(\ python3 ${SCRIPT_DIR}/../tools/detect_cmake_major_version.py \ ${WORKSPACE}/ign-fuel-tools/CMakeLists.txt) -# Check IGN_FUEL_TOOLS version is integer -if ! [[ ${IGN_FUEL_TOOLS_MAJOR_VERSION} =~ ^-?[0-9]+$ ]]; then - echo "Error! IGN_FUEL_TOOLS_MAJOR_VERSION is not an integer, check the detection" +# Check GZ_FUEL_TOOLS version is integer +if ! [[ ${GZ_FUEL_TOOLS_MAJOR_VERSION} =~ ^-?[0-9]+$ ]]; then + echo "Error! GZ_FUEL_TOOLS_MAJOR_VERSION is not an integer, check the detection" exit -1 fi -export GZDEV_PROJECT_NAME="gz-fuel-tools${IGN_FUEL_TOOLS_MAJOR_VERSION}" +export GZDEV_PROJECT_NAME="gz-fuel-tools${GZ_FUEL_TOOLS_MAJOR_VERSION}" . ${SCRIPT_DIR}/lib/generic-building-base.bash diff --git a/jenkins-scripts/docker/ign_gazebo-compilation.bash b/jenkins-scripts/docker/ign_gazebo-compilation.bash index b7f81a6ae..86caabcbc 100644 --- a/jenkins-scripts/docker/ign_gazebo-compilation.bash +++ b/jenkins-scripts/docker/ign_gazebo-compilation.bash @@ -15,22 +15,22 @@ if [[ -z ${DISTRO} ]]; then fi export BUILDING_SOFTWARE_DIRECTORY="ign-gazebo" -export BUILDING_PKG_DEPENDENCIES_VAR_NAME="IGN_GAZEBO_DEPENDENCIES" +export BUILDING_PKG_DEPENDENCIES_VAR_NAME="GZ_SIM_DEPENDENCIES" -# Identify IGN_GAZEBO_MAJOR_VERSION to help with dependency resolution -IGN_GAZEBO_MAJOR_VERSION=$(\ +# Identify GZ_SIM_MAJOR_VERSION to help with dependency resolution +GZ_SIM_MAJOR_VERSION=$(\ python3 ${SCRIPT_DIR}/../tools/detect_cmake_major_version.py \ ${WORKSPACE}/ign-gazebo/CMakeLists.txt) # Check IGN_GAZEBO version is integer -if ! [[ ${IGN_GAZEBO_MAJOR_VERSION} =~ ^-?[0-9]+$ ]]; then - echo "Error! IGN_GAZEBO_MAJOR_VERSION is not an integer, check the detection" +if ! [[ ${GZ_SIM_MAJOR_VERSION} =~ ^-?[0-9]+$ ]]; then + echo "Error! GZ_SIM_MAJOR_VERSION is not an integer, check the detection" exit -1 fi export NEED_C17_COMPILER=true export GPU_SUPPORT_NEEDED=true -export GZDEV_PROJECT_NAME="gz-sim${IGN_GAZEBO_MAJOR_VERSION}" +export GZDEV_PROJECT_NAME="gz-sim${GZ_SIM_MAJOR_VERSION}" . ${SCRIPT_DIR}/lib/generic-building-base.bash diff --git a/jenkins-scripts/docker/ign_gui-compilation.bash b/jenkins-scripts/docker/ign_gui-compilation.bash index 9081132ba..4c00c02d2 100644 --- a/jenkins-scripts/docker/ign_gui-compilation.bash +++ b/jenkins-scripts/docker/ign_gui-compilation.bash @@ -15,24 +15,24 @@ if [[ -z ${DISTRO} ]]; then fi export BUILDING_SOFTWARE_DIRECTORY="ign-gui" -export BUILDING_PKG_DEPENDENCIES_VAR_NAME="IGN_GUI_DEPENDENCIES" +export BUILDING_PKG_DEPENDENCIES_VAR_NAME="GZ_GUI_DEPENDENCIES" -# Identify IGN_GUI_MAJOR_VERSION to help with dependency resolution -IGN_GUI_MAJOR_VERSION=$(\ +# Identify GZ_GUI_MAJOR_VERSION to help with dependency resolution +GZ_GUI_MAJOR_VERSION=$(\ python3 ${SCRIPT_DIR}/../tools/detect_cmake_major_version.py \ ${WORKSPACE}/ign-gui/CMakeLists.txt) -# Check IGN_GUI_MAJOR_VERSION version is integer -if ! [[ ${IGN_GUI_MAJOR_VERSION} =~ ^-?[0-9]+$ ]]; then - echo "Error! IGN_GUI_MAJOR_VERSION is not an integer, check the detection" +# Check GZ_GUI_MAJOR_VERSION version is integer +if ! [[ ${GZ_GUI_MAJOR_VERSION} =~ ^-?[0-9]+$ ]]; then + echo "Error! GZ_GUI_MAJOR_VERSION is not an integer, check the detection" exit -1 fi -if [[ ${IGN_GUI_MAJOR_VERSION} -ge 1 ]]; then +if [[ ${GZ_GUI_MAJOR_VERSION} -ge 1 ]]; then export NEED_C17_COMPILER=true fi export GPU_SUPPORT_NEEDED=true -export GZDEV_PROJECT_NAME="gz-gui${IGN_GUI_MAJOR_VERSION}" +export GZDEV_PROJECT_NAME="gz-gui${GZ_GUI_MAJOR_VERSION}" . ${SCRIPT_DIR}/lib/generic-building-base.bash diff --git a/jenkins-scripts/docker/ign_launch-compilation.bash b/jenkins-scripts/docker/ign_launch-compilation.bash index 691b21cee..7e5923e1c 100644 --- a/jenkins-scripts/docker/ign_launch-compilation.bash +++ b/jenkins-scripts/docker/ign_launch-compilation.bash @@ -15,20 +15,20 @@ if [[ -z ${DISTRO} ]]; then fi export BUILDING_SOFTWARE_DIRECTORY="ign-launch" -export BUILDING_PKG_DEPENDENCIES_VAR_NAME="IGN_LAUNCH_DEPENDENCIES" +export BUILDING_PKG_DEPENDENCIES_VAR_NAME="GZ_LAUNCH_DEPENDENCIES" -# Identify IGN_LAUNCH_MAJOR_VERSION to help with dependency resolution -IGN_LAUNCH_MAJOR_VERSION=$(\ +# Identify GZ_LAUNCH_MAJOR_VERSION to help with dependency resolution +GZ_LAUNCH_MAJOR_VERSION=$(\ python3 ${SCRIPT_DIR}/../tools/detect_cmake_major_version.py \ ${WORKSPACE}/ign-launch/CMakeLists.txt) -# Check IGN_LAUNCH version is integer -if ! [[ ${IGN_LAUNCH_MAJOR_VERSION} =~ ^-?[0-9]+$ ]]; then - echo "Error! IGN_LAUNCH_MAJOR_VERSION is not an integer, check the detection" +# Check GZ_LAUNCH version is integer +if ! [[ ${GZ_LAUNCH_MAJOR_VERSION} =~ ^-?[0-9]+$ ]]; then + echo "Error! GZ_LAUNCH_MAJOR_VERSION is not an integer, check the detection" exit -1 fi export NEED_C17_COMPILER=true -export GZDEV_PROJECT_NAME="gz-launch${IGN_LAUNCH_MAJOR_VERSION}" +export GZDEV_PROJECT_NAME="gz-launch${GZ_LAUNCH_MAJOR_VERSION}" . ${SCRIPT_DIR}/lib/generic-building-base.bash diff --git a/jenkins-scripts/docker/ign_launch-install-test-job.bash b/jenkins-scripts/docker/ign_launch-install-test-job.bash index f41da954b..6b91dbe00 100644 --- a/jenkins-scripts/docker/ign_launch-install-test-job.bash +++ b/jenkins-scripts/docker/ign_launch-install-test-job.bash @@ -10,7 +10,7 @@ export GPU_SUPPORT_NEEDED=true . ${SCRIPT_DIR}/lib/_gazebo_utils.sh INSTALL_JOB_POSTINSTALL_HOOK=""" -${IGN_GAZEBO_RUNTIME_TEST} +${GZ_SIM_RUNTIME_TEST} """ # Need bc to proper testing and parsing the time export DEPENDENCY_PKGS="${DEPENDENCY_PKGS} wget bc" diff --git a/jenkins-scripts/docker/ign_math-compilation.bash b/jenkins-scripts/docker/ign_math-compilation.bash index 528e9fae3..f18a769f3 100644 --- a/jenkins-scripts/docker/ign_math-compilation.bash +++ b/jenkins-scripts/docker/ign_math-compilation.bash @@ -16,23 +16,23 @@ fi export BUILDING_SOFTWARE_DIRECTORY="ign-math" export BUILDING_JOB_REPOSITORIES="stable" -export BUILDING_PKG_DEPENDENCIES_VAR_NAME="IGN_MATH_DEPENDENCIES" +export BUILDING_PKG_DEPENDENCIES_VAR_NAME="GZ_MATH_DEPENDENCIES" -# Identify IGN_MATH_MAJOR_VERSION to help with dependency resolution -IGN_MATH_MAJOR_VERSION=$(\ +# Identify GZ_MATH_MAJOR_VERSION to help with dependency resolution +GZ_MATH_MAJOR_VERSION=$(\ python3 ${SCRIPT_DIR}/../tools/detect_cmake_major_version.py \ ${WORKSPACE}/ign-math/CMakeLists.txt) -# Check IGN_MATH version is integer -if ! [[ ${IGN_MATH_MAJOR_VERSION} =~ ^-?[0-9]+$ ]]; then - echo "Error! IGN_MATH_MAJOR_VERSION is not an integer, check the detection" +# Check GZ_MATH version is integer +if ! [[ ${GZ_MATH_MAJOR_VERSION} =~ ^-?[0-9]+$ ]]; then + echo "Error! GZ_MATH_MAJOR_VERSION is not an integer, check the detection" exit -1 fi -if [[ ${IGN_MATH_MAJOR_VERSION} -ge 6 ]]; then +if [[ ${GZ_MATH_MAJOR_VERSION} -ge 6 ]]; then export NEED_C17_COMPILER=true fi -export GZDEV_PROJECT_NAME="gz-math${IGN_MATH_MAJOR_VERSION}" +export GZDEV_PROJECT_NAME="gz-math${GZ_MATH_MAJOR_VERSION}" . "${SCRIPT_DIR}/lib/generic-building-base.bash" diff --git a/jenkins-scripts/docker/ign_msgs-compilation.bash b/jenkins-scripts/docker/ign_msgs-compilation.bash index ab10e9e7b..8dbcd48e0 100644 --- a/jenkins-scripts/docker/ign_msgs-compilation.bash +++ b/jenkins-scripts/docker/ign_msgs-compilation.bash @@ -15,23 +15,23 @@ if [[ -z ${DISTRO} ]]; then fi export BUILDING_SOFTWARE_DIRECTORY="ign-msgs" -export BUILDING_PKG_DEPENDENCIES_VAR_NAME="IGN_MSGS_DEPENDENCIES" +export BUILDING_PKG_DEPENDENCIES_VAR_NAME="GZ_MSGS_DEPENDENCIES" -# Identify IGN_MSGS_MAJOR_VERSION to help with dependency resolution -IGN_MSGS_MAJOR_VERSION=$(\ +# Identify GZ_MSGS_MAJOR_VERSION to help with dependency resolution +GZ_MSGS_MAJOR_VERSION=$(\ python3 ${SCRIPT_DIR}/../tools/detect_cmake_major_version.py \ ${WORKSPACE}/ign-msgs/CMakeLists.txt) -# Check IGN_MSGS version is integer -if ! [[ ${IGN_MSGS_MAJOR_VERSION} =~ ^-?[0-9]+$ ]]; then - echo "Error! IGN_MSGS_MAJOR_VERSION is not an integer, check the detection" +# Check GZ_MSGS version is integer +if ! [[ ${GZ_MSGS_MAJOR_VERSION} =~ ^-?[0-9]+$ ]]; then + echo "Error! GZ_MSGS_MAJOR_VERSION is not an integer, check the detection" exit -1 fi -if [[ ${IGN_MSGS_MAJOR_VERSION} -ge 3 ]]; then +if [[ ${GZ_MSGS_MAJOR_VERSION} -ge 3 ]]; then export NEED_C17_COMPILER=true fi -export GZDEV_PROJECT_NAME="gz-msgs${IGN_MSGS_MAJOR_VERSION}" +export GZDEV_PROJECT_NAME="gz-msgs${GZ_MSGS_MAJOR_VERSION}" . ${SCRIPT_DIR}/lib/generic-building-base.bash diff --git a/jenkins-scripts/docker/ign_physics-compilation.bash b/jenkins-scripts/docker/ign_physics-compilation.bash index 257e21548..d57594a5c 100644 --- a/jenkins-scripts/docker/ign_physics-compilation.bash +++ b/jenkins-scripts/docker/ign_physics-compilation.bash @@ -15,20 +15,20 @@ if [[ -z ${DISTRO} ]]; then fi export BUILDING_SOFTWARE_DIRECTORY="ign-physics" -export BUILDING_PKG_DEPENDENCIES_VAR_NAME="IGN_PHYSICS_DEPENDENCIES" +export BUILDING_PKG_DEPENDENCIES_VAR_NAME="GZ_PHYSICS_DEPENDENCIES" -# Identify IGN_PHYSICS_MAJOR_VERSION to help with dependency resolution -IGN_PHYSICS_MAJOR_VERSION=$(\ +# Identify GZ_PHYSICS_MAJOR_VERSION to help with dependency resolution +GZ_PHYSICS_MAJOR_VERSION=$(\ python3 ${SCRIPT_DIR}/../tools/detect_cmake_major_version.py \ ${WORKSPACE}/ign-physics/CMakeLists.txt) -# Check IGN_PHYSICS version is integer -if ! [[ ${IGN_PHYSICS_MAJOR_VERSION} =~ ^-?[0-9]+$ ]]; then - echo "Error! IGN_PHYSICS_MAJOR_VERSION is not an integer, check the detection" +# Check GZ_PHYSICS version is integer +if ! [[ ${GZ_PHYSICS_MAJOR_VERSION} =~ ^-?[0-9]+$ ]]; then + echo "Error! GZ_PHYSICS_MAJOR_VERSION is not an integer, check the detection" exit -1 fi export NEED_C17_COMPILER=true -export GZDEV_PROJECT_NAME="gz-physics${IGN_PHYSICS_MAJOR_VERSION}" +export GZDEV_PROJECT_NAME="gz-physics${GZ_PHYSICS_MAJOR_VERSION}" . ${SCRIPT_DIR}/lib/generic-building-base.bash diff --git a/jenkins-scripts/docker/ign_plugin-compilation.bash b/jenkins-scripts/docker/ign_plugin-compilation.bash index 5c722e1a7..f6362e131 100644 --- a/jenkins-scripts/docker/ign_plugin-compilation.bash +++ b/jenkins-scripts/docker/ign_plugin-compilation.bash @@ -15,23 +15,23 @@ if [[ -z ${DISTRO} ]]; then fi export BUILDING_SOFTWARE_DIRECTORY="ign-plugin" -export BUILDING_PKG_DEPENDENCIES_VAR_NAME="IGN_PLUGIN_DEPENDENCIES" +export BUILDING_PKG_DEPENDENCIES_VAR_NAME="GZ_PLUGIN_DEPENDENCIES" -# Identify IGN_PLUGIN_MAJOR_VERSION to help with dependency resolution -IGN_PLUGIN_MAJOR_VERSION=$(\ +# Identify GZ_PLUGIN_MAJOR_VERSION to help with dependency resolution +GZ_PLUGIN_MAJOR_VERSION=$(\ python3 ${SCRIPT_DIR}/../tools/detect_cmake_major_version.py \ ${WORKSPACE}/ign-plugin/CMakeLists.txt) -# Check IGN_PLUGIN version is integer -if ! [[ ${IGN_PLUGIN_MAJOR_VERSION} =~ ^-?[0-9]+$ ]]; then - echo "Error! IGN_PLUGIN_MAJOR_VERSION is not an integer, check the detection" +# Check GZ_PLUGIN version is integer +if ! [[ ${GZ_PLUGIN_MAJOR_VERSION} =~ ^-?[0-9]+$ ]]; then + echo "Error! GZ_PLUGIN_MAJOR_VERSION is not an integer, check the detection" exit -1 fi -if [[ ${IGN_PLUGIN_MAJOR_VERSION} -ge 6 ]]; then +if [[ ${GZ_PLUGIN_MAJOR_VERSION} -ge 6 ]]; then export NEED_C17_COMPILER=true fi -export GZDEV_PROJECT_NAME="gz-plugin${IGN_PLUGIN_MAJOR_VERSION}" +export GZDEV_PROJECT_NAME="gz-plugin${GZ_PLUGIN_MAJOR_VERSION}" . ${SCRIPT_DIR}/lib/generic-building-base.bash diff --git a/jenkins-scripts/docker/ign_rendering-compilation.bash b/jenkins-scripts/docker/ign_rendering-compilation.bash index bf5cde07f..a350093b2 100644 --- a/jenkins-scripts/docker/ign_rendering-compilation.bash +++ b/jenkins-scripts/docker/ign_rendering-compilation.bash @@ -15,25 +15,25 @@ if [[ -z ${DISTRO} ]]; then fi export BUILDING_SOFTWARE_DIRECTORY="ign-rendering" -export BUILDING_PKG_DEPENDENCIES_VAR_NAME="IGN_RENDERING_DEPENDENCIES" +export BUILDING_PKG_DEPENDENCIES_VAR_NAME="GZ_RENDERING_DEPENDENCIES" -# Identify IGN_RENDERING_MAJOR_VERSION to help with dependency resolution -IGN_RENDERING_MAJOR_VERSION=$(\ +# Identify GZ_RENDERING_MAJOR_VERSION to help with dependency resolution +GZ_RENDERING_MAJOR_VERSION=$(\ python3 ${SCRIPT_DIR}/../tools/detect_cmake_major_version.py \ ${WORKSPACE}/ign-rendering/CMakeLists.txt) -# Check IGN_RENDERING version is integer -if ! [[ ${IGN_RENDERING_MAJOR_VERSION} =~ ^-?[0-9]+$ ]]; then - echo "Error! IGN_RENDERING_MAJOR_VERSION is not an integer, check the detection" +# Check GZ_RENDERING version is integer +if ! [[ ${GZ_RENDERING_MAJOR_VERSION} =~ ^-?[0-9]+$ ]]; then + echo "Error! GZ_RENDERING_MAJOR_VERSION is not an integer, check the detection" exit -1 fi -if [[ ${IGN_RENDERING_MAJOR_VERSION} -ge 1 ]]; then +if [[ ${GZ_RENDERING_MAJOR_VERSION} -ge 1 ]]; then export NEED_C17_COMPILER=true fi export GPU_SUPPORT_NEEDED=true -export GZDEV_PROJECT_NAME="gz-rendering${IGN_RENDERING_MAJOR_VERSION}" +export GZDEV_PROJECT_NAME="gz-rendering${GZ_RENDERING_MAJOR_VERSION}" export BUILDING_EXTRA_CMAKE_PARAMS+=" -DSKIP_optix=true" diff --git a/jenkins-scripts/docker/ign_rndf-compilation.bash b/jenkins-scripts/docker/ign_rndf-compilation.bash index af087b81b..6799fc31d 100644 --- a/jenkins-scripts/docker/ign_rndf-compilation.bash +++ b/jenkins-scripts/docker/ign_rndf-compilation.bash @@ -15,7 +15,7 @@ if [[ -z ${DISTRO} ]]; then fi export BUILDING_SOFTWARE_DIRECTORY="ign-rndf" -export BUILDING_PKG_DEPENDENCIES_VAR_NAME="IGN_RNDF_DEPENDENCIES" +export BUILDING_PKG_DEPENDENCIES_VAR_NAME="GZ_RNDF_DEPENDENCIES" export BUILDING_JOB_REPOSITORIES="stable" . ${SCRIPT_DIR}/lib/generic-building-base.bash diff --git a/jenkins-scripts/docker/ign_sensors-compilation.bash b/jenkins-scripts/docker/ign_sensors-compilation.bash index 14623fd11..16c740653 100644 --- a/jenkins-scripts/docker/ign_sensors-compilation.bash +++ b/jenkins-scripts/docker/ign_sensors-compilation.bash @@ -15,22 +15,22 @@ if [[ -z ${DISTRO} ]]; then fi export BUILDING_SOFTWARE_DIRECTORY="ign-sensors" -export BUILDING_PKG_DEPENDENCIES_VAR_NAME="IGN_SENSORS_DEPENDENCIES" +export BUILDING_PKG_DEPENDENCIES_VAR_NAME="GZ_SENSORS_DEPENDENCIES" -# Identify IGN_SENSORS_MAJOR_VERSION to help with dependency resolution -IGN_SENSORS_MAJOR_VERSION=$(\ +# Identify GZ_SENSORS_MAJOR_VERSION to help with dependency resolution +GZ_SENSORS_MAJOR_VERSION=$(\ python3 ${SCRIPT_DIR}/../tools/detect_cmake_major_version.py \ ${WORKSPACE}/ign-sensors/CMakeLists.txt) -# Check IGN_SENSORS version is integer -if ! [[ ${IGN_SENSORS_MAJOR_VERSION} =~ ^-?[0-9]+$ ]]; then - echo "Error! IGN_SENSORS_MAJOR_VERSION is not an integer, check the detection" +# Check GZ_SENSORS version is integer +if ! [[ ${GZ_SENSORS_MAJOR_VERSION} =~ ^-?[0-9]+$ ]]; then + echo "Error! GZ_SENSORS_MAJOR_VERSION is not an integer, check the detection" exit -1 fi export NEED_C17_COMPILER=true export GPU_SUPPORT_NEEDED=true -export GZDEV_PROJECT_NAME="gz-sensors${IGN_SENSORS_MAJOR_VERSION}" +export GZDEV_PROJECT_NAME="gz-sensors${GZ_SENSORS_MAJOR_VERSION}" . ${SCRIPT_DIR}/lib/generic-building-base.bash diff --git a/jenkins-scripts/docker/ign_transport-compilation.bash b/jenkins-scripts/docker/ign_transport-compilation.bash index c7e393f1d..c6152eea9 100644 --- a/jenkins-scripts/docker/ign_transport-compilation.bash +++ b/jenkins-scripts/docker/ign_transport-compilation.bash @@ -15,23 +15,23 @@ if [[ -z ${DISTRO} ]]; then fi export BUILDING_SOFTWARE_DIRECTORY="ign-transport" - export BUILDING_PKG_DEPENDENCIES_VAR_NAME="IGN_TRANSPORT_DEPENDENCIES" + export BUILDING_PKG_DEPENDENCIES_VAR_NAME="GZ_TRANSPORT_DEPENDENCIES" -# Identify IGN_TRANSPORT_MAJOR_VERSION to help with dependency resolution -IGN_TRANSPORT_MAJOR_VERSION=$(\ +# Identify GZ_TRANSPORT_MAJOR_VERSION to help with dependency resolution +GZ_TRANSPORT_MAJOR_VERSION=$(\ python3 ${SCRIPT_DIR}/../tools/detect_cmake_major_version.py \ ${WORKSPACE}/ign-transport/CMakeLists.txt) -# Check IGN_TRANSPORT version is integer -if ! [[ ${IGN_TRANSPORT_MAJOR_VERSION} =~ ^-?[0-9]+$ ]]; then - echo "Error! IGN_TRANSPORT_MAJOR_VERSION is not an integer, check the detection" +# Check GZ_TRANSPORT version is integer +if ! [[ ${GZ_TRANSPORT_MAJOR_VERSION} =~ ^-?[0-9]+$ ]]; then + echo "Error! GZ_TRANSPORT_MAJOR_VERSION is not an integer, check the detection" exit -1 fi -if [[ ${IGN_TRANSPORT_MAJOR_VERSION} -ge 6 ]]; then +if [[ ${GZ_TRANSPORT_MAJOR_VERSION} -ge 6 ]]; then export NEED_C17_COMPILER=true fi -export GZDEV_PROJECT_NAME="gz-transport${IGN_TRANSPORT_MAJOR_VERSION}" +export GZDEV_PROJECT_NAME="gz-transport${GZ_TRANSPORT_MAJOR_VERSION}" . "${SCRIPT_DIR}/lib/generic-building-base.bash" diff --git a/jenkins-scripts/docker/ign_utils-compilation.bash b/jenkins-scripts/docker/ign_utils-compilation.bash index b35263a63..a30db9f35 100644 --- a/jenkins-scripts/docker/ign_utils-compilation.bash +++ b/jenkins-scripts/docker/ign_utils-compilation.bash @@ -15,21 +15,21 @@ if [[ -z ${DISTRO} ]]; then fi export BUILDING_SOFTWARE_DIRECTORY="ign-utils" -export BUILDING_PKG_DEPENDENCIES_VAR_NAME="IGN_UTILS_DEPENDENCIES" +export BUILDING_PKG_DEPENDENCIES_VAR_NAME="GZ_UTILS_DEPENDENCIES" -# Identify IGN_UTILS_MAJOR_VERSION to help with dependency resolution -IGN_UTILS_MAJOR_VERSION=$(\ +# Identify GZ_UTILS_MAJOR_VERSION to help with dependency resolution +GZ_UTILS_MAJOR_VERSION=$(\ python3 ${SCRIPT_DIR}/../tools/detect_cmake_major_version.py \ ${WORKSPACE}/ign-utils/CMakeLists.txt) -# Check IGN_UTILS version is integer -if ! [[ ${IGN_UTILS_MAJOR_VERSION} =~ ^-?[0-9]+$ ]]; then - echo "Error! IGN_UTILS_MAJOR_VERSION is not an integer, check the detection" +# Check GZ_UTILS version is integer +if ! [[ ${GZ_UTILS_MAJOR_VERSION} =~ ^-?[0-9]+$ ]]; then + echo "Error! GZ_UTILS_MAJOR_VERSION is not an integer, check the detection" exit -1 fi export NEED_C17_COMPILER=true -export GZDEV_PROJECT_NAME="gz-utils${IGN_UTILS_MAJOR_VERSION}" +export GZDEV_PROJECT_NAME="gz-utils${GZ_UTILS_MAJOR_VERSION}" . ${SCRIPT_DIR}/lib/generic-building-base.bash diff --git a/jenkins-scripts/docker/ignition-abichecker.bash b/jenkins-scripts/docker/ignition-abichecker.bash index bc9dc7dcd..2dd18646a 100644 --- a/jenkins-scripts/docker/ignition-abichecker.bash +++ b/jenkins-scripts/docker/ignition-abichecker.bash @@ -20,39 +20,39 @@ if [[ -z ${ABI_JOB_SOFTWARE_NAME} ]]; then exit 1 fi -# convert from ign-package to IGN_PACKAGE_DEPENDENCIES -IGN_NAME_PREFIX=$(\ +# convert from ign-package to GZ_PACKAGE_DEPENDENCIES +GZ_NAME_PREFIX=$(\ echo ${ABI_JOB_SOFTWARE_NAME} | tr '[:lower:]-' '[:upper:]_') -ABI_JOB_PKG_DEPENDENCIES_VAR_NAME=${IGN_NAME_PREFIX}_DEPENDENCIES -DART_FROM_PKGS_VAR_NAME=${IGN_NAME_PREFIX}_DART_FROM_PKGS +ABI_JOB_PKG_DEPENDENCIES_VAR_NAME=${GZ_NAME_PREFIX}_DEPENDENCIES +DART_FROM_PKGS_VAR_NAME=${GZ_NAME_PREFIX}_DART_FROM_PKGS -# Identify IGN_MSGS_MAJOR_VERSION to help with dependency resolution -export IGN_NAME_PREFIX_MAJOR_VERSION=$(\ +# Identify GZ_MSGS_MAJOR_VERSION to help with dependency resolution +export GZ_NAME_PREFIX_MAJOR_VERSION=$(\ python3 ${SCRIPT_DIR}/../tools/detect_cmake_major_version.py \ ${WORKSPACE}/${ABI_JOB_SOFTWARE_NAME}/CMakeLists.txt) -export ${IGN_NAME_PREFIX}_MAJOR_VERSION=${IGN_NAME_PREFIX_MAJOR_VERSION} +export ${GZ_NAME_PREFIX}_MAJOR_VERSION=${GZ_NAME_PREFIX_MAJOR_VERSION} # check if NEED_C17_COMPILER should be set if [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-gazebo" ]] || \ [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-physics" ]] || \ [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-sensors" ]] || \ - [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-common" && ${IGN_NAME_PREFIX_MAJOR_VERSION} -ge 3 ]] || \ - [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-gui" && ${IGN_NAME_PREFIX_MAJOR_VERSION} -ge 1 ]] || \ - [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-math" && ${IGN_NAME_PREFIX_MAJOR_VERSION} -ge 6 ]] || \ - [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-msgs" && ${IGN_NAME_PREFIX_MAJOR_VERSION} -ge 3 ]] || \ - [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-plugin" && ${IGN_NAME_PREFIX_MAJOR_VERSION} -ge 1 ]] || \ - [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-rendering" && ${IGN_NAME_PREFIX_MAJOR_VERSION} -ge 1 ]] || \ - [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-transport" && ${IGN_NAME_PREFIX_MAJOR_VERSION} -ge 6 ]] + [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-common" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -ge 3 ]] || \ + [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-gui" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -ge 1 ]] || \ + [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-math" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -ge 6 ]] || \ + [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-msgs" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -ge 3 ]] || \ + [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-plugin" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -ge 1 ]] || \ + [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-rendering" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -ge 1 ]] || \ + [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-transport" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -ge 6 ]] then export NEED_C17_COMPILER=true fi # check if OGRE-2.2 include paths are needed -if [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-gazebo" && ${IGN_NAME_PREFIX_MAJOR_VERSION} -ge 6 ]] || \ - [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-gui" && ${IGN_NAME_PREFIX_MAJOR_VERSION} -ge 6 ]] || \ - [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-launch" && ${IGN_NAME_PREFIX_MAJOR_VERSION} -ge 5 ]] || \ - [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-sensors" && ${IGN_NAME_PREFIX_MAJOR_VERSION} -ge 6 ]] || \ - [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-rendering" && ${IGN_NAME_PREFIX_MAJOR_VERSION} -ge 6 ]] +if [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-gazebo" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -ge 6 ]] || \ + [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-gui" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -ge 6 ]] || \ + [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-launch" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -ge 5 ]] || \ + [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-sensors" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -ge 6 ]] || \ + [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-rendering" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -ge 6 ]] then export EXTRA_INCLUDES=""" @@ -65,6 +65,6 @@ fi export ABI_JOB_REPOS="stable" # set GZDEV_PROJECT_NAME so it can override repos if necessary -export GZDEV_PROJECT_NAME=${ABI_JOB_SOFTWARE_NAME/ign-/ignition-}${IGN_NAME_PREFIX_MAJOR_VERSION} +export GZDEV_PROJECT_NAME=${ABI_JOB_SOFTWARE_NAME/ign-/ignition-}${GZ_NAME_PREFIX_MAJOR_VERSION} . ${SCRIPT_DIR}/lib/generic-abi-base.bash diff --git a/jenkins-scripts/docker/lib/_gazebo_utils.sh b/jenkins-scripts/docker/lib/_gazebo_utils.sh index 1f5059c26..9c3a39468 100644 --- a/jenkins-scripts/docker/lib/_gazebo_utils.sh +++ b/jenkins-scripts/docker/lib/_gazebo_utils.sh @@ -23,7 +23,7 @@ mkdir -p ~/.gazebo/models tar -xf /tmp/master.tar.gz -C ~/.gazebo/models --strip 1 >/dev/null 2>&1 rm /tmp/master.tar.gz""" -IGN_GAZEBO_RUNTIME_TEST=""" +GZ_SIM_RUNTIME_TEST=""" echo '# BEGIN SECTION: test the script' TEST_START=\`date +%s\` timeout --preserve-status 180 ign gazebo -v -r camera_sensor.sdf || true diff --git a/jenkins-scripts/docker/lib/_generic_linux_compilation_build.sh.bash b/jenkins-scripts/docker/lib/_generic_linux_compilation_build.sh.bash index 8576dcb68..9d63185d9 100644 --- a/jenkins-scripts/docker/lib/_generic_linux_compilation_build.sh.bash +++ b/jenkins-scripts/docker/lib/_generic_linux_compilation_build.sh.bash @@ -7,7 +7,7 @@ # - GENERIC_ENABLE_TESTS (optional) [default true] run tests # - BUILDING_EXTRA_CMAKE_PARAMS (optional) extra cmake params # - BUILDING_EXTRA_MAKETEST_PARAMS (optional) extra "make test ARGS=" params -# - BUILD_ (optional) build dependency from source, for example, BUILD_IGN_MATH +# - BUILD_ (optional) build dependency from source, for example, BUILD_GZ_MATH # - _BRANCH (optional [default: master]) branch for BUILD_ if [[ -z ${SOFTWARE_DIR} ]]; then @@ -29,7 +29,7 @@ fi DELIM_HEADER # Process the source build of dependencies if needed -OSRF_DEPS="IGN_CMAKE IGN_UTILS IGN_TOOLS IGN_MATH IGN_MSGS IGN_TRANSPORT IGN_COMMON IGN_FUEL_TOOLS SDFORMAT IGN_PHYSICS IGN_RENDERING IGN_SENSORS IGN_GUI IGN_GAZEBO" +OSRF_DEPS="GZ_CMAKE GZ_UTILS GZ_TOOLS GZ_MATH GZ_MSGS GZ_TRANSPORT GZ_COMMON GZ_FUEL_TOOLS SDFORMAT GZ_PHYSICS GZ_RENDERING GZ_SENSORS GZ_GUI GZ_SIM" OSRF_DEPS_DONE="" for dep_uppercase in $OSRF_DEPS; do dep=`echo $dep_uppercase | tr '[:upper:]' '[:lower:]'` diff --git a/jenkins-scripts/docker/lib/gazebo-base-default.bash b/jenkins-scripts/docker/lib/gazebo-base-default.bash index afbfd8b28..db2441ce8 100644 --- a/jenkins-scripts/docker/lib/gazebo-base-default.bash +++ b/jenkins-scripts/docker/lib/gazebo-base-default.bash @@ -6,13 +6,13 @@ # piece of code to run in the testing section # - GAZEBO_BUILD_$DEP (optional) [default false] # build dependencies from source. -# DEP = SDFORMAT | IGN_MATH | IGN_TRANSPORT | IGN_GUI | IGN_COMMON +# DEP = SDFORMAT | GZ_MATH | GZ_TRANSPORT | GZ_GUI | GZ_COMMON # branch parameter = $DEP_BRANCH #stop on error set -e -GAZEBO_OSRF_DEPS="SDFORMAT IGN_MATH IGN_TRANSPORT IGN_GUI IGN_COMMON" +GAZEBO_OSRF_DEPS="SDFORMAT GZ_MATH GZ_TRANSPORT GZ_GUI GZ_COMMON" . ${SCRIPT_DIR}/lib/_gazebo_version_hook.bash @@ -242,29 +242,29 @@ DEPENDENCY_PKGS="${BASE_DEPENDENCIES} \ ${GAZEBO_EXTRA_DEPENDENCIES} \ ${EXTRA_PACKAGES}" -[[ -z ${GAZEBO_BUILD_IGN_MATH} ]] && GAZEBO_BUILD_IGN_MATH=false -if $GAZEBO_BUILD_IGN_MATH; then - DEPENDENCY_PKGS="${DEPENDENCY_PKGS} ${IGN_MATH_DEPENDENCIES}" +[[ -z ${GAZEBO_BUILD_GZ_MATH} ]] && GAZEBO_BUILD_GZ_MATH=false +if $GAZEBO_BUILD_GZ_MATH; then + DEPENDENCY_PKGS="${DEPENDENCY_PKGS} ${GZ_MATH_DEPENDENCIES}" fi -[[ -z ${GAZEBO_BUILD_IGN_MSGS} ]] && GAZEBO_BUILD_IGN_MSGS=false -if $GAZEBO_BUILD_IGN_MSGS; then - DEPENDENCY_PKGS="${DEPENDENCY_PKGS} ${IGN_MSGS_DEPENDENCIES}" +[[ -z ${GAZEBO_BUILD_GZ_MSGS} ]] && GAZEBO_BUILD_GZ_MSGS=false +if $GAZEBO_BUILD_GZ_MSGS; then + DEPENDENCY_PKGS="${DEPENDENCY_PKGS} ${GZ_MSGS_DEPENDENCIES}" fi -[[ -z ${GAZEBO_BUILD_IGN_TRANSPORT} ]] && GAZEBO_BUILD_IGN_TRANSPORT=false -if $GAZEBO_BUILD_IGN_TRANSPORT; then - DEPENDENCY_PKGS="${DEPENDENCY_PKGS} ${IGN_TRANSPORT_DEPENDENCIES}" +[[ -z ${GAZEBO_BUILD_GZ_TRANSPORT} ]] && GAZEBO_BUILD_GZ_TRANSPORT=false +if $GAZEBO_BUILD_GZ_TRANSPORT; then + DEPENDENCY_PKGS="${DEPENDENCY_PKGS} ${GZ_TRANSPORT_DEPENDENCIES}" fi -[[ -z ${GAZEBO_BUILD_IGN_GUI} ]] && GAZEBO_BUILD_IGN_GUI=false -if $GAZEBO_BUILD_IGN_GUI; then - DEPENDENCY_PKGS="${DEPENDENCY_PKGS} ${IGN_GUI_DEPENDENCIES}" +[[ -z ${GAZEBO_BUILD_GZ_GUI} ]] && GAZEBO_BUILD_GZ_GUI=false +if $GAZEBO_BUILD_GZ_GUI; then + DEPENDENCY_PKGS="${DEPENDENCY_PKGS} ${GZ_GUI_DEPENDENCIES}" fi -[[ -z ${GAZEBO_BUILD_IGN_COMMON} ]] && GAZEBO_BUILD_IGN_COMMON=false -if $GAZEBO_BUILD_IGN_COMMON; then - DEPENDENCY_PKGS="${DEPENDENCY_PKGS} ${IGN_COMMON_DEPENDENCIES}" +[[ -z ${GAZEBO_BUILD_GZ_COMMON} ]] && GAZEBO_BUILD_GZ_COMMON=false +if $GAZEBO_BUILD_GZ_COMMON; then + DEPENDENCY_PKGS="${DEPENDENCY_PKGS} ${GZ_COMMON_DEPENDENCIES}" fi [[ -z ${GAZEBO_BUILD_SDFORMAT} ]] && GAZEBO_BUILD_SDFORMAT=false diff --git a/jenkins-scripts/docker/lib/generic-building-base.bash b/jenkins-scripts/docker/lib/generic-building-base.bash index a2fb1b946..846d0014d 100644 --- a/jenkins-scripts/docker/lib/generic-building-base.bash +++ b/jenkins-scripts/docker/lib/generic-building-base.bash @@ -12,59 +12,59 @@ fi OSRF_REPOS_TO_USE=${BUILDING_JOB_REPOSITORIES} DEPENDENCY_PKGS="${BASE_DEPENDENCIES} ${ARCHIVE_PROJECT_DEPENDECIES} ${BUILDING_DEPENDENCIES}" -[[ -z ${BUILD_IGN_MATH} ]] && BUILD_IGN_MATH=false -if $BUILD_IGN_MATH; then - DEPENDENCY_PKGS="${DEPENDENCY_PKGS} ${IGN_MATH_DEPENDENCIES}" +[[ -z ${BUILD_GZ_MATH} ]] && BUILD_GZ_MATH=false +if $BUILD_GZ_MATH; then + DEPENDENCY_PKGS="${DEPENDENCY_PKGS} ${GZ_MATH_DEPENDENCIES}" fi -[[ -z ${BUILD_IGN_MSGS} ]] && BUILD_IGN_MSGS=false -if $BUILD_IGN_MSGS; then - DEPENDENCY_PKGS="${DEPENDENCY_PKGS} ${IGN_MSGS_DEPENDENCIES}" +[[ -z ${BUILD_GZ_MSGS} ]] && BUILD_GZ_MSGS=false +if $BUILD_GZ_MSGS; then + DEPENDENCY_PKGS="${DEPENDENCY_PKGS} ${GZ_MSGS_DEPENDENCIES}" fi -[[ -z ${BUILD_IGN_TRANSPORT} ]] && BUILD_IGN_TRANSPORT=false -if $BUILD_IGN_TRANSPORT; then - DEPENDENCY_PKGS="${DEPENDENCY_PKGS} ${IGN_TRANSPORT_DEPENDENCIES}" +[[ -z ${BUILD_GZ_TRANSPORT} ]] && BUILD_GZ_TRANSPORT=false +if $BUILD_GZ_TRANSPORT; then + DEPENDENCY_PKGS="${DEPENDENCY_PKGS} ${GZ_TRANSPORT_DEPENDENCIES}" fi -[[ -z ${BUILD_IGN_GAZEBO} ]] && BUILD_IGN_GAZEBO=false -if $BUILD_IGN_GAZEBO; then - DEPENDENCY_PKGS="${DEPENDENCY_PKGS} ${IGN_GAZEBO_DEPENDENCIES}" +[[ -z ${BUILD_GZ_SIM} ]] && BUILD_GZ_SIM=false +if $BUILD_GZ_SIM; then + DEPENDENCY_PKGS="${DEPENDENCY_PKGS} ${GZ_SIM_DEPENDENCIES}" fi -[[ -z ${BUILD_IGN_GUI} ]] && BUILD_IGN_GUI=false -if $BUILD_IGN_GUI; then - DEPENDENCY_PKGS="${DEPENDENCY_PKGS} ${IGN_GUI_DEPENDENCIES}" +[[ -z ${BUILD_GZ_GUI} ]] && BUILD_GZ_GUI=false +if $BUILD_GZ_GUI; then + DEPENDENCY_PKGS="${DEPENDENCY_PKGS} ${GZ_GUI_DEPENDENCIES}" fi -[[ -z ${BUILD_IGN_COMMON} ]] && BUILD_IGN_COMMON=false -if $BUILD_IGN_COMMON; then - DEPENDENCY_PKGS="${DEPENDENCY_PKGS} ${IGN_COMMON_DEPENDENCIES}" +[[ -z ${BUILD_GZ_COMMON} ]] && BUILD_GZ_COMMON=false +if $BUILD_GZ_COMMON; then + DEPENDENCY_PKGS="${DEPENDENCY_PKGS} ${GZ_COMMON_DEPENDENCIES}" fi -[[ -z ${BUILD_IGN_FUEL_TOOLS} ]] && BUILD_IGN_FUEL_TOOLS=false -if $BUILD_IGN_FUEL_TOOLS; then - DEPENDENCY_PKGS="${DEPENDENCY_PKGS} ${IGN_FUEL_TOOLS_DEPENDENCIES}" +[[ -z ${BUILD_GZ_FUEL_TOOLS} ]] && BUILD_GZ_FUEL_TOOLS=false +if $BUILD_GZ_FUEL_TOOLS; then + DEPENDENCY_PKGS="${DEPENDENCY_PKGS} ${GZ_FUEL_TOOLS_DEPENDENCIES}" fi -[[ -z ${BUILD_IGN_SENSORS} ]] && BUILD_IGN_SENSORS=false -if $BUILD_IGN_SENSORS; then - DEPENDENCY_PKGS="${DEPENDENCY_PKGS} ${IGN_SENSORS_DEPENDENCIES}" +[[ -z ${BUILD_GZ_SENSORS} ]] && BUILD_GZ_SENSORS=false +if $BUILD_GZ_SENSORS; then + DEPENDENCY_PKGS="${DEPENDENCY_PKGS} ${GZ_SENSORS_DEPENDENCIES}" fi -[[ -z ${BUILD_IGN_PHYSICS} ]] && BUILD_IGN_PHYSICS=false -if $BUILD_IGN_PHYSICS; then - DEPENDENCY_PKGS="${DEPENDENCY_PKGS} ${IGN_PHYSICS_DEPENDENCIES}" +[[ -z ${BUILD_GZ_PHYSICS} ]] && BUILD_GZ_PHYSICS=false +if $BUILD_GZ_PHYSICS; then + DEPENDENCY_PKGS="${DEPENDENCY_PKGS} ${GZ_PHYSICS_DEPENDENCIES}" fi -[[ -z ${BUILD_IGN_RENDERING} ]] && BUILD_IGN_RENDERING=false -if $BUILD_IGN_RENDERING; then - DEPENDENCY_PKGS="${DEPENDENCY_PKGS} ${IGN_RENDERING_DEPENDENCIES}" +[[ -z ${BUILD_GZ_RENDERING} ]] && BUILD_GZ_RENDERING=false +if $BUILD_GZ_RENDERING; then + DEPENDENCY_PKGS="${DEPENDENCY_PKGS} ${GZ_RENDERING_DEPENDENCIES}" fi -[[ -z ${BUILD_IGN_TOOLS} ]] && BUILD_IGN_TOOLS=false -if $BUILD_IGN_TOOLS; then - DEPENDENCY_PKGS="${DEPENDENCY_PKGS} ${IGN_TOOLS_DEPENDENCIES}" +[[ -z ${BUILD_GZ_TOOLS} ]] && BUILD_GZ_TOOLS=false +if $BUILD_GZ_TOOLS; then + DEPENDENCY_PKGS="${DEPENDENCY_PKGS} ${GZ_TOOLS_DEPENDENCIES}" fi [[ -z ${BUILD_SDFORMAT} ]] && BUILD_SDFORMAT=false @@ -72,9 +72,9 @@ if $BUILD_SDFORMAT; then DEPENDENCY_PKGS="${DEPENDENCY_PKGS} ${SDFORMAT_BASE_DEPENDENCIES}" fi -[[ -z ${BUILD_IGN_UTILS} ]] && BUILD_IGN_UTILS=false -if $BUILD_IGN_UTILS; then - DEPENDENCY_PKGS="${DEPENDENCY_PKGS} ${IGN_UTILS_BASE_DEPENDENCIES}" +[[ -z ${BUILD_GZ_UTILS} ]] && BUILD_GZ_UTILS=false +if $BUILD_GZ_UTILS; then + DEPENDENCY_PKGS="${DEPENDENCY_PKGS} ${GZ_UTILS_BASE_DEPENDENCIES}" fi SOFTWARE_DIR="${BUILDING_SOFTWARE_DIRECTORY}" diff --git a/jenkins-scripts/dsl/gazebo.dsl b/jenkins-scripts/dsl/gazebo.dsl index 336fa2d6e..d947b587a 100644 --- a/jenkins-scripts/dsl/gazebo.dsl +++ b/jenkins-scripts/dsl/gazebo.dsl @@ -216,8 +216,8 @@ ci_distro.each { distro -> export ARCH=${arch} export GPU_SUPPORT_NEEDED=true export GAZEBO_BUILD_SDFORMAT=true - export GAZEBO_BUILD_IGN_MATH=true - export GAZEBO_BUILD_IGN_TRANSPORT=true + export GAZEBO_BUILD_GZ_MATH=true + export GAZEBO_BUILD_GZ_TRANSPORT=true /bin/bash -xe ./scripts/jenkins-scripts/docker/gazebo-compilation.bash """.stripIndent()) } diff --git a/jenkins-scripts/dsl/sdformat.dsl b/jenkins-scripts/dsl/sdformat.dsl index d6d975816..33507ff5e 100644 --- a/jenkins-scripts/dsl/sdformat.dsl +++ b/jenkins-scripts/dsl/sdformat.dsl @@ -381,7 +381,7 @@ all_versions.each { version -> steps { batchFile("""\ - set USE_IGNITION_ZIP=FALSE + set USE_GZ_ZIP=FALSE call "./scripts/jenkins-scripts/sdformat-default-devel-windows-amd64.bat" """.stripIndent()) } diff --git a/jenkins-scripts/ign_common-default-devel-windows-amd64.bat b/jenkins-scripts/ign_common-default-devel-windows-amd64.bat index c6f6b31aa..39f0ecae9 100644 --- a/jenkins-scripts/ign_common-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_common-default-devel-windows-amd64.bat @@ -5,12 +5,12 @@ set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true set DEPEN_PKGS=assimp ffmpeg freeimage gts tinyxml2 -for /f %%i in ('python "%SCRIPT_DIR%\tools\detect_cmake_major_version.py" "%WORKSPACE%\%VCS_DIRECTORY%\CMakeLists.txt"') do set IGN_MAJOR_VERSION=%%i -if %IGN_MAJOR_VERSION% GEQ 5 ( +for /f %%i in ('python "%SCRIPT_DIR%\tools\detect_cmake_major_version.py" "%WORKSPACE%\%VCS_DIRECTORY%\CMakeLists.txt"') do set GZ_MAJOR_VERSION=%%i +if %GZ_MAJOR_VERSION% GEQ 5 ( set DEPEN_PKGS=%DEPEN_PKGS% gdal ) -set COLCON_PACKAGE=ignition-common +set COLCON_PACKAGE=gz-common set COLCON_AUTO_MAJOR_VERSION=true call "%SCRIPT_DIR%\lib\colcon-default-devel-windows.bat" diff --git a/jenkins-scripts/ign_fuel-tools-default-devel-windows-amd64.bat b/jenkins-scripts/ign_fuel-tools-default-devel-windows-amd64.bat index 0021cc5b7..36c81fee0 100644 --- a/jenkins-scripts/ign_fuel-tools-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_fuel-tools-default-devel-windows-amd64.bat @@ -7,7 +7,7 @@ set IGN_CLEAN_WORKSPACE=true :: tinyxml2 from msgs set DEPEN_PKGS=libyaml libzip tinyxml2 openssl protobuf curl -set COLCON_PACKAGE=ignition-fuel_tools +set COLCON_PACKAGE=gz-fuel_tools set COLCON_AUTO_MAJOR_VERSION=true call "%SCRIPT_DIR%\lib\colcon-default-devel-windows.bat" \ No newline at end of file diff --git a/jenkins-scripts/ign_gazebo-default-devel-windows-amd64.bat b/jenkins-scripts/ign_gazebo-default-devel-windows-amd64.bat index 210551130..2b057d01d 100644 --- a/jenkins-scripts/ign_gazebo-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_gazebo-default-devel-windows-amd64.bat @@ -6,12 +6,12 @@ set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true :: dlfcn set DEPEN_PKGS=assimp boost-core ccd cppzmq cuda curl dlfcn-win32 eigen3 fcl ffmpeg freeimage gflags glib gts jsoncpp libyaml libzip ogre ogre2 ogre22 openssl protobuf qt5 qwt tinyxml2 zeromq -for /f %%i in ('python "%SCRIPT_DIR%\tools\detect_cmake_major_version.py" "%WORKSPACE%\%VCS_DIRECTORY%\CMakeLists.txt"') do set IGN_MAJOR_VERSION=%%i -if %IGN_MAJOR_VERSION% GEQ 7 ( +for /f %%i in ('python "%SCRIPT_DIR%\tools\detect_cmake_major_version.py" "%WORKSPACE%\%VCS_DIRECTORY%\CMakeLists.txt"') do set GZ_MAJOR_VERSION=%%i +if %GZ_MAJOR_VERSION% GEQ 7 ( set DEPEN_PKGS=%DEPEN_PKGS% gdal ) :: This needs to be migrated to DSL to get multi-major versions correctly -set COLCON_PACKAGE=ignition-gazebo +set COLCON_PACKAGE=gz-sim set COLCON_AUTO_MAJOR_VERSION=true call "%SCRIPT_DIR%\lib\colcon-default-devel-windows.bat" diff --git a/jenkins-scripts/ign_gui-default-devel-windows-amd64.bat b/jenkins-scripts/ign_gui-default-devel-windows-amd64.bat index 3ab72a3bb..31707055f 100644 --- a/jenkins-scripts/ign_gui-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_gui-default-devel-windows-amd64.bat @@ -7,12 +7,12 @@ set IGN_CLEAN_WORKSPACE=true :: ogre2 from vcpkg-ports set DEPEN_PKGS=qt5 qt5-winextras qwt protobuf tinyxml2 freeimage ogre ogre22 -for /f %%i in ('python "%SCRIPT_DIR%\tools\detect_cmake_major_version.py" "%WORKSPACE%\%VCS_DIRECTORY%\CMakeLists.txt"') do set IGN_MAJOR_VERSION=%%i -if %IGN_MAJOR_VERSION% GEQ 7 ( +for /f %%i in ('python "%SCRIPT_DIR%\tools\detect_cmake_major_version.py" "%WORKSPACE%\%VCS_DIRECTORY%\CMakeLists.txt"') do set GZ_MAJOR_VERSION=%%i +if %GZ_MAJOR_VERSION% GEQ 7 ( set DEPEN_PKGS=%DEPEN_PKGS% gdal ) :: This needs to be migrated to DSL to get multi-major versions correctly -set COLCON_PACKAGE=ignition-gui +set COLCON_PACKAGE=gz-gui set COLCON_AUTO_MAJOR_VERSION=true call "%SCRIPT_DIR%\lib\colcon-default-devel-windows.bat" diff --git a/jenkins-scripts/ign_launch-default-devel-windows-amd64.bat b/jenkins-scripts/ign_launch-default-devel-windows-amd64.bat index 823e281fb..19b8edbe2 100644 --- a/jenkins-scripts/ign_launch-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_launch-default-devel-windows-amd64.bat @@ -5,11 +5,11 @@ set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true set DEPEN_PKGS=assimp boost-core ccd cppzmq cuda curl dlfcn-win32 eigen3 fcl ffmpeg freeimage gflags glib gts jsoncpp libyaml libzip ogre ogre2 ogre22 openssl protobuf qt5 qwt tinyxml2 zeromq -for /f %%i in ('python "%SCRIPT_DIR%\tools\detect_cmake_major_version.py" "%WORKSPACE%\%VCS_DIRECTORY%\CMakeLists.txt"') do set IGN_MAJOR_VERSION=%%i -if %IGN_MAJOR_VERSION% GEQ 6 ( +for /f %%i in ('python "%SCRIPT_DIR%\tools\detect_cmake_major_version.py" "%WORKSPACE%\%VCS_DIRECTORY%\CMakeLists.txt"') do set GZ_MAJOR_VERSION=%%i +if %GZ_MAJOR_VERSION% GEQ 6 ( set DEPEN_PKGS=%DEPEN_PKGS% gdal ) -set COLCON_PACKAGE=ignition-launch +set COLCON_PACKAGE=gz-launch set COLCON_AUTO_MAJOR_VERSION=true call "%SCRIPT_DIR%\lib\colcon-default-devel-windows.bat" diff --git a/jenkins-scripts/ign_math-default-devel-windows-amd64.bat b/jenkins-scripts/ign_math-default-devel-windows-amd64.bat index 91b5315c7..0c383bf0a 100644 --- a/jenkins-scripts/ign_math-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_math-default-devel-windows-amd64.bat @@ -3,7 +3,7 @@ set SCRIPT_DIR=%~dp0 set VCS_DIRECTORY=gz-math set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true -set COLCON_PACKAGE=ignition-math +set COLCON_PACKAGE=gz-math set COLCON_AUTO_MAJOR_VERSION=true call "%SCRIPT_DIR%\lib\colcon-default-devel-windows.bat" diff --git a/jenkins-scripts/ign_msgs-default-devel-windows-amd64.bat b/jenkins-scripts/ign_msgs-default-devel-windows-amd64.bat index 9fbdedce2..2c1387f75 100644 --- a/jenkins-scripts/ign_msgs-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_msgs-default-devel-windows-amd64.bat @@ -5,7 +5,7 @@ set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true set DEPEN_PKGS=protobuf tinyxml2 -set COLCON_PACKAGE=ignition-msgs +set COLCON_PACKAGE=gz-msgs set COLCON_AUTO_MAJOR_VERSION=true call "%SCRIPT_DIR%\lib\colcon-default-devel-windows.bat" diff --git a/jenkins-scripts/ign_physics-default-devel-windows-amd64.bat b/jenkins-scripts/ign_physics-default-devel-windows-amd64.bat index c1e9ec62e..529d41c83 100644 --- a/jenkins-scripts/ign_physics-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_physics-default-devel-windows-amd64.bat @@ -5,12 +5,12 @@ set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true set DEPEN_PKGS=eigen3 tinyxml2 -for /f %%i in ('python "%SCRIPT_DIR%\tools\detect_cmake_major_version.py" "%WORKSPACE%\%VCS_DIRECTORY%\CMakeLists.txt"') do set IGN_MAJOR_VERSION=%%i -if %IGN_MAJOR_VERSION% GEQ 6 ( +for /f %%i in ('python "%SCRIPT_DIR%\tools\detect_cmake_major_version.py" "%WORKSPACE%\%VCS_DIRECTORY%\CMakeLists.txt"') do set GZ_MAJOR_VERSION=%%i +if %GZ_MAJOR_VERSION% GEQ 6 ( set DEPEN_PKGS=%DEPEN_PKGS% gdal ) :: This needs to be migrated to DSL to get multi-major versions correctly -set COLCON_PACKAGE=ignition-physics +set COLCON_PACKAGE=gz-physics set COLCON_AUTO_MAJOR_VERSION=true call "%SCRIPT_DIR%/lib/colcon-default-devel-windows.bat" diff --git a/jenkins-scripts/ign_plugin-default-devel-windows-amd64.bat b/jenkins-scripts/ign_plugin-default-devel-windows-amd64.bat index f90eac5c2..cb06bbcd9 100644 --- a/jenkins-scripts/ign_plugin-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_plugin-default-devel-windows-amd64.bat @@ -6,7 +6,7 @@ set IGN_CLEAN_WORKSPACE=true set DEPEN_PKGS="dlfcn-win32" -set COLCON_PACKAGE=ignition-plugin +set COLCON_PACKAGE=gz-plugin set COLCON_AUTO_MAJOR_VERSION=true call "%SCRIPT_DIR%/lib/colcon-default-devel-windows.bat" diff --git a/jenkins-scripts/ign_rendering-default-devel-windows-amd64.bat b/jenkins-scripts/ign_rendering-default-devel-windows-amd64.bat index bcf0a19aa..c180f5118 100644 --- a/jenkins-scripts/ign_rendering-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_rendering-default-devel-windows-amd64.bat @@ -6,16 +6,16 @@ set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true set DEPEN_PKGS=boost-core cuda dlfcn-win32 eigen3 freeimage glib gts ogre ogre2 -for /f %%i in ('python "%SCRIPT_DIR%\tools\detect_cmake_major_version.py" "%WORKSPACE%\%VCS_DIRECTORY%\CMakeLists.txt"') do set IGN_MAJOR_VERSION=%%i -if %IGN_MAJOR_VERSION% GEQ 5 ( +for /f %%i in ('python "%SCRIPT_DIR%\tools\detect_cmake_major_version.py" "%WORKSPACE%\%VCS_DIRECTORY%\CMakeLists.txt"') do set GZ_MAJOR_VERSION=%%i +if %GZ_MAJOR_VERSION% GEQ 5 ( set DEPEN_PKGS=%DEPEN_PKGS% ogre22 ) -if %IGN_MAJOR_VERSION% GEQ 7 ( +if %GZ_MAJOR_VERSION% GEQ 7 ( set DEPEN_PKGS=%DEPEN_PKGS% gdal ) :: This needs to be migrated to DSL to get multi-major versions correctly -set COLCON_PACKAGE=ignition-rendering +set COLCON_PACKAGE=gz-rendering set COLCON_AUTO_MAJOR_VERSION=true call "%SCRIPT_DIR%\lib\colcon-default-devel-windows.bat" diff --git a/jenkins-scripts/ign_sensors-default-devel-windows-amd64.bat b/jenkins-scripts/ign_sensors-default-devel-windows-amd64.bat index a4b5bede6..206b5ee4e 100644 --- a/jenkins-scripts/ign_sensors-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_sensors-default-devel-windows-amd64.bat @@ -6,12 +6,12 @@ set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true set DEPEN_PKGS=protobuf tinyxml2 -for /f %%i in ('python "%SCRIPT_DIR%\tools\detect_cmake_major_version.py" "%WORKSPACE%\%VCS_DIRECTORY%\CMakeLists.txt"') do set IGN_MAJOR_VERSION=%%i -if %IGN_MAJOR_VERSION% GEQ 7 ( +for /f %%i in ('python "%SCRIPT_DIR%\tools\detect_cmake_major_version.py" "%WORKSPACE%\%VCS_DIRECTORY%\CMakeLists.txt"') do set GZ_MAJOR_VERSION=%%i +if %GZ_MAJOR_VERSION% GEQ 7 ( set DEPEN_PKGS=%DEPEN_PKGS% gdal ) :: This needs to be migrated to DSL to get multi-major versions correctly -set COLCON_PACKAGE=ignition-sensors +set COLCON_PACKAGE=gz-sensors set COLCON_AUTO_MAJOR_VERSION=true call "%SCRIPT_DIR%\lib\colcon-default-devel-windows.bat" diff --git a/jenkins-scripts/ign_tools-default-devel-windows-amd64.bat b/jenkins-scripts/ign_tools-default-devel-windows-amd64.bat index d9b607aeb..89b4adff8 100644 --- a/jenkins-scripts/ign_tools-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_tools-default-devel-windows-amd64.bat @@ -4,7 +4,7 @@ set VCS_DIRECTORY=gz-tools set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true -set COLCON_PACKAGE=ignition-tools +set COLCON_PACKAGE=gz-tools :: override logic @ colcon-default-devel-windows.bat to handle ign-tools1 case on windows setlocal ENABLEDELAYEDEXPANSION @@ -16,14 +16,14 @@ if "!PKG_MAJOR_VERSION!" == "1" ( set COLCON_PACKAGE=%COLCON_PACKAGE%!PKG_MAJOR_VERSION! ) -echo # BEGIN SECTION: Update package !COLCON_PACKAGE! from ignition to gz if needed +echo # BEGIN SECTION: Update package !COLCON_PACKAGE! from gz to ignition echo Packages in workspace: colcon list --names-only colcon list --names-only | find "!COLCON_PACKAGE!" if errorlevel 1 ( - set COLCON_PACKAGE=%COLCON_PACKAGE:ignition=gz% - set COLCON_PACKAGE=!COLCON_PACKAGE:gazebo=sim! + set COLCON_PACKAGE=!COLCON_PACKAGE:gz=ignition! + set COLCON_PACKAGE=!COLCON_PACKAGE:sim=gazebo! ) colcon list --names-only | find "!COLCON_PACKAGE!" if errorlevel 1 ( diff --git a/jenkins-scripts/ign_transport-default-devel-windows-amd64.bat b/jenkins-scripts/ign_transport-default-devel-windows-amd64.bat index a1cc75ed6..1f2142f30 100644 --- a/jenkins-scripts/ign_transport-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_transport-default-devel-windows-amd64.bat @@ -7,7 +7,7 @@ set PLATFORM_TO_BUILD=amd64 set IGN_CLEAN_WORKSPACE=true set DEPEN_PKGS=cppzmq dlfcn-win32 protobuf sqlite3 tinyxml2 zeromq -set COLCON_PACKAGE=ignition-transport +set COLCON_PACKAGE=gz-transport set COLCON_AUTO_MAJOR_VERSION=true call "%SCRIPT_DIR%\lib\colcon-default-devel-windows.bat" diff --git a/jenkins-scripts/ign_utils-default-devel-windows-amd64.bat b/jenkins-scripts/ign_utils-default-devel-windows-amd64.bat index 6a7acb02e..8da64ffcb 100644 --- a/jenkins-scripts/ign_utils-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_utils-default-devel-windows-amd64.bat @@ -9,7 +9,7 @@ set IGN_CLEAN_WORKSPACE=true :: set DEPEN_PKGS= :: This needs to be migrated to DSL to get multi-major versions correctly -set COLCON_PACKAGE=ignition-utils +set COLCON_PACKAGE=gz-utils set COLCON_AUTO_MAJOR_VERSION=true call "%SCRIPT_DIR%\lib\colcon-default-devel-windows.bat" diff --git a/jenkins-scripts/lib/colcon-default-devel-windows.bat b/jenkins-scripts/lib/colcon-default-devel-windows.bat index 7dbdfeb8a..7c2aac7a7 100644 --- a/jenkins-scripts/lib/colcon-default-devel-windows.bat +++ b/jenkins-scripts/lib/colcon-default-devel-windows.bat @@ -36,14 +36,14 @@ if "%COLCON_AUTO_MAJOR_VERSION%" == "true" ( ) :: Check if package is in colcon workspace -echo # BEGIN SECTION: Update package !COLCON_PACKAGE! from ignition to gz if needed +echo # BEGIN SECTION: Update package !COLCON_PACKAGE! from gz to ignition echo Packages in workspace: colcon list --names-only colcon list --names-only | find "!COLCON_PACKAGE!" if errorlevel 1 ( - set COLCON_PACKAGE=!COLCON_PACKAGE:ignition=gz! - set COLCON_PACKAGE=!COLCON_PACKAGE:gazebo=sim! + set COLCON_PACKAGE=!COLCON_PACKAGE:gz=ignition! + set COLCON_PACKAGE=!COLCON_PACKAGE:sim=gazebo! ) colcon list --names-only | find "!COLCON_PACKAGE!" if errorlevel 1 ( diff --git a/jenkins-scripts/lib/dependencies_archive.sh b/jenkins-scripts/lib/dependencies_archive.sh index 6d0531ec9..0d0e25065 100644 --- a/jenkins-scripts/lib/dependencies_archive.sh +++ b/jenkins-scripts/lib/dependencies_archive.sh @@ -55,18 +55,18 @@ BREW_BASE_DEPENDCIES="git cmake" # 1. SDFORMAT # ruby for xml_schemas generation and libxml2-utils for xmllint used in tests -SDFORMAT_NO_IGN_DEPENDENCIES="${pythonv} \\ +SDFORMAT_NO_GZ_DEPENDENCIES="${pythonv} \\ libxml2-utils \\ libtinyxml-dev" # SDFORMAT 10 and above use tinyxml2 if [[ ${SDFORMAT_MAJOR_VERSION} -ge 10 ]]; then - SDFORMAT_NO_IGN_DEPENDENCIES="${SDFORMAT_NO_IGN_DEPENDENCIES} \\ + SDFORMAT_NO_GZ_DEPENDENCIES="${SDFORMAT_NO_GZ_DEPENDENCIES} \\ libtinyxml2-dev" fi if [[ ${SDFORMAT_MAJOR_VERSION} -lt 8 ]]; then -SDFORMAT_NO_IGN_DEPENDENCIES="${SDFORMAT_NO_IGN_DEPENDENCIES} \\ +SDFORMAT_NO_GZ_DEPENDENCIES="${SDFORMAT_NO_GZ_DEPENDENCIES} \\ libboost-system-dev \\ libboost-filesystem-dev \\ libboost-program-options-dev \\ @@ -74,7 +74,7 @@ SDFORMAT_NO_IGN_DEPENDENCIES="${SDFORMAT_NO_IGN_DEPENDENCIES} \\ libboost-iostreams-dev" fi -SDFORMAT_NO_IGN_DEPENDENCIES="${SDFORMAT_NO_IGN_DEPENDENCIES} \\ +SDFORMAT_NO_GZ_DEPENDENCIES="${SDFORMAT_NO_GZ_DEPENDENCIES} \\ ruby-dev \\ ruby" @@ -86,17 +86,17 @@ fi if [[ ${SDFORMAT_MAJOR_VERSION} -ge 9 && ${SDFORMAT_MAJOR_VERSION} -lt 13 ]]; then # sdformat9 requires ignition-math6 and # uses ignition-tools for a test - SDFORMAT_BASE_DEPENDENCIES="${SDFORMAT_NO_IGN_DEPENDENCIES} \\ + SDFORMAT_BASE_DEPENDENCIES="${SDFORMAT_NO_GZ_DEPENDENCIES} \\ libignition-math6-dev \\ libignition-tools-dev" elif [[ ${SDFORMAT_MAJOR_VERSION} -eq 6 ]]; then # sdformat6 requires ignition-math4 and # uses ignition-tools for a test - SDFORMAT_BASE_DEPENDENCIES="${SDFORMAT_NO_IGN_DEPENDENCIES} \\ + SDFORMAT_BASE_DEPENDENCIES="${SDFORMAT_NO_GZ_DEPENDENCIES} \\ libignition-math4-dev \\ libignition-tools-dev" else - SDFORMAT_BASE_DEPENDENCIES="${SDFORMAT_NO_IGN_DEPENDENCIES}" + SDFORMAT_BASE_DEPENDENCIES="${SDFORMAT_NO_GZ_DEPENDENCIES}" fi # GAZEBO related dependencies. Default value points to the development version @@ -392,15 +392,15 @@ fi # # Completely rely on packages.apt from Garden -if [[ ${IGN_MATH_MAJOR_VERSION} -lt 7 ]]; then - if [[ ${IGN_MATH_MAJOR_VERSION} -eq 4 ]]; then - IGN_MATH_DEPENDENCIES="libeigen3-dev \\ +if [[ ${GZ_MATH_MAJOR_VERSION} -lt 7 ]]; then + if [[ ${GZ_MATH_MAJOR_VERSION} -eq 4 ]]; then + GZ_MATH_DEPENDENCIES="libeigen3-dev \\ libpython3-dev \\ ruby-dev \\ swig \\ libignition-cmake-dev" else - IGN_MATH_DEPENDENCIES="libeigen3-dev \\ + GZ_MATH_DEPENDENCIES="libeigen3-dev \\ libpython3-dev \\ ruby-dev \\ swig \\ @@ -410,13 +410,13 @@ fi # IGN_TRANSPORT related dependencies. Default value points to the development # version -if [[ -z ${IGN_TRANSPORT_MAJOR_VERSION} ]]; then - IGN_TRANSPORT_MAJOR_VERSION=5 +if [[ -z ${GZ_TRANSPORT_MAJOR_VERSION} ]]; then + GZ_TRANSPORT_MAJOR_VERSION=5 fi # Completely rely on packages.apt from Garden -if [[ ${IGN_TRANSPORT_MAJOR_VERSION} -lt 12 ]]; then - IGN_TRANSPORT_NO_IGN_DEPENDENCIES="pkg-config \\ +if [[ ${GZ_TRANSPORT_MAJOR_VERSION} -lt 12 ]]; then + GZ_TRANSPORT_NO_GZ_DEPENDENCIES="pkg-config \\ ${pythonv} \\ ruby-ronn \\ libprotoc-dev \\ @@ -425,23 +425,23 @@ if [[ ${IGN_TRANSPORT_MAJOR_VERSION} -lt 12 ]]; then uuid-dev \\ libzmq3-dev \\ libczmq-dev" - if [[ ${IGN_TRANSPORT_MAJOR_VERSION} -eq 4 ]]; then - export IGN_TRANSPORT_DEPENDENCIES="${IGN_TRANSPORT_NO_IGN_DEPENDENCIES} \\ + if [[ ${GZ_TRANSPORT_MAJOR_VERSION} -eq 4 ]]; then + export GZ_TRANSPORT_DEPENDENCIES="${GZ_TRANSPORT_NO_GZ_DEPENDENCIES} \\ libignition-cmake-dev \\ libignition-msgs-dev" - elif [[ ${IGN_TRANSPORT_MAJOR_VERSION} -ge 8 ]]; then - export IGN_TRANSPORT_DEPENDENCIES="${IGN_TRANSPORT_NO_IGN_DEPENDENCIES} \\ + elif [[ ${GZ_TRANSPORT_MAJOR_VERSION} -ge 8 ]]; then + export GZ_TRANSPORT_DEPENDENCIES="${GZ_TRANSPORT_NO_GZ_DEPENDENCIES} \\ libignition-cmake2-dev \\ libignition-msgs5-dev \\ libsqlite3-dev \\ ruby-ffi" fi - export IGN_TRANSPORT_DEPENDENCIES="${IGN_TRANSPORT_DEPENDENCIES} libignition-tools-dev" + export GZ_TRANSPORT_DEPENDENCIES="${GZ_TRANSPORT_DEPENDENCIES} libignition-tools-dev" fi # Completely rely on packages.apt from Garden -if [[ ${IGN_COMMON_MAJOR_VERSION} -lt 5 ]]; then - IGN_COMMON_NO_IGN_DEPENDENCIES="pkg-config \\ +if [[ ${GZ_COMMON_MAJOR_VERSION} -lt 5 ]]; then + GZ_COMMON_NO_GZ_DEPENDENCIES="pkg-config \\ ${pythonv} \\ ruby-ronn \\ uuid-dev \\ @@ -455,7 +455,7 @@ if [[ ${IGN_COMMON_MAJOR_VERSION} -lt 5 ]]; then libtinyxml2-dev \\ uuid-dev" - IGN_COMMON_DEPENDENCIES="${IGN_COMMON_NO_IGN_DEPENDENCIES} \\ + GZ_COMMON_DEPENDENCIES="${GZ_COMMON_NO_GZ_DEPENDENCIES} \\ libignition-cmake-dev \\ libignition-cmake2-dev \\ libignition-math4-dev \\ @@ -463,8 +463,8 @@ if [[ ${IGN_COMMON_MAJOR_VERSION} -lt 5 ]]; then fi # Completely rely on packages.apt from Garden -if [[ ${IGN_FUEL_TOOLS_MAJOR_VERSION} -lt 8 ]]; then - IGN_FUEL_TOOLS_DEPENDENCIES="libignition-cmake2-dev \\ +if [[ ${GZ_FUEL_TOOLS_MAJOR_VERSION} -lt 8 ]]; then + GZ_FUEL_TOOLS_DEPENDENCIES="libignition-cmake2-dev \\ libignition-common3-dev \\ libignition-tools-dev \\ libcurl4-openssl-dev \\ @@ -472,26 +472,26 @@ if [[ ${IGN_FUEL_TOOLS_MAJOR_VERSION} -lt 8 ]]; then libyaml-dev \\ libzip-dev" - if [[ ${IGN_FUEL_TOOLS_MAJOR_VERSION} -eq 1 ]]; then - IGN_FUEL_TOOLS_DEPENDENCIES="${IGN_FUEL_TOOLS_DEPENDENCIES} \\ + if [[ ${GZ_FUEL_TOOLS_MAJOR_VERSION} -eq 1 ]]; then + GZ_FUEL_TOOLS_DEPENDENCIES="${GZ_FUEL_TOOLS_DEPENDENCIES} \\ libignition-cmake-dev \\ libignition-common-dev" else - IGN_FUEL_TOOLS_DEPENDENCIES="${IGN_FUEL_TOOLS_DEPENDENCIES} \\ + GZ_FUEL_TOOLS_DEPENDENCIES="${GZ_FUEL_TOOLS_DEPENDENCIES} \\ libignition-cmake2-dev \\ libignition-common3-dev \\ libtinyxml2-dev" fi - if [[ ${IGN_FUEL_TOOLS_MAJOR_VERSION} -ge 4 ]]; then - IGN_FUEL_TOOLS_DEPENDENCIES="${IGN_FUEL_TOOLS_DEPENDENCIES} \\ + if [[ ${GZ_FUEL_TOOLS_MAJOR_VERSION} -ge 4 ]]; then + GZ_FUEL_TOOLS_DEPENDENCIES="${GZ_FUEL_TOOLS_DEPENDENCIES} \\ libignition-msgs5-dev" fi fi # Completely rely on packages.apt from Garden -if [[ ${IGN_MSGS_MAJOR_VERSION} -lt 9 ]]; then - IGN_MSGS_DEPENDENCIES="libignition-tools-dev \\ +if [[ ${GZ_MSGS_MAJOR_VERSION} -lt 9 ]]; then + GZ_MSGS_DEPENDENCIES="libignition-tools-dev \\ libprotobuf-dev \\ libprotoc-dev \\ protobuf-compiler \\ @@ -499,20 +499,20 @@ if [[ ${IGN_MSGS_MAJOR_VERSION} -lt 9 ]]; then ruby \\ ruby-dev" - if [[ -n ${IGN_MSGS_MAJOR_VERSION} && ${IGN_MSGS_MAJOR_VERSION} -eq 1 ]]; then - IGN_MSGS_DEPENDENCIES="${IGN_MSGS_DEPENDENCIES} \\ + if [[ -n ${GZ_MSGS_MAJOR_VERSION} && ${GZ_MSGS_MAJOR_VERSION} -eq 1 ]]; then + GZ_MSGS_DEPENDENCIES="${GZ_MSGS_DEPENDENCIES} \\ libignition-cmake-dev \\ libignition-math4-dev" - elif [[ -n ${IGN_MSGS_MAJOR_VERSION} && ${IGN_MSGS_MAJOR_VERSION} -ge 5 ]]; then - IGN_MSGS_DEPENDENCIES="${IGN_MSGS_DEPENDENCIES} \\ + elif [[ -n ${GZ_MSGS_MAJOR_VERSION} && ${GZ_MSGS_MAJOR_VERSION} -ge 5 ]]; then + GZ_MSGS_DEPENDENCIES="${GZ_MSGS_DEPENDENCIES} \\ libignition-cmake2-dev \\ libignition-math6-dev" fi fi # Completely rely on packages.apt from Garden -if [[ ${IGN_GUI_MAJOR_VERSION} -lt 7 ]]; then - IGN_GUI_NO_IGN_DEPENDENCIES="qtbase5-dev \\ +if [[ ${GZ_GUI_MAJOR_VERSION} -lt 7 ]]; then + GZ_GUI_NO_GZ_DEPENDENCIES="qtbase5-dev \\ qtdeclarative5-dev \\ libtinyxml2-dev \\ libqwt-qt5-dev \\ @@ -530,18 +530,18 @@ if [[ ${IGN_GUI_MAJOR_VERSION} -lt 7 ]]; then qml-module-qtquick-window2 \\ qml-module-qtquick2 \\ qtquickcontrols2-5-dev" - IGN_GUI_DEPENDENCIES="${IGN_GUI_NO_IGN_DEPENDENCIES} \\ + GZ_GUI_DEPENDENCIES="${GZ_GUI_NO_GZ_DEPENDENCIES} \\ libignition-cmake2-dev \\ libignition-common3-dev \\ libignition-math6-dev \\ libignition-plugin-dev \\ libignition-tools-dev" - if [[ -n "${IGN_GUI_MAJOR_VERSION}" && ${IGN_GUI_MAJOR_VERSION} -ge 4 ]]; then - IGN_GUI_DEPENDENCIES="${IGN_GUI_DEPENDENCIES} \\ + if [[ -n "${GZ_GUI_MAJOR_VERSION}" && ${GZ_GUI_MAJOR_VERSION} -ge 4 ]]; then + GZ_GUI_DEPENDENCIES="${GZ_GUI_DEPENDENCIES} \\ libignition-common3-dev" - elif [[ -n "${IGN_GUI_MAJOR_VERSION}" && ${IGN_GUI_MAJOR_VERSION} -eq 3 ]]; then - IGN_GUI_DEPENDENCIES="${IGN_GUI_DEPENDENCIES} \\ + elif [[ -n "${GZ_GUI_MAJOR_VERSION}" && ${GZ_GUI_MAJOR_VERSION} -eq 3 ]]; then + GZ_GUI_DEPENDENCIES="${GZ_GUI_DEPENDENCIES} \\ libignition-common3-dev \\ libignition-msgs5-dev \\ libignition-rendering3-dev \\ @@ -552,13 +552,13 @@ fi IGN_PHYSICS_DART_FROM_PKGS="true" # Completely rely on packages.apt from Garden -if [[ ${IGN_PLUGIN_MAJOR_VERSION} -lt 2 ]]; then - IGN_PLUGIN_DEPENDENCIES="libignition-cmake2-dev" +if [[ ${GZ_PLUGIN_MAJOR_VERSION} -lt 2 ]]; then + GZ_PLUGIN_DEPENDENCIES="libignition-cmake2-dev" fi # Completely rely on packages.apt from Garden -if [[ ${IGN_LAUNCH_MAJOR_VERSION} -lt 6 ]]; then - IGN_LAUNCH_COMMON_DEPENDENCIES="libignition-cmake2-dev \\ +if [[ ${GZ_LAUNCH_MAJOR_VERSION} -lt 6 ]]; then + GZ_LAUNCH_COMMON_DEPENDENCIES="libignition-cmake2-dev \\ libignition-common3-dev \\ libignition-math6-dev \\ libignition-plugin-dev \\ @@ -567,8 +567,8 @@ if [[ ${IGN_LAUNCH_MAJOR_VERSION} -lt 6 ]]; then qtquickcontrols2-5-dev \\ libqt5core5a" - if [[ -n "${IGN_LAUNCH_MAJOR_VERSION}" && ${IGN_LAUNCH_MAJOR_VERSION} -ge 3 ]]; then - IGN_LAUNCH_DEPENDENCIES="${IGN_LAUNCH_COMMON_DEPENDENCIES} \\ + if [[ -n "${GZ_LAUNCH_MAJOR_VERSION}" && ${GZ_LAUNCH_MAJOR_VERSION} -ge 3 ]]; then + GZ_LAUNCH_DEPENDENCIES="${GZ_LAUNCH_COMMON_DEPENDENCIES} \\ libignition-gui4-dev \\ libignition-fuel-tools5-dev \\ libignition-gazebo4-dev \\ @@ -578,7 +578,7 @@ if [[ ${IGN_LAUNCH_MAJOR_VERSION} -lt 6 ]]; then libignition-transport9-dev \\ libsdformat10-dev" else - IGN_LAUNCH_DEPENDENCIES="${IGN_LAUNCH_DEPENDENCIES} \\ + GZ_LAUNCH_DEPENDENCIES="${GZ_LAUNCH_DEPENDENCIES} \\ libignition-gui3-dev \\ libignition-fuel-tools4-dev \\ libignition-gazebo3-dev \\ @@ -593,8 +593,8 @@ if [[ ${IGN_LAUNCH_MAJOR_VERSION} -lt 6 ]]; then fi # Completely rely on packages.apt from Garden -if [[ ${IGN_RENDERING_MAJOR_VERSION} -lt 7 ]]; then - IGN_RENDERING_NO_IGN_DEPENDENCIES="${ogre_pkg}\\ +if [[ ${GZ_RENDERING_MAJOR_VERSION} -lt 7 ]]; then + GZ_RENDERING_NO_GZ_DEPENDENCIES="${ogre_pkg}\\ freeglut3-dev \\ libfreeimage-dev \\ libglew-dev \\ @@ -603,7 +603,7 @@ if [[ ${IGN_RENDERING_MAJOR_VERSION} -lt 7 ]]; then mesa-common-dev \\ mesa-utils" - IGN_RENDERING_DEPENDENCIES="${IGN_RENDERING_NO_IGN_DEPENDENCIES} \\ + GZ_RENDERING_DEPENDENCIES="${GZ_RENDERING_NO_GZ_DEPENDENCIES} \\ libignition-cmake2-dev \\ libignition-common3-dev \\ libignition-plugin-dev \\ @@ -611,21 +611,21 @@ if [[ ${IGN_RENDERING_MAJOR_VERSION} -lt 7 ]]; then fi # Completely rely on packages.apt from Garden -if [[ ${IGN_SENSORS_MAJOR_VERSION} -lt 7 ]]; then - IGN_SENSORS_DEPENDENCIES="libignition-common3-dev \\ +if [[ ${GZ_SENSORS_MAJOR_VERSION} -lt 7 ]]; then + GZ_SENSORS_DEPENDENCIES="libignition-common3-dev \\ libignition-cmake2-dev \\ libignition-math6-dev \\ libignition-plugin-dev \\ libignition-tools-dev" - if [[ -n "${IGN_SENSORS_MAJOR_VERSION}" && ${IGN_SENSORS_MAJOR_VERSION} -ge 4 ]]; then - IGN_SENSORS_DEPENDENCIES="${IGN_SENSORS_DEPENDENCIES} \\ + if [[ -n "${GZ_SENSORS_MAJOR_VERSION}" && ${GZ_SENSORS_MAJOR_VERSION} -ge 4 ]]; then + GZ_SENSORS_DEPENDENCIES="${GZ_SENSORS_DEPENDENCIES} \\ libignition-msgs6-dev \\ libignition-rendering4-dev \\ libignition-transport9-dev \\ libsdformat10-dev" - elif [[ -n "${IGN_SENSORS_MAJOR_VERSION}" && ${IGN_SENSORS_MAJOR_VERSION} -ge 3 ]]; then - IGN_SENSORS_DEPENDENCIES="${IGN_SENSORS_DEPENDENCIES} \\ + elif [[ -n "${GZ_SENSORS_MAJOR_VERSION}" && ${GZ_SENSORS_MAJOR_VERSION} -ge 3 ]]; then + GZ_SENSORS_DEPENDENCIES="${GZ_SENSORS_DEPENDENCIES} \\ libignition-msgs5-dev \\ libignition-rendering3-dev \\ libignition-transport8-dev \\ @@ -634,18 +634,18 @@ if [[ ${IGN_SENSORS_MAJOR_VERSION} -lt 7 ]]; then fi # Completely rely on packages.apt from Garden -if [[ ${IGN_GAZEBO_MAJOR_VERSION} -lt 7 ]]; then - IGN_GAZEBO_DEPENDENCIES="libignition-common3-dev \\ +if [[ ${GZ_SIM_MAJOR_VERSION} -lt 7 ]]; then + GZ_SIM_DEPENDENCIES="libignition-common3-dev \\ libignition-cmake2-dev \\ libgflags-dev \\ libignition-math6-dev \\ libignition-math6-eigen3-dev \\ libbenchmark-dev" - if [[ -n "${IGN_GAZEBO_MAJOR_VERSION}" && ${IGN_GAZEBO_MAJOR_VERSION} -ge 4 ]]; then - IGN_GAZEBO_DEPENDENCIES="${IGN_GAZEBO_DEPENDENCIES} \\ + if [[ -n "${GZ_SIM_MAJOR_VERSION}" && ${GZ_SIM_MAJOR_VERSION} -ge 4 ]]; then + GZ_SIM_DEPENDENCIES="${GZ_SIM_DEPENDENCIES} \\ libsdformat10-dev" - elif [[ -n "${IGN_GAZEBO_MAJOR_VERSION}" && ${IGN_GAZEBO_MAJOR_VERSION} -eq 3 ]]; then - IGN_GAZEBO_DEPENDENCIES="${IGN_GAZEBO_DEPENDENCIES} \\ + elif [[ -n "${GZ_SIM_MAJOR_VERSION}" && ${GZ_SIM_MAJOR_VERSION} -eq 3 ]]; then + GZ_SIM_DEPENDENCIES="${GZ_SIM_DEPENDENCIES} \\ libignition-fuel-tools4-dev \\ libignition-gui3-dev \\ libignition-msgs5-dev \\ @@ -661,6 +661,6 @@ if [[ ${IGN_GAZEBO_MAJOR_VERSION} -lt 7 ]]; then fi # Completely rely on packages.apt from Garden -if [[ ${IGN_UTILS_MAJOR_VERSION} -lt 2 ]]; then - IGN_UTILS_DEPENDENCIES="libignition-cmake2-dev" +if [[ ${GZ_UTILS_MAJOR_VERSION} -lt 2 ]]; then + GZ_UTILS_DEPENDENCIES="libignition-cmake2-dev" fi diff --git a/jenkins-scripts/lib/sdformat-base-windows.bat b/jenkins-scripts/lib/sdformat-base-windows.bat index 9ea8ba71b..a31286de3 100644 --- a/jenkins-scripts/lib/sdformat-base-windows.bat +++ b/jenkins-scripts/lib/sdformat-base-windows.bat @@ -1,7 +1,7 @@ :: sdformat base script :: :: Parameters: -:: - USE_IGNITION_ZIP : (default true) [true | false]. Use zip to install ignition +:: - USE_GZ_ZIP : (default true) [true | false]. Use zip to install ignition :: instead of compile :: - BUILD_TYPE : (default Release) [ Release | Debug ] Build type to use :: @@ -13,7 +13,7 @@ findstr /r /b "find_package(ignition-math" %WORKSPACE%\sdformat\cmake\SearchForS set /p IGN_MATH_REQUIRED_VERSION= install_boost.log -IF %USE_IGNITION_ZIP% == TRUE ( +IF %USE_GZ_ZIP% == TRUE ( call %win_lib% :wget http://packages.osrfoundation.org/win32/deps/%IGNMATH_ZIP%.zip %IGNMATH_ZIP%.zip call %win_lib% :unzip_7za %IGNMATH_ZIP%.zip ) diff --git a/jenkins-scripts/lib/windows_library.bat b/jenkins-scripts/lib/windows_library.bat index 7f5f54d79..1b5b90990 100644 --- a/jenkins-scripts/lib/windows_library.bat +++ b/jenkins-scripts/lib/windows_library.bat @@ -211,21 +211,8 @@ goto :EOF set COLCON_PACKAGE=%1 :: Check if package is in colcon workspace -echo # BEGIN SECTION: Update package %COLCON_PACKAGE% from ignition to gz if needed -echo Packages in workspace: +echo # BEGIN SECTION Packages in workspace: colcon list --names-only - -colcon list --names-only | find "%COLCON_PACKAGE%" -if errorlevel 1 ( - set COLCON_PACKAGE=!COLCON_PACKAGE:ignition=gz! - set COLCON_PACKAGE=!COLCON_PACKAGE:gazebo=sim! -) -colcon list --names-only | find "!COLCON_PACKAGE!" -if errorlevel 1 ( - echo Failed to find package !COLCON_PACKAGE! in workspace. - goto :error -) -echo Using package name !COLCON_PACKAGE! echo # END SECTION :: two runs to get the dependencies built with testing and the package under From 1bd029984cb9aaba54d16fdedddf5b5cd9636d50 Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Thu, 18 Aug 2022 12:40:23 -0700 Subject: [PATCH 194/540] Garden view: use stable branch CI jobs (#793) * Garden view: use some stable branches Signed-off-by: Steve Peters * Use gazebosim/gz- URL Signed-off-by: Steve Peters * Use more stable garden branches Signed-off-by: Steve Peters * Use some windows stable branch jobs Signed-off-by: Steve Peters * Use more windows etable branch jobs Signed-off-by: Steve Peters Signed-off-by: Steve Peters --- jenkins-scripts/dsl/ignition_collection.dsl | 86 ++++++++++----------- 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index 93fa2eabb..de25e5348 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -204,82 +204,82 @@ ignition_collection_jobs = 'sdformat-install-sdformat12_pkg-focal-amd64' ], 'garden' : [ - 'ign_common-ci-win', - 'ign_fuel-tools-ci-win', + 'ign_common-gz-5-win', + 'ign_fuel-tools-gz-8-win', 'ign_gazebo-ci-win', - 'ign_gui-ci-win', - 'ign_launch-ci-win', - 'ign_math-ci-win', - 'ign_msgs-ci-win', - 'ign_physics-ci-win', - 'ign_plugin-ign-1-win', - 'ign_rendering-ci-win', - 'ign_sensors-ci-win', - 'ign_tools-ign-1-win', - 'ign_transport-ci-win', - 'ign_utils-ign-1-win', - 'ignition_cmake-ci-ign-cmake2-focal-amd64', - 'ignition_cmake-ci-ign-cmake2-homebrew-amd64', - 'ignition_cmake-ci-ign-cmake2-windows7-amd64', + 'ign_gui-gz-7-win', + 'ign_launch-gz-6-win', + 'ign_math-gz-7-win', + 'ign_msgs-gz-9-win', + 'ign_physics-gz-6-win', + 'ign_plugin-gz-2-win', + 'ign_rendering-gz-7-win', + 'ign_sensors-gz-7-win', + 'ign_tools-gz-2-win', + 'ign_transport-gz-12-win', + 'ign_utils-gz-2-win', + 'ignition_cmake-ci-gz-cmake3-focal-amd64', + 'ignition_cmake-ci-gz-cmake3-homebrew-amd64', + 'ignition_cmake-ci-gz-cmake3-windows7-amd64', 'ignition_cmake2-install-pkg-focal-amd64', 'ignition_cmake2-install_bottle-homebrew-amd64', - 'ignition_common-ci-main-focal-amd64', - 'ignition_common-ci-main-homebrew-amd64', + 'ignition_common-ci-gz-common5-focal-amd64', + 'ignition_common-ci-gz-common5-homebrew-amd64', 'ignition_common4-install-pkg-focal-amd64', 'ignition_common4-install_bottle-homebrew-amd64', 'ignition_garden-ci-main-homebrew-amd64', 'ignition_garden-install-pkg-focal-amd64', 'ignition_garden-install_bottle-homebrew-amd64', - 'ignition_fuel-tools-ci-main-focal-amd64', - 'ignition_fuel-tools-ci-main-homebrew-amd64', + 'ignition_fuel-tools-ci-gz-fuel-tools8-focal-amd64', + 'ignition_fuel-tools-ci-gz-fuel-tools8-homebrew-amd64', 'ignition_fuel-tools7-install-pkg-focal-amd64', 'ignition_fuel-tools7-install_bottle-homebrew-amd64', 'ignition_gazebo-ci-main-focal-amd64', 'ignition_gazebo-ci-main-homebrew-amd64', 'ignition_gazebo6-install-pkg-focal-amd64', 'ignition_gazebo6-install_bottle-homebrew-amd64', - 'ignition_gui-ci-main-focal-amd64', - 'ignition_gui-ci-main-homebrew-amd64', + 'ignition_gui-ci-gz-gui7-focal-amd64', + 'ignition_gui-ci-gz-gui7-homebrew-amd64', 'ignition_gui6-install-pkg-focal-amd64', 'ignition_gui6-install_bottle-homebrew-amd64', - 'ignition_launch-ci-main-focal-amd64', - 'ignition_launch-ci-main-homebrew-amd64', + 'ignition_launch-ci-gz-launch6-focal-amd64', + 'ignition_launch-ci-gz-launch6-homebrew-amd64', 'ignition_launch5-install-pkg-focal-amd64', 'ignition_launch5-install_bottle-homebrew-amd64', - 'ignition_math-ci-main-focal-amd64', - 'ignition_math-ci-main-homebrew-amd64', + 'ignition_math-ci-gz-math7-focal-amd64', + 'ignition_math-ci-gz-math7-homebrew-amd64', 'ignition_math6-install-pkg-focal-amd64', 'ignition_math6-install_bottle-homebrew-amd64', - 'ignition_msgs-ci-main-focal-amd64', - 'ignition_msgs-ci-main-homebrew-amd64', + 'ignition_msgs-ci-gz-msgs9-focal-amd64', + 'ignition_msgs-ci-gz-msgs9-homebrew-amd64', 'ignition_msgs8-install-pkg-focal-amd64', 'ignition_msgs8-install_bottle-homebrew-amd64', - 'ignition_physics-ci-main-focal-amd64', - 'ignition_physics-ci-main-homebrew-amd64', + 'ignition_physics-ci-gz-physics6-focal-amd64', + 'ignition_physics-ci-gz-physics6-homebrew-amd64', 'ignition_physics5-install-pkg-focal-amd64', 'ignition_physics5-install_bottle-homebrew-amd64', - 'ignition_plugin-ci-ign-plugin1-focal-amd64', - 'ignition_plugin-ci-ign-plugin1-homebrew-amd64', + 'ignition_plugin-ci-gz-plugin2-focal-amd64', + 'ignition_plugin-ci-gz-plugin2-homebrew-amd64', 'ignition_plugin-install-pkg-focal-amd64', 'ignition_plugin1-install_bottle-homebrew-amd64', - 'ignition_rendering-ci-main-focal-amd64', - 'ignition_rendering-ci-main-homebrew-amd64', + 'ignition_rendering-ci-gz-rendering7-focal-amd64', + 'ignition_rendering-ci-gz-rendering7-homebrew-amd64', 'ignition_rendering6-install-pkg-focal-amd64', 'ignition_rendering6-install_bottle-homebrew-amd64', - 'ignition_sensors-ci-main-focal-amd64', - 'ignition_sensors-ci-main-homebrew-amd64', + 'ignition_sensors-ci-gz-sensors7-focal-amd64', + 'ignition_sensors-ci-gz-sensors7-homebrew-amd64', 'ignition_sensors6-install-pkg-focal-amd64', 'ignition_sensors6-install_bottle-homebrew-amd64', - 'ignition_tools-ci-ign-tools1-focal-amd64', - 'ignition_tools-ci-ign-tools1-homebrew-amd64', + 'ignition_tools-ci-gz-tools2-focal-amd64', + 'ignition_tools-ci-gz-tools2-homebrew-amd64', 'ignition_tools-install-pkg-focal-amd64', 'ignition_tools1-install_bottle-homebrew-amd64', - 'ignition_transport-ci-main-focal-amd64', - 'ignition_transport-ci-main-homebrew-amd64', + 'ignition_transport-ci-gz-transport12-focal-amd64', + 'ignition_transport-ci-gz-transport12-homebrew-amd64', 'ignition_transport11-install-pkg-focal-amd64', 'ignition_transport11-install_bottle-homebrew-amd64', - 'ignition_utils-ci-ign-utils1-focal-amd64', - 'ignition_utils-ci-ign-utils1-homebrew-amd64', + 'ignition_utils-ci-gz-utils2-focal-amd64', + 'ignition_utils-ci-gz-utils2-homebrew-amd64', 'ignition_utils-install-pkg-focal-amd64', 'ignition_utils1-install_bottle-homebrew-amd64', 'sdformat-ci-main-focal-amd64', @@ -346,7 +346,7 @@ ignition_collections.each { ign_collection -> def ignition_brew_ci_job = job("ignition_${ign_collection_name}-ci-main-homebrew-amd64") OSRFBrewCompilation.create(ignition_brew_ci_job, DISABLE_TESTS) OSRFGitHub.create(ignition_brew_ci_job, - "gazebosim/ign-${ign_collection_name}", + "gazebosim/gz-${ign_collection_name}", "main", "ign-${ign_collection_name}") ignition_brew_ci_job.with From e0811d86d677eb2453c09ff64e7a8aaa4933baa1 Mon Sep 17 00:00:00 2001 From: "Addisu Z. Taddese" Date: Thu, 18 Aug 2022 14:41:38 -0500 Subject: [PATCH 195/540] Add pytest to list of pip installed dependencies for sdformat (#759) * Add pytest to list of pip installed dependencies for sdformat Signed-off-by: Addisu Z. Taddese * Make pytest a global dependency Signed-off-by: Addisu Z. Taddese Signed-off-by: Addisu Z. Taddese --- jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash | 3 +++ 1 file changed, 3 insertions(+) diff --git a/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash b/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash index 6e22201d4..72421907d 100644 --- a/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash +++ b/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash @@ -82,6 +82,9 @@ brew install ${PROJECT_FORMULA} ${PROJECT_ARGS} --only-dependencies # the following is needed to install :build dependencies of a formula brew install $(brew deps --1 --include-build ${PROJECT_FORMULA}) +# pytest is needed to run python tests with junit xml output +PIP_PACKAGES_NEEDED="${PIP_PACKAGES_NEEDED} pytest" + if [[ "${RERUN_FAILED_TESTS}" -gt 0 ]]; then # Install lxml for flaky_junit_merge.py PIP_PACKAGES_NEEDED="${PIP_PACKAGES_NEEDED} lxml" From db7a501df0ce46368e2a83501236161002dba197 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Mon, 22 Aug 2022 18:26:12 +0200 Subject: [PATCH 196/540] Run mk-build-deps in a temporary directory (2nd round) (#796) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Refactor mk-build-dep: run mk-build-deps from a temporary directory. Newer versions of mk-build-deps leave behind residual files. Before other scripts in this repository, notably jenkins-scripts/docker/lib/debbuild-base.bash handle this by explicitly removing the created files, but I am proposing that we handle it by creating the package in a temporary directory and then cleaning the entire directory. This way any changes in the future which, for example, generate additional leftovers, will be handled transparently rather than requiring an update. * Disable zstd compression for now (#771) * Disable xz compression for now Signed-off-by: Michael Carroll Signed-off-by: Jose Luis Rivero Co-authored-by: Steven! Ragnarök Co-authored-by: Michael Carroll --- jenkins-scripts/docker/lib/_gazebo_utils.sh | 22 +++++++++++++++++++ jenkins-scripts/docker/lib/debbuild-base.bash | 15 +------------ .../docker/lib/debbuild-bloom-base.bash | 4 +--- .../docker/lib/debian-git-repo-base.bash | 15 +++++++++---- 4 files changed, 35 insertions(+), 21 deletions(-) diff --git a/jenkins-scripts/docker/lib/_gazebo_utils.sh b/jenkins-scripts/docker/lib/_gazebo_utils.sh index 9c3a39468..e31c349d6 100644 --- a/jenkins-scripts/docker/lib/_gazebo_utils.sh +++ b/jenkins-scripts/docker/lib/_gazebo_utils.sh @@ -57,3 +57,25 @@ set -e echo '# END SECTION' fi """ + +MKBUILD_INSTALL_DEPS=""" +echo '# BEGIN SECTION: install build dependencies from debian/control' +mkdir build-deps +cd build-deps +# Wait 60 seconds and try again. Designed to avoid some race conditions with +# nightlies uploads. +update_done=false +seconds_waiting=0 +while (! \$update_done); do + sudo DEBIAN_FRONTEND=noninteractive mk-build-deps \ + -r -i ../debian/control \ + --tool 'apt-get --yes -o Debug::pkgProblemResolver=yes -o Debug::BuildDeps=yes' \ + && break + sleep 60 && seconds_waiting=\$((seconds_waiting+60)) + [ \$seconds_waiting -ge \$timeout ] && exit 1 +done +cd .. +# clean up files leftover by mk-build-deps +rm -rf build-deps +echo '# END SECTION' +""" diff --git a/jenkins-scripts/docker/lib/debbuild-base.bash b/jenkins-scripts/docker/lib/debbuild-base.bash index 2dce40924..f8de0d537 100644 --- a/jenkins-scripts/docker/lib/debbuild-base.bash +++ b/jenkins-scripts/docker/lib/debbuild-base.bash @@ -191,20 +191,7 @@ if ${NIGHTLY_MODE}; then timeout=300 fi -update_done=false -seconds_waiting=0 -while (! \$update_done); do - sudo DEBIAN_FRONTEND=noninteractive mk-build-deps \ - -r -i debian/control \ - --tool 'apt-get --yes -o Debug::pkgProblemResolver=yes -o Debug::BuildDeps=yes' \ - && break - sleep 60 && seconds_waiting=\$((seconds_waiting+60)) - [ \$seconds_waiting -ge \$timeout ] && exit 1 -done - -# new versions of mk-build-deps > 2.21.1 left buildinfo and changes files in the code -rm -f \${PACKAGE_ALIAS}-build-deps_*.{buildinfo,changes} -echo '# END SECTION' +${MKBUILD_INSTALL_DEPS} if [ -f /usr/bin/rosdep ]; then rosdep init diff --git a/jenkins-scripts/docker/lib/debbuild-bloom-base.bash b/jenkins-scripts/docker/lib/debbuild-bloom-base.bash index a9a8eae56..e758f40f6 100644 --- a/jenkins-scripts/docker/lib/debbuild-bloom-base.bash +++ b/jenkins-scripts/docker/lib/debbuild-bloom-base.bash @@ -52,9 +52,7 @@ cat debian/control fi echo '# END SECTION' -echo '# BEGIN SECTION: install build dependencies' -sudo mk-build-deps -r -i debian/control --tool 'apt-get --yes -o Debug::pkgProblemResolver=yes -o Debug::BuildDeps=yes' -echo '# END SECTION' +${MKBUILD_INSTALL_DEPS} echo '# BEGIN SECTION: run rosdep' sudo rosdep init diff --git a/jenkins-scripts/docker/lib/debian-git-repo-base.bash b/jenkins-scripts/docker/lib/debian-git-repo-base.bash index 1578ebf2c..ebede7a4a 100644 --- a/jenkins-scripts/docker/lib/debian-git-repo-base.bash +++ b/jenkins-scripts/docker/lib/debian-git-repo-base.bash @@ -56,10 +56,17 @@ if [[ ${DISTRO} == 'focal' && ${ARCH} == 'arm64' ]]; then sudo ln -sf /bin/true /usr/bin/lintian fi -echo '# BEGIN SECTION: install build dependencies' -cat debian/changelog -sudo mk-build-deps -r -i debian/control --tool 'apt-get --yes -o Debug::pkgProblemResolver=yes -o Debug::BuildDeps=yes' -echo '# END SECTION' +# our packages.o.o running xenial does not support default zstd compression of +# .deb files in jammy. Keep using xz. Not a trivial change, requires wrapper over dpkg-deb +if [[ ${DISTRO} == 'jammy' ]]; then + sudo bash -c 'echo \#\!/bin/bash > /usr/local/bin/dpkg-deb' + sudo bash -c 'echo "/usr/bin/dpkg-deb -Zxz \\\$@" >> /usr/local/bin/dpkg-deb' + sudo cat /usr/local/bin/dpkg-deb + sudo chmod +x /usr/local/bin/dpkg-deb + export PATH=/usr/local/bin:\$PATH +fi + +${MKBUILD_INSTALL_DEPS} VERSION=\$(dpkg-parsechangelog | grep Version | awk '{print \$2}') VERSION_NO_REVISION=\$(echo \$VERSION | sed 's:-.*::') From 891cf11bbfd9f12e95423826d023440c2d559b80 Mon Sep 17 00:00:00 2001 From: methylDragon Date: Mon, 22 Aug 2022 09:27:34 -0700 Subject: [PATCH 197/540] ign -> gz : Release Tools Migrate DSL (#750) * Migrate homebrew simulation and DSL replacements in bump_dependency * Migrate ignition and ign_collection_name, ign_sw DSL vars Change should bring no change in XML configuration files produced. Signed-off-by: methylDragon Co-authored-by: Jose Luis Rivero --- jenkins-scripts/dsl/ignition.dsl | 405 +++++++++++--------- jenkins-scripts/dsl/ignition_collection.dsl | 64 ++-- jenkins-scripts/lib/dependencies_archive.sh | 2 +- source-repo-scripts/bump_dependency.bash | 62 +-- 4 files changed, 280 insertions(+), 253 deletions(-) diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index 2c8fc63e5..9323b33e8 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -2,28 +2,28 @@ import _configs_.* import javaposse.jobdsl.dsl.Job // IGNITION PACKAGES -ignition_software = [ 'cmake', - 'common', - 'fuel-tools', - 'gazebo', - 'gui', - 'launch', - 'math', - 'msgs', - 'physics', - 'plugin', - 'rendering', - 'sensors', - 'tools', - 'transport', - 'utils' ] +gz_software = [ 'cmake', + 'common', + 'fuel-tools', + 'gazebo', + 'gui', + 'launch', + 'math', + 'msgs', + 'physics', + 'plugin', + 'rendering', + 'sensors', + 'tools', + 'transport', + 'utils' ] // DESC: need gpu/display for tests -ignition_gpu = [ 'gazebo', - 'gui', - 'rendering', - 'sensors' ] +gz_gpu = [ 'gazebo', + 'gui', + 'rendering', + 'sensors' ] // DESC: software does not support cmake warnings enabled -ignition_no_cmake_warnings = [ 'cmake', +gz_no_cmake_warnings = [ 'cmake', 'common', 'fuel-tools', 'gazebo', @@ -38,10 +38,10 @@ ignition_no_cmake_warnings = [ 'cmake', 'transport', 'utils' ] // DESC: software does not have tests -ignition_no_test = [ ] +gz_no_test = [ ] // DESC: major series supported and released. The branches get CI, install pkg // testing and debbuild job. -// No branches in ignition_branches means no released branches (only CI on +// No branches in gz_branches means no released branches (only CI on // main, ABI check, install pkg) ignition_branches = [ 'cmake' : [ '2' ], 'common' : [ '1', '3', '4' ], @@ -58,6 +58,7 @@ ignition_branches = [ 'cmake' : [ '2' ], 'tools' : [ '1' ], 'transport' : [ '4', '8', '11' ], 'utils' : [ '1' ]] + gz_branches = [ 'cmake' : [ '3' ], 'common' : [ '5' ], 'fuel-tools' : [ '8' ], @@ -76,35 +77,35 @@ gz_branches = [ 'cmake' : [ '3' ], // DESC: prerelease branches are managed as any other supported branches for // special cases different to major branches: get compilation CI on the branch // physics/sensors don't need to be included since they use main for gz11 -ignition_prerelease_branches = [] +gz_prerelease_branches = [] // DESC: versioned names to generate debbuild jobs for special cases that -// don't appear in ignition_branches (like nightly builders or 0-debbuild +// don't appear in gz_branches (like nightly builders or 0-debbuild // jobs for the special cases of foo0 packages) -ignition_extra_debbuild = [ 'gazebo7', - 'utils1'] // see comment https://github.com/gazebo-tooling/release-tools/pull/431#issuecomment-815099918 +gz_extra_debbuild = [ 'gazebo7', + 'utils1' ] // see comment https://github.com/gazebo-tooling/release-tools/pull/431#issuecomment-815099918 // DESC: exclude ignition from generate any install testing job -ignition_no_pkg_yet = [ ] +gz_no_pkg_yet = [ ] // DESC: major versions that has a package in the prerelease repo. Should -// not appear in ignition_no_pkg_yet nor in ignition_branches -ignition_prerelease_pkgs = [ 'placeholder' : [ - '1': [ 'bionic' ]], - ] +// not appear in gz_no_pkg_yet nor in gz_branches +gz_prerelease_pkgs = [ 'placeholder' : [ + '1': [ 'bionic' ]], + ] // packages using colcon for windows compilation while migrating all them to // this solution -ignition_colcon_win = [ 'common', - 'fuel-tools', - 'gazebo', - 'gui', - 'launch', - 'math', - 'msgs', - 'physics', - 'plugin', - 'rendering', - 'sensors', - 'tools', - 'transport', - 'utils' ] +gz_colcon_win = [ 'common', + 'fuel-tools', + 'gazebo', + 'gui', + 'launch', + 'math', + 'msgs', + 'physics', + 'plugin', + 'rendering', + 'sensors', + 'tools', + 'transport', + 'utils' ] // Main platform using for quick CI def ci_distro = Globals.get_ci_distro() @@ -130,9 +131,9 @@ String ci_distro_str = ci_distro[0] // Map of lists to use in CIWorkflow ci_pr_any_list = [:] -ignition_software.each { ign_sw -> +gz_software.each { gz_sw -> def list_empty = [] - ci_pr_any_list[ign_sw] = list_empty + ci_pr_any_list[gz_sw] = list_empty } /** @@ -166,6 +167,9 @@ ArrayList supported_ign_branches(String ign_software) return major_versions_registered } + +// return major versions supported or empty if just 0,1 series under +// -dev package. ArrayList supported_gz_branches(String gz_software) { major_versions_registered = gz_branches["${gz_software}"] @@ -177,9 +181,9 @@ ArrayList supported_gz_branches(String gz_software) } // return prerelease branch names -ArrayList prerelease_branches(String ign_software) +ArrayList prerelease_branches(String gz_software) { - pre_branches = ignition_prerelease_branches["${ign_software}"] + pre_branches = gz_prerelease_branches["${gz_software}"] if (pre_branches == null) return [ '' ] @@ -188,21 +192,21 @@ ArrayList prerelease_branches(String ign_software) } // return all ci branch names -ArrayList all_branches(String ign_software) +ArrayList all_branches(String software_name) { List branches = new ArrayList(); - supported_ign_branches("${ign_software}").each { major_version -> + supported_ign_branches("${software_name}").each { major_version -> if ("${major_version}") { - branches.add("ign-${ign_software}${major_version}") + branches.add("ign-${software_name}${major_version}") } } - supported_gz_branches("${ign_software}").each { major_version -> + supported_gz_branches("${software_name}").each { major_version -> if ("${major_version}") { - branches.add("gz-${ign_software}${major_version}") + branches.add("gz-${software_name}${major_version}") } } branches.add('main') - prerelease_branches("${ign_software}").each { branch -> + prerelease_branches("${software_name}").each { branch -> if ("${branch}") { branches.add(branch) } @@ -210,34 +214,34 @@ ArrayList all_branches(String ign_software) return branches } -// + ArrayList all_debbuilders() { List branches = new ArrayList(); // add all supported branches - ignition_software.each { ign_software -> - supported_ign_branches("${ign_software}").each { major_version -> + gz_software.each { software_name -> + supported_ign_branches("${software_name}").each { major_version -> if (major_version) { // No 1-debbuild versions, they use the unversioned job if ("${major_version}" == "0" || "${major_version}" == "1" ) major_version = "" - branches.add("ign-${ign_software}${major_version}") + branches.add("ign-${software_name}${major_version}") } } - supported_gz_branches("${ign_software}").each { major_version -> + supported_gz_branches("${software_name}").each { major_version -> if (major_version) { // No 1-debbuild versions, they use the unversioned job if ("${major_version}" == "0" || "${major_version}" == "1" ) major_version = "" - branches.add("ign-${ign_software}${major_version}") + branches.add("ign-${software_name}${major_version}") } } } // add all extra debbuilders - ignition_extra_debbuild.each { ign_name -> - branches.add("ign-${ign_name}") + gz_extra_debbuild.each { gz_name -> + branches.add("ign-${gz_name}") } return branches @@ -247,34 +251,34 @@ ArrayList all_debbuilders() // Map with the form of: major versions as keys. // Lists of distros supported as values // @TODO[scpeters] update this to support gz_branches after garden release -Map supported_install_pkg_branches(String ign_software) +Map supported_install_pkg_branches(String gz_software) { - major_versions_prerelease = ignition_prerelease_pkgs["${ign_software}"] + major_versions_prerelease = gz_prerelease_pkgs["${gz_software}"] - // construct a map of stable packages based on supported_ign_branches and + // construct a map of stable packages based on supported_gz_branches and // all_supported_distros map_of_stable_versions = [:] - map_of_stable_versions[ign_software] = [:] - supported_ign_branches(ign_software).each { major_version -> + map_of_stable_versions[gz_software] = [:] + supported_ign_branches(gz_software).each { major_version -> new_relation = [:] new_relation[major_version] = all_supported_distros - map_of_stable_versions[ign_software] << new_relation + map_of_stable_versions[gz_software] << new_relation } if (major_versions_prerelease == null) - return map_of_stable_versions[ign_software]; + return map_of_stable_versions[gz_software]; - return merge_maps(map_of_stable_versions[ign_software], + return merge_maps(map_of_stable_versions[gz_software], major_versions_prerelease) } -void include_gpu_label_if_needed(Job job, String ign_software_name) +void include_gpu_label_if_needed(Job job, String gz_software_name) { job.with { - ignition_gpu.each { ign_each -> - if (ign_software_name == ign_each) + gz_gpu.each { gz_each -> + if (gz_software_name == gz_each) { label "gpu-reliable" @@ -291,25 +295,25 @@ void include_gpu_label_if_needed(Job job, String ign_software_name) } } -boolean enable_cmake_warnings(String ign_software_name) +boolean enable_cmake_warnings(String gz_software_name) { - if (ign_software_name in ignition_no_cmake_warnings) + if (gz_software_name in gz_no_cmake_warnings) return false return true } -boolean enable_testing(String ign_software_name) +boolean enable_testing(String gz_software_name) { - if (ign_software_name in ignition_no_test) + if (gz_software_name in gz_no_test) return false return true } -boolean is_a_colcon_package(String ign_software_name) +boolean is_a_colcon_package(String gz_software_name) { - if (ign_software_name in ignition_colcon_win) + if (gz_software_name in gz_colcon_win) return true return false @@ -317,24 +321,29 @@ boolean is_a_colcon_package(String ign_software_name) // ABI Checker job // Need to be before the ci-pr_any so the abi job name is defined -ignition_software.each { ign_sw -> +gz_software.each { gz_sw -> abi_distro.each { distro -> supported_arches.each { arch -> // Packages without ABI - if (ign_sw == 'tools' || ign_sw == 'cmake') + if (gz_sw == 'tools' || gz_sw == 'cmake') return - abi_job_names[ign_sw] = "ignition_${ign_sw}-abichecker-any_to_any-ubuntu_auto-${arch}" - def abi_job = job(abi_job_names[ign_sw]) - checkout_subdir = "ign-${ign_sw}" + software_name = gz_sw // Necessary substitution. gz_sw won't overwrite + + if (gz_sw == 'sim') + software_name = "gazebo" + + abi_job_names[software_name] = "ignition_${software_name}-abichecker-any_to_any-ubuntu_auto-${arch}" + def abi_job = job(abi_job_names[software_name]) + checkout_subdir = "ign-${software_name}" OSRFLinuxABIGitHub.create(abi_job) GenericAnyJobGitHub.create(abi_job, - "gazebosim/ign-${ign_sw}", - all_branches(ign_sw) - [ 'main']) + "gazebosim/ign-${software_name}", + all_branches(software_name) - [ 'main']) abi_job.with { extra_str="" - if (ign_sw == 'physics') + if (gz_sw == 'physics') { label "huge-memory" // on ARM native nodes in buildfarm we need to restrict to 1 the @@ -365,20 +374,25 @@ ignition_software.each { ign_sw -> } // end of ignition // MAIN CI JOBS (check every 5 minutes) -ignition_software.each { ign_sw -> +gz_software.each { gz_sw -> supported_arches.each { arch -> // -------------------------------------------------------------- // 1. Create the any job - def ignition_ci_job_name = "ignition_${ign_sw}-ci-pr_any-ubuntu_auto-${arch}" - def ignition_ci_any_job = job(ignition_ci_job_name) - def ignition_checkout_dir = "ign-${ign_sw}" - OSRFLinuxCompilationAnyGitHub.create(ignition_ci_any_job, - "gazebosim/${ignition_checkout_dir}", - enable_testing(ign_sw)) - include_gpu_label_if_needed(ignition_ci_any_job, ign_sw) - ignition_ci_any_job.with + software_name = gz_sw // Necessary substitution. gz_sw won't overwrite + + if (gz_sw == 'sim') + software_name = "gazebo" + + def gz_ci_job_name = "ignition_${software_name}-ci-pr_any-ubuntu_auto-${arch}" + def gz_ci_any_job = job(gz_ci_job_name) + def gz_checkout_dir = "ign-${software_name}" + OSRFLinuxCompilationAnyGitHub.create(gz_ci_any_job, + "gazebosim/${gz_checkout_dir}", + enable_testing(software_name)) + include_gpu_label_if_needed(gz_ci_any_job, software_name) + gz_ci_any_job.with { - if (ign_sw == 'physics') + if (gz_sw == 'physics') label "huge-memory" steps @@ -392,40 +406,40 @@ ignition_software.each { ign_sw -> export ARCH=${arch} - /bin/bash -xe ./scripts/jenkins-scripts/docker/ign_${ign_sw}-compilation.bash + /bin/bash -xe ./scripts/jenkins-scripts/docker/ign_${software_name}-compilation.bash """.stripIndent()) } // end of steps } // end of ci_any_job // add ci-pr_any to the list for CIWorkflow - ci_pr_any_list[ign_sw] << ignition_ci_job_name + ci_pr_any_list[software_name] << gz_ci_job_name } } // INSTALL PACKAGE ALL PLATFORMS / DAILY -ignition_software.each { ign_sw -> +gz_software.each { gz_sw -> // Exclusion list - if (ign_sw in ignition_no_pkg_yet) + if (gz_sw in gz_no_pkg_yet) return supported_arches.each { arch -> - supported_install_pkg_branches(ign_sw).each { major_version, supported_distros -> + supported_install_pkg_branches(gz_sw).each { major_version, supported_distros -> supported_distros.each { distro -> extra_repos_str="" - if ((ign_sw in ignition_prerelease_pkgs) && - (major_version in ignition_prerelease_pkgs[ign_sw]) && - (distro in ignition_prerelease_pkgs[ign_sw][major_version])) + if ((gz_sw in gz_prerelease_pkgs) && + (major_version in gz_prerelease_pkgs[gz_sw]) && + (distro in gz_prerelease_pkgs[gz_sw][major_version])) extra_repos_str="prerelease" // No 1-dev or 0-dev packages (except special cases see - // ignition_debbuild variable), unversioned + // gz_debbuild variable), unversioned if ("${major_version}" == "0" || "${major_version}" == "1") major_version = "" // -------------------------------------------------------------- - def install_default_job = job("ignition_${ign_sw}${major_version}-install-pkg-${distro}-${arch}") + def install_default_job = job("ignition_${gz_sw}${major_version}-install-pkg-${distro}-${arch}") OSRFLinuxInstall.create(install_default_job) - include_gpu_label_if_needed(install_default_job, ign_sw) + include_gpu_label_if_needed(install_default_job, gz_sw) install_default_job.with { @@ -433,8 +447,8 @@ ignition_software.each { ign_sw -> cron(Globals.CRON_EVERY_THREE_DAYS) } - def dev_package = "libignition-${ign_sw}${major_version}-dev" - def gzdev_project = "ignition-${ign_sw}${major_version}" + def dev_package = "libignition-${gz_sw}${major_version}-dev" + def gzdev_project = "ignition-${gz_sw}${major_version}" steps { shell("""\ @@ -455,25 +469,25 @@ ignition_software.each { ign_sw -> } } -void generate_asan_ci_job(ignition_ci_job, ign_sw, branch, distro, arch) +void generate_asan_ci_job(gz_ci_job, gz_sw, branch, distro, arch) { - generate_ci_job(ignition_ci_job, ign_sw, branch, distro, arch, + generate_ci_job(gz_ci_job, gz_sw, branch, distro, arch, '-DIGN_SANITIZER=Address', Globals.MAKETEST_SKIP_IGN) } -void generate_ci_job(ignition_ci_job, ign_sw, branch, distro, arch, +void generate_ci_job(gz_ci_job, gz_sw, branch, distro, arch, extra_cmake = '', extra_test = '') { - OSRFLinuxCompilation.create(ignition_ci_job, enable_testing(ign_sw)) - OSRFGitHub.create(ignition_ci_job, - "gazebosim/ign-${ign_sw}", - "${branch}", "ign-${ign_sw}") + OSRFLinuxCompilation.create(gz_ci_job, enable_testing(software_name)) + OSRFGitHub.create(gz_ci_job, + "gazebosim/ign-${software_name}", + "${branch}", "ign-${software_name}") - include_gpu_label_if_needed(ignition_ci_job, ign_sw) - ignition_ci_job.with + include_gpu_label_if_needed(gz_ci_job, gz_sw) + gz_ci_job.with { - if (ign_sw == 'physics') + if (gz_sw == 'physics') label "huge-memory" steps { @@ -485,32 +499,37 @@ void generate_ci_job(ignition_ci_job, ign_sw, branch, distro, arch, export BUILDING_EXTRA_MAKETEST_PARAMS="${extra_test}" export DISTRO=${distro} export ARCH=${arch} - /bin/bash -xe ./scripts/jenkins-scripts/docker/ign_${ign_sw}-compilation.bash + /bin/bash -xe ./scripts/jenkins-scripts/docker/ign_${software_name}-compilation.bash """.stripIndent()) } } } // OTHER CI SUPPORTED JOBS -ignition_software.each { ign_sw -> +gz_software.each { gz_sw -> all_supported_distros.each { distro -> supported_arches.each { arch -> // -------------------------------------------------------------- // branches CI job scm@daily - all_branches("${ign_sw}").each { branch -> + all_branches("${gz_sw}").each { branch -> // 1. Standard CI - def ignition_ci_job = job("ignition_${ign_sw}-ci-${branch}-${distro}-${arch}") - generate_ci_job(ignition_ci_job, ign_sw, branch, distro, arch) - ignition_ci_job.with + software_name = gz_sw // Necessary substitution. gz_sw won't overwrite + + if (gz_sw == 'sim') + software_name = "gazebo" + + def gz_ci_job = job("ignition_${software_name}-ci-${branch}-${distro}-${arch}") + generate_ci_job(gz_ci_job, software_name, branch, distro, arch) + gz_ci_job.with { triggers { scm('@daily') } } // 2. ASAN CI - def ignition_ci_asan_job = job("ignition_${ign_sw}-ci_asan-${branch}-${distro}-${arch}") - generate_asan_ci_job(ignition_ci_asan_job, ign_sw, branch, distro, arch) - ignition_ci_asan_job.with + def gz_ci_asan_job = job("ignition_${software_name}-ci_asan-${branch}-${distro}-${arch}") + generate_asan_ci_job(gz_ci_asan_job, software_name, branch, distro, arch) + gz_ci_asan_job.with { triggers { scm(Globals.CRON_ON_WEEKEND) @@ -533,7 +552,6 @@ all_debbuilders().each { debbuilder_name -> if (debbuilder_name.contains("-physics")) extra_str += '\nif [ $(uname -m) = "aarch64" ]; then export MAKE_JOBS=1; fi' - println("Generating: ${debbuilder_name}-debbuilder") def build_pkg_job = job("${debbuilder_name}-debbuilder") OSRFLinuxBuildPkg.create(build_pkg_job) build_pkg_job.with @@ -562,30 +580,35 @@ all_debbuilders().each { debbuilder_name -> // BREW: CI jobs // 1. any job -ignition_software.each { ign_sw -> - String ignition_brew_ci_any_job_name = "ignition_${ign_sw}-ci-pr_any-homebrew-amd64" +gz_software.each { gz_sw -> + software_name = gz_sw // Necessary substitution. gz_sw won't overwrite + + if (gz_sw == 'sim') + software_name = "gazebo" + + String gz_brew_ci_any_job_name = "ignition_${software_name}-ci-pr_any-homebrew-amd64" - def ignition_brew_ci_any_job = job(ignition_brew_ci_any_job_name) - OSRFBrewCompilationAnyGitHub.create(ignition_brew_ci_any_job, - "gazebosim/ign-${ign_sw}", - enable_testing(ign_sw), + def gz_brew_ci_any_job = job(gz_brew_ci_any_job_name) + OSRFBrewCompilationAnyGitHub.create(gz_brew_ci_any_job, + "gazebosim/ign-${software_name}", + enable_testing(software_name), GITHUB_SUPPORT_ALL_BRANCHES, ENABLE_GITHUB_PR_INTEGRATION, - enable_cmake_warnings(ign_sw)) - ignition_brew_ci_any_job.with + enable_cmake_warnings(software_name)) + gz_brew_ci_any_job.with { steps { shell("""\ #!/bin/bash -xe - export HOMEBREW_SCRIPT="./scripts/jenkins-scripts/ign_${ign_sw}-default-devel-homebrew-amd64.bash" + export HOMEBREW_SCRIPT="./scripts/jenkins-scripts/ign_${software_name}-default-devel-homebrew-amd64.bash" if [ -s "\$HOMEBREW_SCRIPT" ] then /bin/bash -xe "\$HOMEBREW_SCRIPT" else - software_name="gz-${ign_sw}" - [[ ${ign_sw} == 'gazebo' ]] && software_name="gz-sim" + software_name="gz-${software_name}" + [[ ${software_name} == 'gazebo' ]] && software_name="gz-sim" /bin/bash -xe "./scripts/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash" "\${software_name}" fi """.stripIndent()) @@ -593,18 +616,18 @@ ignition_software.each { ign_sw -> } // add ci-pr_any to the list for CIWorkflow - ci_pr_any_list[ign_sw] << ignition_brew_ci_any_job_name + ci_pr_any_list[software_name] << gz_brew_ci_any_job_name // 2. main, release branches - all_branches("${ign_sw}").each { branch -> - def ignition_brew_ci_job = job("ignition_${ign_sw}-ci-${branch}-homebrew-amd64") - OSRFBrewCompilation.create(ignition_brew_ci_job, - enable_testing(ign_sw), - enable_cmake_warnings(ign_sw)) - OSRFGitHub.create(ignition_brew_ci_job, - "gazebosim/ign-${ign_sw}", - "${branch}", "ign-${ign_sw}") - ignition_brew_ci_job.with + all_branches("${software_name}").each { branch -> + def gz_brew_ci_job = job("ignition_${software_name}-ci-${branch}-homebrew-amd64") + OSRFBrewCompilation.create(gz_brew_ci_job, + enable_testing(software_name), + enable_cmake_warnings(software_name)) + OSRFGitHub.create(gz_brew_ci_job, + "gazebosim/ign-${software_name}", + "${branch}", "ign-${software_name}") + gz_brew_ci_job.with { triggers { scm('@daily') @@ -614,13 +637,13 @@ ignition_software.each { ign_sw -> shell("""\ #!/bin/bash -xe - export HOMEBREW_SCRIPT="./scripts/jenkins-scripts/ign_${ign_sw}-default-devel-homebrew-amd64.bash" + export HOMEBREW_SCRIPT="./scripts/jenkins-scripts/ign_${software_name}-default-devel-homebrew-amd64.bash" if [ -s "\$HOMEBREW_SCRIPT" ] then /bin/bash -xe "\$HOMEBREW_SCRIPT" else - software_name="gz-${ign_sw}" - [[ ${ign_sw} == 'gazebo' ]] && software_name="gz-sim" + software_name="gz-${software_name}" + [[ ${software_name} == 'gazebo' ]] && software_name="gz-sim" /bin/bash -xe "./scripts/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash" "\${software_name}" fi """.stripIndent()) @@ -629,21 +652,21 @@ ignition_software.each { ign_sw -> } // 3. install jobs to test bottles - supported_install_pkg_branches(ign_sw).each { major_version, supported_distros -> - def install_default_job = job("ignition_${ign_sw}${major_version}-install_bottle-homebrew-amd64") + supported_install_pkg_branches(software_name).each { major_version, supported_distros -> + def install_default_job = job("ignition_${software_name}${major_version}-install_bottle-homebrew-amd64") OSRFBrewInstall.create(install_default_job) install_default_job.with { // disable some bottles - if (("${ign_sw}" == "gui" && "${major_version}" == "0")) + if (("${gz_sw}" == "gui" && "${major_version}" == "0")) disabled() triggers { cron('@daily') } - def bottle_name = "ignition-${ign_sw}${major_version}" + def bottle_name = "ignition-${software_name}${major_version}" steps { shell("""\ @@ -671,70 +694,72 @@ ignition_software.each { ign_sw -> // WINDOWS: CI job // 1. any -ignition_software.each { ign_sw -> +gz_software.each { gz_sw -> - if (is_a_colcon_package(ign_sw)) { + if (is_a_colcon_package(gz_sw)) { // colcon uses long paths and windows has a hard limit of 260 chars. Keep // names minimal - ignition_win_ci_any_job_name = "ign_${ign_sw}-pr-win" + gz_win_ci_any_job_name = "ign_${gz_sw}-pr-win" Globals.gazebodistro_branch = true } else { - ignition_win_ci_any_job_name = "ignition_${ign_sw}-ci-pr_any-windows7-amd64" + gz_win_ci_any_job_name = "ignition_${gz_sw}-ci-pr_any-windows7-amd64" Globals.gazebodistro_branch = false } supported_branches = [] + gz_software_name = gz_sw // Necessary substitution. gz_sw won't overwrite // ign-gazebo only support Windows from ign-gazebo5 - if (ign_sw == 'gazebo') + if (gz_sw == 'gazebo') supported_branches = [ 'ign-gazebo6', 'gz-sim7', 'main' ] + gz_software_name = "sim" // ign-launch only support Windows from ign-launch5 - if (ign_sw == 'launch') + if (gz_sw == 'launch') supported_branches = [ 'ign-launch5', 'gz-launch6', 'main' ] - def ignition_win_ci_any_job = job(ignition_win_ci_any_job_name) - OSRFWinCompilationAnyGitHub.create(ignition_win_ci_any_job, - "gazebosim/gz-${ign_sw}", - enable_testing(ign_sw), + def gz_win_ci_any_job = job(gz_win_ci_any_job_name) + OSRFWinCompilationAnyGitHub.create(gz_win_ci_any_job, + "gazebosim/gz-${gz_sw}", + enable_testing(gz_sw), supported_branches, ENABLE_GITHUB_PR_INTEGRATION, - enable_cmake_warnings(ign_sw)) - ignition_win_ci_any_job.with + enable_cmake_warnings(gz_sw)) + gz_win_ci_any_job.with { steps { batchFile("""\ - call "./scripts/jenkins-scripts/ign_${ign_sw}-default-devel-windows-amd64.bat" + call "./scripts/jenkins-scripts/ign_${gz_sw}-default-devel-windows-amd64.bat" """.stripIndent()) } } // add ci-pr_any to the list for CIWorkflow - ci_pr_any_list[ign_sw] << ignition_win_ci_any_job_name + ci_pr_any_list[gz_sw] << gz_win_ci_any_job_name // 2. main, release branches - all_branches("${ign_sw}").each { branch -> - if (is_a_colcon_package(ign_sw)) { + all_branches("${gz_sw}").each { branch -> + if (is_a_colcon_package(gz_sw)) { // colcon uses long paths and windows has a hard limit of 260 chars. Keep // names minimal if (branch == 'main') branch_name = "ci" else - branch_name = branch - ign_sw - ignition_win_ci_job_name = "ign_${ign_sw}-${branch_name}-win" + branch_name = branch - gz_sw + gz_win_ci_job_name = "ign_${gz_sw}-${branch_name}-win" } else { - ignition_win_ci_job_name = "ignition_${ign_sw}-ci-${branch}-windows7-amd64" + gz_win_ci_job_name = "ignition_${gz_sw}-ci-${branch}-windows7-amd64" } - def ignition_win_ci_job = job(ignition_win_ci_job_name) - OSRFWinCompilation.create(ignition_win_ci_job, - enable_testing(ign_sw), - enable_cmake_warnings(ign_sw)) - OSRFGitHub.create(ignition_win_ci_job, - "gazebosim/gz-${ign_sw}", + def gz_win_ci_job = job(gz_win_ci_job_name) + OSRFWinCompilation.create(gz_win_ci_job, + enable_testing(gz_sw), + enable_cmake_warnings(gz_sw)) + OSRFGitHub.create(gz_win_ci_job, + "gazebosim/gz-${gz_sw}", "${branch}") - ignition_win_ci_job.with + gz_win_ci_job.with { // ign-gazebo only works on Windows from ign-gazebo5 if (branch == 'ign-gazebo3') @@ -750,7 +775,7 @@ ignition_software.each { ign_sw -> steps { batchFile("""\ - call "./scripts/jenkins-scripts/ign_${ign_sw}-default-devel-windows-amd64.bat" + call "./scripts/jenkins-scripts/ign_${gz_sw}-default-devel-windows-amd64.bat" """.stripIndent()) } } @@ -758,8 +783,8 @@ ignition_software.each { ign_sw -> } // Main CI workflow -ignition_software.each { ign_sw -> - def String ci_main_name = "ignition_${ign_sw}-ci-manual_any" - def ign_ci_main = pipelineJob(ci_main_name) - OSRFCIWorkFlowMultiAnyGitHub.create(ign_ci_main, ci_pr_any_list[ign_sw]) +gz_software.each { gz_sw -> + def String ci_main_name = "ignition_${gz_sw}-ci-manual_any" + def gz_ci_main = pipelineJob(ci_main_name) + OSRFCIWorkFlowMultiAnyGitHub.create(gz_ci_main, ci_pr_any_list[gz_sw]) } diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index de25e5348..f4c3266aa 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -1,12 +1,12 @@ import _configs_.* import javaposse.jobdsl.dsl.Job -// IGNITION COLLECTIONS +// GZ COLLECTIONS arch = 'amd64' -ignition_nightly = 'garden' +gz_nightly = 'garden' -ignition_collections = [ +gz_collections = [ [ name : 'citadel', distros : [ 'bionic' ], ], @@ -39,7 +39,7 @@ ignition_collections = [ ], ] -ignition_collection_jobs = +gz_collection_jobs = [ 'citadel' : [ 'ign_common-ign-3-win', @@ -292,27 +292,28 @@ ignition_collection_jobs = def DISABLE_TESTS = false // Testing compilation from source -ignition_collections.each { ign_collection -> +gz_collections.each { gz_collection -> // COLCON - Windows - ign_collection_name = ign_collection.get('name') - def ignition_win_ci_job = job("ign_${ign_collection_name}-ci-win") + gz_collection_name = gz_collection.get('name') + + def gz_win_ci_job = job("ign_${gz_collection_name}-ci-win") Globals.gazebodistro_branch = true - OSRFWinCompilation.create(ignition_win_ci_job, false) - ignition_win_ci_job.with + OSRFWinCompilation.create(gz_win_ci_job, false) + gz_win_ci_job.with { steps { batchFile("""\ - set IGNITION_COLLECTION=${ign_collection_name} + set IGNITION_COLLECTION=${gz_collection_name} call "./scripts/jenkins-scripts/lib/ign_collection-base.bat" """.stripIndent()) } } Globals.gazebodistro_branch = false - ign_collection.get('distros').each { distro -> + gz_collection.get('distros').each { distro -> // INSTALL JOBS: // -------------------------------------------------------------- - def install_default_job = job("ignition_${ign_collection_name}-install-pkg-${distro}-${arch}") + def install_default_job = job("ignition_${gz_collection_name}-install-pkg-${distro}-${arch}") OSRFLinuxInstall.create(install_default_job) install_default_job.with @@ -321,7 +322,7 @@ ignition_collections.each { ign_collection -> cron(Globals.CRON_EVERY_THREE_DAYS) } - def dev_package = "ignition-${ign_collection_name}" + def dev_package = "ignition-${gz_collection_name}" label "gpu-reliable" @@ -343,35 +344,36 @@ ignition_collections.each { ign_collection -> // MAC Brew CI job // -------------------------------------------------------------- - def ignition_brew_ci_job = job("ignition_${ign_collection_name}-ci-main-homebrew-amd64") - OSRFBrewCompilation.create(ignition_brew_ci_job, DISABLE_TESTS) - OSRFGitHub.create(ignition_brew_ci_job, - "gazebosim/gz-${ign_collection_name}", + def gz_brew_ci_job = job("ignition_${gz_collection_name}-ci-main-homebrew-amd64") + OSRFBrewCompilation.create(gz_brew_ci_job, DISABLE_TESTS) + OSRFGitHub.create(gz_brew_ci_job, + "gazebosim/gz-${gz_collection_name}", "main", - "ign-${ign_collection_name}") - ignition_brew_ci_job.with + "ign-${gz_collection_name}") + gz_brew_ci_job.with { steps { shell("""\ #!/bin/bash -xe - /bin/bash -xe "./scripts/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash" "gz-${ign_collection_name}" + /bin/bash -xe + "./scripts/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash" "gz-${gz_collection_name}" """.stripIndent()) } } // MAC Brew bottle install job // -------------------------------------------------------------- - def ignition_brew_install_bottle_job = job("ignition_${ign_collection_name}-install_bottle-homebrew-amd64") - OSRFBrewInstall.create(ignition_brew_install_bottle_job) + def gz_brew_install_bottle_job = job("ignition_${gz_collection_name}-install_bottle-homebrew-amd64") + OSRFBrewInstall.create(gz_brew_install_bottle_job) - ignition_brew_install_bottle_job.with + gz_brew_install_bottle_job.with { triggers { cron('@daily') } - def bottle_name = "ignition-${ign_collection_name}" + def bottle_name = "ignition-${gz_collection_name}" steps { shell("""\ @@ -395,7 +397,7 @@ ignition_collections.each { ign_collection -> // DEBBUILD: linux package builder // -------------------------------------------------------------- - def build_pkg_job = job("ign-${ign_collection_name}-debbuilder") + def build_pkg_job = job("ign-${gz_collection_name}-debbuilder") OSRFLinuxBuildPkg.create(build_pkg_job) build_pkg_job.with { @@ -409,15 +411,15 @@ ignition_collections.each { ign_collection -> } // Gazebo dashboards - dashboardView("ign-${ign_collection_name}") + dashboardView("ign-${gz_collection_name}") { jobs { - ignition_collection_jobs["${ign_collection_name}"].each { jobname -> + gz_collection_jobs["${gz_collection_name}"].each { jobname -> name(jobname) } - if (ign_collection_name == ignition_nightly) { + if (gz_collection_name == gz_nightly) { // add nightly debbuild jobs too - ignition_collections.find { it.get('name') == ignition_nightly }.get('nightly_jobs').each { job -> + gz_collections.find { it.get('name') == gz_nightly }.get('nightly_jobs').each { job -> name(job.getValue().get('debbuild') + '-debbuilder') } } @@ -471,7 +473,7 @@ def get_nightly_branch(collection_data, ign_package) collection_data = [] list_of_pkgs = "" -collection_data = ignition_collections.find { it.get('name') == ignition_nightly } +collection_data = gz_collections.find { it.get('name') == gz_nightly } collection_data = collection_data.get('nightly_jobs') collection_data.each { job -> @@ -479,7 +481,7 @@ collection_data.each { job -> list_of_pkgs = "${list_of_pkgs} ${debbuild}" } -def nightly_scheduler_job = job("ignition-${ignition_nightly}-nightly-scheduler") +def nightly_scheduler_job = job("ignition-${gz_nightly}-nightly-scheduler") OSRFUNIXBase.create(nightly_scheduler_job) nightly_scheduler_job.with diff --git a/jenkins-scripts/lib/dependencies_archive.sh b/jenkins-scripts/lib/dependencies_archive.sh index 0d0e25065..1ae78f86c 100644 --- a/jenkins-scripts/lib/dependencies_archive.sh +++ b/jenkins-scripts/lib/dependencies_archive.sh @@ -549,7 +549,7 @@ if [[ ${GZ_GUI_MAJOR_VERSION} -lt 7 ]]; then fi fi -IGN_PHYSICS_DART_FROM_PKGS="true" +GZ_PHYSICS_DART_FROM_PKGS="true" # Completely rely on packages.apt from Garden if [[ ${GZ_PLUGIN_MAJOR_VERSION} -lt 2 ]]; then diff --git a/source-repo-scripts/bump_dependency.bash b/source-repo-scripts/bump_dependency.bash index d7e86274c..671b4f8f9 100755 --- a/source-repo-scripts/bump_dependency.bash +++ b/source-repo-scripts/bump_dependency.bash @@ -21,10 +21,10 @@ # Usage: # $ ./bump_dependency.bash ; ; [] # -# For example, to bump to `ign-rendering6` and all its dependencies, as well as +# For example, to bump to `gz-rendering6` and all its dependencies, as well as # `sdf12` and its dependencies on fortress using the `chapulina/fortress` branch for `docs`: # -# ./bump_dependency.bash fortress "ign-rendering;sdformat" "6;12" 428 edifice "chapulina/fortress" +# ./bump_dependency.bash fortress "gz-rendering;sdformat" "6;12" 428 edifice "chapulina/fortress" # # The `docs_branch` parameter is optional and defaults to `master` if not specified. # @@ -47,7 +47,7 @@ WHITE_BG="\e[107m" BLUE_BG="\e[44m" GREEN_BG="\e[42m" -IGN_ORG="gazebosim" +GZ_ORG="gazebosim" OSRF_ORG="osrf" TOOLING_ORG="gazebo-tooling" RELEASE_ORG="gazebo-release" @@ -211,7 +211,7 @@ cloneIfNeeded ${TOOLING_ORG} gazebodistro startFromCleanBranch bump_${COLLECTION} master # docs -cloneIfNeeded ${IGN_ORG} docs +cloneIfNeeded ${GZ_ORG} docs startFromCleanBranch bump_${COLLECTION} ${DOCS_BRANCH} # homebrew @@ -296,13 +296,13 @@ cd ${TEMP_DIR}/gazebodistro git reset --hard # Add collection to libraries, without version -LIBRARIES+=(ign-$COLLECTION) +LIBRARIES+=(gz-$COLLECTION) ################## # docs ################## cd ${TEMP_DIR}/docs -commitAndPR ${IGN_ORG} ${DOCS_BRANCH} +commitAndPR ${GZ_ORG} ${DOCS_BRANCH} for ((i = 0; i < "${#LIBRARIES[@]}"; i++)); do @@ -311,8 +311,8 @@ for ((i = 0; i < "${#LIBRARIES[@]}"; i++)); do LIB_=${LIB//-/_} # For fuel_tools VER=${VERSIONS[$i]} PREV_VER="$((${VER}-1))" - LIB_UPPER=`echo ${LIB#"ign-"} | tr a-z A-Z` - ORG=${IGN_ORG} + LIB_UPPER=`echo ${LIB#"gz-"} | tr a-z A-Z` + ORG=${GZ_ORG} BUMP_BRANCH="ci_matching_branch/bump_${COLLECTION}_${LIB}${VER}" echo -e "${BLUE_BG}Processing [${LIB}]${DEFAULT_BG}" @@ -329,7 +329,7 @@ for ((i = 0; i < "${#LIBRARIES[@]}"; i++)); do for ((j = 0; j < "${#LIBRARIES[@]}"; j++)); do - DEP_LIB=${LIBRARIES[$j]#"ign-"} + DEP_LIB=${LIBRARIES[$j]#"gz-"} DEP_VER=${VERSIONS[$j]} DEP_PREV_VER="$((${DEP_VER}-1))" @@ -347,8 +347,6 @@ for ((i = 0; i < "${#LIBRARIES[@]}"; i++)); do cd ${TEMP_DIR}/homebrew-simulation startFromCleanBranch ${BUMP_BRANCH} master - # expand ign-* to ignition-* - FORMULA_BASE=${LIB/ign/ignition} # construct path with major version suffix FORMULA="Formula/${FORMULA_BASE}${VER}.rb" if [ ! -f "$FORMULA" ]; then @@ -358,6 +356,7 @@ for ((i = 0; i < "${#LIBRARIES[@]}"; i++)); do # Collection if ! [[ $VER == ?(-)+([0-9]) ]] ; then + # TODO(CH3): Change this to gz once we migrate the homebrew formulae cp Formula/ignition-${PREV_COLLECTION}.rb $FORMULA else cp Formula/${FORMULA_BASE}${PREV_VER}.rb $FORMULA @@ -370,12 +369,12 @@ for ((i = 0; i < "${#LIBRARIES[@]}"; i++)); do URL="https://github.com/${ORG}/${LIB}.git" # libN - sed -i -E "s ((${LIB#"ign-"}))${PREV_VER} \1${VER} g" $FORMULA - sed -i -E "s ((${LIB_#"ign_"}))${PREV_VER} \1${VER} g" $FORMULA - # ign-libN -> main + sed -i -E "s ((${LIB#"gz-"}))${PREV_VER} \1${VER} g" $FORMULA + sed -i -E "s ((${LIB_#"gz_"}))${PREV_VER} \1${VER} g" $FORMULA + # gz-libN -> main sed -i "s ${LIB}${PREV_VER} main g" $FORMULA - # class IgnitionLibN - sed -i -E "s/((class Ignition.*))${PREV_VER}/\1${VER}/g" $FORMULA + # class GzLibN + sed -i -E "s/((class Gz.*))${PREV_VER}/\1${VER}/g" $FORMULA sed -i -E "s/((class Sdformat))${PREV_VER}/\1${VER}/g" $FORMULA # remove bottle - TODO: this is only needed for new formulae sed -i -e "/bottle do/,/end/d" $FORMULA @@ -394,7 +393,8 @@ for ((i = 0; i < "${#LIBRARIES[@]}"; i++)); do for ((j = 0; j < "${#LIBRARIES[@]}"; j++)); do - DEP_LIB=${LIBRARIES[$j]#"ign-"} + DEP_LIB=${LIBRARIES[$j]#"gz-"} + DEP_VER=${VERSIONS[$j]} DEP_PREV_VER="$((${DEP_VER}-1))" @@ -441,7 +441,6 @@ for ((i = 0; i < "${#LIBRARIES[@]}"; i++)); do # Check if main branch of that library is the correct version PROJECT_NAME="${LIB_}${VER}" - PROJECT_NAME="${PROJECT_NAME/ign_/ignition-}" PROJECT="project.*(${PROJECT_NAME}" if ! grep -q ${PROJECT} "CMakeLists.txt"; then echo -e "${RED}Wrong project name on [CMakeLists.txt], looking for [$PROJECT_NAME].${DEFAULT}" @@ -451,32 +450,33 @@ for ((i = 0; i < "${#LIBRARIES[@]}"; i++)); do echo -e "${GREEN}${LIB}: Updating source code${DEFAULT}" for ((j = 0; j < "${#LIBRARIES[@]}"; j++)); do - DEP_LIB=${LIBRARIES[$j]#"ign-"} + DEP_LIB=${LIBRARIES[$j]#"gz-"} + DEP_VER=${VERSIONS[$j]} DEP_PREV_VER="$((${DEP_VER}-1))" - # Replace lines like "find_package(ignition-cmake2 2.0.0)" - # with "find_package(ignition-cmake3)" + # Replace lines like "find_package(gz-cmake2 2.0.0)" + # with "find_package(gz-cmake3)" find . -type f -name 'CMakeLists.txt' -print0 | xargs -0 sed -i "s@\(find_package.*${DEP_LIB}\)${DEP_PREV_VER} \+${DEP_PREV_VER}[^ )]*@\1${DEP_VER}@g" - # Replace lines like "ign_find_package(ignition-math6 VERSION 6.5.0)" - # with "ign_find_package(ignition-math7)" + # Replace lines like "gz_find_package(gz-math6 VERSION 6.5.0)" + # with "gz_find_package(gz-math7)" # Preserves other args and handles edge cases: - # like "ign_find_package(ignition-math6 VERSION 6.5.0 REQUIRED)" - # with "ign_find_package(ignition-math6 REQUIRED)" - # like "ign_find_package(ignition-math6 REQUIRED COMPONENTS VERSION 6.10 eigen3)" - # with "ign_find_package(ignition-math7 REQUIRED COMPONENTS eigen3)" + # like "gz_find_package(gz-math6 VERSION 6.5.0 REQUIRED)" + # with "gz_find_package(gz-math6 REQUIRED)" + # like "gz_find_package(gz-math6 REQUIRED COMPONENTS VERSION 6.10 eigen3)" + # with "gz_find_package(gz-math7 REQUIRED COMPONENTS eigen3)" find . -type f -name 'CMakeLists.txt' -print0 | xargs -0 sed -i "s@\(find_package.*${DEP_LIB}\)${DEP_PREV_VER}\(.*\) \+VERSION \+${DEP_PREV_VER}[^ )]*@\1${DEP_VER}\2@g" # Rule: *plugin2 -> *plugin3 - # Replace lines like: "find_package(ignition-cmake2)" - # with: "find_package(ignition-cmake3)" + # Replace lines like: "find_package(gz-cmake2)" + # with: "find_package(gz-cmake3)" find . -type f ! -name 'Changelog.md' ! -name 'Migration.md' -print0 | xargs -0 sed -i "s ${DEP_LIB}${DEP_PREV_VER} ${DEP_LIB}${DEP_VER} g" # Replace collection yaml branch names with main - if [[ "${LIB}" == "ign-${COLLECTION}" ]]; then - find . -type f -name "collection-${COLLECTION}.yaml" -print0 | xargs -0 sed -i "s ign-${DEP_LIB}${DEP_VER} main g" + if [[ "${LIB}" == "gz-${COLLECTION}" ]]; then + find . -type f -name "collection-${COLLECTION}.yaml" -print0 | xargs -0 sed -i "s gz-${DEP_LIB}${DEP_VER} main g" fi # Second run with _ instead of -, to support multiple variations of fuel-tools From dfb3ef95337121df3c88508f6f9077a83170c0ff Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 23 Aug 2022 17:13:51 +0200 Subject: [PATCH 198/540] Improve creation of new unofficial Gazebo ROS wrappers (#197) * Improve creation of new unofficial Gazebo ROS wrappers * Improve bloom gazebo_ros_pkgs release procedure * Import documentation from gazebo_ros_pkgs --- bloom/README.md | 224 +++++++++++++++++- bloom/git_clone_all_remote_branches.bash | 5 - bloom/initial_info_for_new_release_repo.bash | 64 +++++ bloom/rename-gazebo-ros-pkgs.bash | 21 +- .../docker/lib/debbuild-bloom-base.bash | 7 + 5 files changed, 311 insertions(+), 10 deletions(-) delete mode 100644 bloom/git_clone_all_remote_branches.bash create mode 100755 bloom/initial_info_for_new_release_repo.bash diff --git a/bloom/README.md b/bloom/README.md index 999b583d2..43af4d34e 100644 --- a/bloom/README.md +++ b/bloom/README.md @@ -1,6 +1,8 @@ -## Scripts +# Scripts -### Release new version of Gazebo unofficial wrappers +## Release new version of Gazebo unofficial wrappers + +### Quick commands 1. bloom in the osrf/ repository ```bash @@ -17,3 +19,221 @@ Example: $ ros_gazebo_pkgs-release.py.bash 3.4.4 https://github.com/osrf/gazebo11_ros2_pkgs-release foxy xxx -r 1 --dry-run ``` + +### Background + +Each ROS release defines one version of Gazebo supported officially through +all the ROS packages. The different combinations of ROS <-> Gazebo can be +found in the [REP-3](http://www.ros.org/reps/rep-0003.html). Some examples: + + * ROS Kinetic: Gazebo 7 + * ROS Lunar: Gazebo 7 + * ROS Melodic: Gazebo 9 + +Some use cases require the use of alternative combinations of ROS and Gazebo +versions. The `gazebo_ros_pkgs` code is usually prepared to be compatible with +different versions of Gazebo, especially the latest ones. + +To release a modified version of `gazebo_ros_pkgs` which supports a different +major version of gazebo, before running bloom some actions need to be taken: + + 1. Make the system resolve the Ubuntu package `gazebo` as `gazeboX`. + 2. Fork and modify the official [gbp -release repo](https://github.com/ros-gbp/gazebo_ros_pkgs-release). + 3. Run bloom on the modified release repository. + 4. Custom infrastructure to create .deb packages. + +We'll go over these steps in more detail below. + +### Upstream versions released using this tutorial + +The `gbp -release repository` hosts the latest version released by the maintainers +of `gazebo_ros_pkgs`. When using these instructions to release a new custom version +the version of `gazebo_ros_pkgs` released will be the latest one existing in the +official `gbp -release repository`. The version would be the same but the release number +will be one above since bloom needs to be run on top of the modified `-release repository`. + +### Create a new gazeboX_{ros,ros2}_pkgs repository + +For a new official wrappers the notation used below correspond to: +`gazeboX_rosY_pkgs` (`X` is major version in Gazebo, `Y` is 1 or 2 for ROS) + + 1. Create new repo under github.com/osrf/ organization named: + - ROS1 https://github.com/osrf/gazeboX_ros_pkgs-release + - ROS2 https://github.com/osrf/gazeboX_ros2_pkgs-release + + 1. Run initial_info_for_new_release_repo.bash + It will import all branches and tags from official gbp into the new repo + - Usage: *$ initial_info_for_new_release_repo X rosY* + + 1. Clone the new repo, go to the directory and run rename-gazebo-ros-pkgs.bash + - Usage: *$ rename-gazebo-ros-pkgs.bash X * + +### Make the system resolve gazeboX + +To have `gazebo` packages be resolved to `gazeboX` in the local +machine where bloom-release will be run, `rosdep` should be configured +accordingly. + +The repository `https://github.com/osrf/osrf-rosdep/` hosts different sets +of custom rosdep keys that can be deployed in a system in order to override +the default configuration of resolving gazebo packages. + +At `/etc/ros/rosdep/sources.list.d` there should be a default file named +`20-default.list`. To change the default keys by a custom version of gazebo +packages, the desired `00-gazeboX.list` from the previously indicated repository +needs to be download to the same directory. + +Update the rosdep cache: + + $ rosdep update + +To double check: run rosdep resolve, it should resolve to right major versions: + + $ rosdep resolve gazebo + $ rosdep resolve gazebo_ros + + +### Run Bloom on the custom -release repository + +The point of this last step is to generate the ubuntu packaging metadata from bloom templates since they were +modified. The metadata will be uploaded to branches in the custom fork created before. + +`bloom-release` needs to be executed once per each desired `ROS_DISTRO` to generate metadata from. The name +`gazeboX_ros_pkgs` is just a stub to avoid use the official name: + + bloom-release --track ${ROS_DISTRO} --ros-distro ${ROS_DISTRO} gazeboX_ros_pkgs --edit + +Bloom will stop at some points asking for the user input: + + * `Release repository url [press enter to abort]:` the URL of the forked github repo needs to be set here + * `Repository Name:` `gazeboX_ros_pkgs` (being `X` the custom gazebo major version) + * The other options can be set to default just by pressing `enter` + +Bloom must be stopped after generating and pushing the new tags and branches. + +#### Example Releasing Gazebo8 for Kinetic + + $ bloom-release --track kinetic --ros-distro kinetic gazeboX_ros_pkgs --edit + + Specified repository 'gazeboX_ros_pkgs' is not in the distribution file located at 'https://raw.github.com/ros/rosdistro/master/kinetic/distribution.yaml' + Did you mean one of these: 'gazebo_ros_pkgs'? + Could not determine release repository url for repository 'gazeboX_ros_pkgs' of distro 'kinetic' + You can continue the release process by manually specifying the location of the RELEASE repository. + To be clear this is the url of the RELEASE repository not the upstream repository. + For release repositories on github, you should provide the `https://` url which should end in `.git`. + Release repository url [press enter to abort]: git@github.com:osrf/gazebo8_ros_pkgs-release.git + The release repository url you provided is not a `https://` address. + Would you like to enter the address again? + Continue [Y/n]? n + Very well, the address 'git@github.com:osrf/gazebo8_ros_pkgs-release.git' will be used as is. + ==> Fetching 'gazeboX_ros_pkgs' repository from 'git@github.com:osrf/gazebo8_ros_pkgs-release.git' + Cloning into '/tmp/tmpRGYh76'... + remote: Counting objects: 16388, done. + remote: Total 16388 (delta 0), reused 0 (delta 0) + Receiving objects: 100% (16388/16388), 4.02 MiB | 954.00 KiB/s, done. + Resolving deltas: 100% (4232/4232), done. + Checking connectivity... done. + Track 'kinetic' exists, editing... + Repository Name: + upstream + Default value, leave this as upstream if you are unsure + + Name of the repository (used in the archive name) + ['gazebo4_ros_pkgs']: + Upstream Repository URI: + + Any valid URI. This variable can be templated, for example an svn url + can be templated as such: "https://svn.foo.com/foo/tags/foo-:{version}" + where the :{version} token will be replaced with the version for this release. + ['https://github.com/ros-simulation/gazebo_ros_pkgs.git']: + Upstream VCS Type: + svn + Upstream URI is a svn repository + git + Upstream URI is a git repository + hg + Upstream URI is a hg repository + tar + Upstream URI is a tarball + ['git']: + Version: + :{ask} + This means that the user will be prompted for the version each release. + This also means that the upstream devel will be ignored. + :{auto} + This means the version will be guessed from the devel branch. + This means that the devel branch must be set, the devel branch must exist, + and there must be a valid package.xml in the upstream devel branch. + + This will be the version used. + It must be updated for each new upstream version. + [':{auto}']: + Release Tag: + :{none} + For svn and tar only you can set the release tag to :{none}, so that + it is ignored. For svn this means no revision number is used. + :{ask} + This means the user will be prompted for the release tag on each release. + :{version} + This means that the release tag will match the :{version} tag. + This can be further templated, for example: "foo-:{version}" or "v:{version}" + + This can describe any vcs reference. For git that means {tag, branch, hash}, + for hg that means {tag, branch, hash}, for svn that means a revision number. + For tar this value doubles as the sub directory (if the repository is + in foo/ of the tar ball, putting foo here will cause the contents of + foo/ to be imported to upstream instead of foo itself). + [':{version}']: + Upstream Devel Branch: + + Branch in upstream repository on which to search for the version. + This is used only when version is set to ':{auto}'. + ['kinetic-devel']: + ROS Distro: + + This can be any valid ROS distro, e.g. groovy, hydro + ['kinetic']: + Patches Directory: + :{none} + Use this if you want to disable overlaying of files. + + This can be any valid relative path in the bloom branch. The contents + of this folder will be overlaid onto the upstream branch after each + import-upstream. Additionally, any package.xml files found in the + overlay will have the :{version} string replaced with the current + version being released. + [None]: + Release Repository Push URL: + :{none} + This indicates that the default release url should be used. + + (optional) Used when pushing to remote release repositories. This is only + needed when the release uri which is in the rosdistro file is not writable. + This is useful, for example, when a releaser would like to use a ssh url + to push rather than a https:// url. + [None]: + Saving 'kinetic' track. + + ... (all bloom output while generating) ... + + <== Pushed tags successfully + ==> Generating pull request to distro file located at 'https://raw.github.com/ros/rosdistro/master/kinetic/distribution.yaml' + Would you like to add documentation information for this repository? [Y/n]? + + + +### Custom infrastructure to create .deb packages + +All the previous steps are designed to generate the appropriate Ubuntu metadata +inside the `forked -prerelease repository`. The metadata is being hosted in +git tags following the schema: + + + release/// + debian/ros--__ + +The `debian/...` tag contains the upstream code together with the `debian/` metadata repository ready to be +build using `debbuild` or any other debian generation tool. + +For reference, the OSRF buildfarm is using [this script](https://github.com/gazebo-tooling/release-tools/blob/master/jenkins-scripts/docker/lib/debbuild-bloom-base.bash) + diff --git a/bloom/git_clone_all_remote_branches.bash b/bloom/git_clone_all_remote_branches.bash deleted file mode 100644 index eb8bc1df2..000000000 --- a/bloom/git_clone_all_remote_branches.bash +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -# source: https://coderwall.com/p/0ypmka -for branch in `git branch -a | grep remotes | grep -v HEAD | grep -v master `; do - git branch --track ${branch#remotes/origin/} $branch -done diff --git a/bloom/initial_info_for_new_release_repo.bash b/bloom/initial_info_for_new_release_repo.bash new file mode 100755 index 000000000..9b8bcea69 --- /dev/null +++ b/bloom/initial_info_for_new_release_repo.bash @@ -0,0 +1,64 @@ +#!/bin/bash -e +# +# This script will populate an empty gazeboX_rosY_pkgs-release repository with +# information from current official -release repositories +# + +if [[ ${#} -lt 2 ]]; then + echo "Usage: ${0} [ros1|ros2]" + exit -1 +fi + +GZ_MAJOR=${1} +ROS_VERSION=${2} + +SOURCE_REPO="https://github.com/${ROS_VERSION}-gbp/gazebo_ros_pkgs-release" +DEST_REPO="https://github.com/osrf/gazebo${GZ_MAJOR}_${ROS_VERSION}_pkgs-release" +TEST_DIR=$(mktemp -d) + +safety_check() +{ + RAW_URL="${DEST_REPO/github.com/raw.githubusercontent.com}/master/tracks.yaml" + + if wget -q "${RAW_URL}"; then + echo "!! Looks like the ${DEST_REPO} exists!" + echo "!! this tool is only for new repos" + exit -1 + fi +} + +clone_all_remote_branches() +{ + # source: https://coderwall.com/p/0ypmka + for branch in `git branch -a | grep remotes | grep -v HEAD | grep -v master `; do + git branch --track ${branch#remotes/origin/} $branch + done +} + +echo "Summary: " +echo " - The source repo to use would be: ${SOURCE_REPO}" +echo " - The destination repo would be : ${DEST_REPO}" + +pushd ${TEST_DIR} +safety_check + +git clone "${DEST_REPO}" dest_repo +git clone "${SOURCE_REPO}" source_repo +cp source_repo/* dest_repo/ +cd dest_repo +git add * +git commit -m "Import metadata from gazebo_ros_pkgs repository" +git remote rename origin gazebo${GZ_MAJOR} +git remote add origin ${SOURCE_REPO} +git fetch origin +clone_all_remote_branches +git remote remove origin +git remote rename gazebo${GZ_MAJOR} origin +# 7. Push all the local branch and tags to the new repo +git push origin --all +git push origin --tags + +echo " ** READY" +echo " remember to run: " +echo " - rename-gazebo-ros-pkgs.bash ${GZ_MAJOR} " +popd 2> /dev/null diff --git a/bloom/rename-gazebo-ros-pkgs.bash b/bloom/rename-gazebo-ros-pkgs.bash index 47eea8e61..352ed3a45 100755 --- a/bloom/rename-gazebo-ros-pkgs.bash +++ b/bloom/rename-gazebo-ros-pkgs.bash @@ -25,10 +25,25 @@ PKGS="gazebo_dev gazebo_plugins gazebo_msgs gazebo_ros gazebo_ros_control gazebo for pkg in ${PKGS}; do for distro in ${ROS_DISTROS}; do echo " - Processing $pkg in $distro" - git checkout "debian/$distro/$pkg" + if ! git checkout "debian/$distro/$pkg"; then + if [[ ${pkg} == "gazebo_ros_control" ]]; then + echo " [??] gazebo_ros_control not found in the repository" + echo " [??] assuming not avilable in this platform" + continue + fi + echo "The branch debian/$distro/$pkg was not found in the repo" + echo "Did you forget to run git fetch?" + exit 1 + fi + if grep 'Package.replace' debian/control.em; then + echo " + skip ${pkg} for ${distro}: seems to have changes in place" + continue + fi # Modify package name - sed -i -e "s/Package: @(Package)/Package:@(Package.replace('gazebo-','gazebo${MAJOR_VERSION}-'))/" debian/control.em - git commit debian/control.em -m "Patch name to release ${MAJOR_VERSION} version" + sed -i -e "s/Package: @(Package)/Package: @(Package.replace('gazebo-','gazebo${MAJOR_VERSION}-'))/" debian/control.em + sed -i -e "s/Source: @(Package)/Source: @(Package.replace('gazebo-','gazebo${MAJOR_VERSION}-'))/" debian/control.em + sed -i -e "s/@(Package)/@(Package.replace('gazebo-','gazebo${MAJOR_VERSION}-'))/" debian/changelog.em + git commit debian/control.em debian/changelog.em -m "Patch name to release ${MAJOR_VERSION} version" # Include conflict with same package (not current) sed -i -e "/^Depends/aConflicts: @(Package) ${CONFLICTS}" debian/control.em git commit debian/control.em -m "Set up a conflict with official ROS packages and the two previous gazebo versions" diff --git a/jenkins-scripts/docker/lib/debbuild-bloom-base.bash b/jenkins-scripts/docker/lib/debbuild-bloom-base.bash index e758f40f6..7fea34302 100644 --- a/jenkins-scripts/docker/lib/debbuild-bloom-base.bash +++ b/jenkins-scripts/docker/lib/debbuild-bloom-base.bash @@ -98,6 +98,13 @@ PKGS=\`find .. -name '*.deb' || true\` FOUND_PKG=0 for pkg in \${PKGS}; do echo "found \$pkg" + # Check for problems in gazebo_ros_pkgs unofficial wrappers to avoid + # uploads with the same name than official ROS packages + # ros-DISTRO-gazebo-* instead of ros-DISTRO-gazeboX-* + if [[ \${pkg} != \${pkg/-gazebo-} ]]; then + echo "Detected official ROS names in gazebo_ros_pkgs" + exit -1 + fi # Check for correctly generated packages size > 3Kb [[ \$(find \$pkg -size +3k) ]] || echo "WARNING: empty package?" cp \${pkg} $WORKSPACE/pkgs From 1988bf3da2b701a472f148e1b7a0e835eddcf35e Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 19 Aug 2022 17:45:52 +0200 Subject: [PATCH 199/540] Export Jenkins XML configuration diffs for DSL changed files in the CI --- .github/workflows/ci.yaml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c927566ce..127b24092 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -50,3 +50,33 @@ jobs: sudo chown -R ${USER} /var/lib/jenkins cd jenkins-scripts/dsl java -jar ../../jobdsl.jar *.dsl + # export files for later diff + mkdir /tmp/pr_xml_configuration + mv *.xml /tmp/pr_xml_configuration/ + - name: Geneate master DSL files + if: steps.dsl_check.outputs.run_job == 'true' + run: | + git clean -f -e jobdsl.jar + git checkout master + cd jenkins-scripts/dsl + java -jar ../../jobdsl.jar *.dsl + mkdir /tmp/current_xml_configuration + mv *.xml /tmp/current_xml_configuration/ + - name: Generating diffs + if: steps.dsl_check.outputs.run_job == 'true' + run: | + # somehow the Jenkins views changed the portlet_ id on every run. + diff -qr -I '.*dashboard_portlet_.*.*' /tmp/current_xml_configuration /tmp/pr_xml_configuration > /tmp/xml_config_files_changed.diff || true + diff -ur -I '.*dashboard_portlet_.*.*' /tmp/current_xml_configuration /tmp/pr_xml_configuration > /tmp/xml_config_content_changed.diff || true + - name: Archive files changes + if: steps.dsl_check.outputs.run_job == 'true' + uses: actions/upload-artifact@v3 + with: + name: xml_config_files_changed + path: /tmp/xml_config_files_changed.diff + - name: Archive content changes + if: steps.dsl_check.outputs.run_job == 'true' + uses: actions/upload-artifact@v3 + with: + name: xml_config_content_changed + path: /tmp/xml_config_content_changed.diff From f716c771ea2cd129e349d02ad71d3371399ed3dc Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 24 Aug 2022 18:05:02 +0200 Subject: [PATCH 200/540] Use gz- for debbuilders related to garden --- jenkins-scripts/dsl/ignition.dsl | 2 +- jenkins-scripts/dsl/ignition_collection.dsl | 32 ++++++++++----------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index 9323b33e8..c6a75f0f8 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -235,7 +235,7 @@ ArrayList all_debbuilders() if ("${major_version}" == "0" || "${major_version}" == "1" ) major_version = "" - branches.add("ign-${software_name}${major_version}") + branches.add("gz-${software_name}${major_version}") } } } diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index f4c3266aa..008d70264 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -18,23 +18,23 @@ gz_collections = [ // These are the branches currently targeted at the upcoming collection // They're in topological order nightly_jobs: [ - 'cmake' : [ debbuild: 'ign-cmake3' , branch: 'gz-cmake3' ], - 'tools' : [ debbuild: 'ign-tools2' , branch: 'gz-tools2' ], - 'utils' : [ debbuild: 'ign-utils2' , branch: 'gz-utils2' ], - 'math' : [ debbuild: 'ign-math7' , branch: 'gz-math7' ], - 'plugin' : [ debbuild: 'ign-plugin2' , branch: 'gz-plugin2' ], - 'common' : [ debbuild: 'ign-common5' , branch: 'gz-common5' ], - 'msgs' : [ debbuild: 'ign-msgs9' , branch: 'gz-msgs9' ], - 'rendering' : [ debbuild: 'ign-rendering7' , branch: 'gz-rendering7' ], + 'cmake' : [ debbuild: 'gz-cmake3' , branch: 'gz-cmake3' ], + 'tools' : [ debbuild: 'gz-tools2' , branch: 'gz-tools2' ], + 'utils' : [ debbuild: 'gz-utils2' , branch: 'gz-utils2' ], + 'math' : [ debbuild: 'gz-math7' , branch: 'gz-math7' ], + 'plugin' : [ debbuild: 'gz-plugin2' , branch: 'gz-plugin2' ], + 'common' : [ debbuild: 'gz-common5' , branch: 'gz-common5' ], + 'msgs' : [ debbuild: 'gz-msgs9' , branch: 'gz-msgs9' ], + 'rendering' : [ debbuild: 'gz-rendering7' , branch: 'gz-rendering7' ], 'sdformat' : [ debbuild: 'sdformat13' , branch: 'sdf13' ], - 'fuel-tools': [ debbuild: 'ign-fuel-tools8' , branch: 'gz-fuel-tools8' ], - 'transport' : [ debbuild: 'ign-transport12' , branch: 'gz-transport12' ], - 'gui' : [ debbuild: 'ign-gui7' , branch: 'gz-gui7' ], - 'sensors' : [ debbuild: 'ign-sensors7' , branch: 'gz-sensors7' ], - 'physics' : [ debbuild: 'ign-physics6' , branch: 'gz-physics6' ], - 'gazebo' : [ debbuild: 'ign-gazebo7' , branch: 'gz-sim7' ], - 'launch' : [ debbuild: 'ign-launch6' , branch: 'gz-launch6' ], - 'garden' : [ debbuild: 'ign-garden' , branch: 'main' ], + 'fuel-tools': [ debbuild: 'gz-fuel-tools8' , branch: 'gz-fuel-tools8' ], + 'transport' : [ debbuild: 'gz-transport12' , branch: 'gz-transport12' ], + 'gui' : [ debbuild: 'gz-gui7' , branch: 'gz-gui7' ], + 'sensors' : [ debbuild: 'gz-sensors7' , branch: 'gz-sensors7' ], + 'physics' : [ debbuild: 'gz-physics6' , branch: 'gz-physics6' ], + 'gazebo' : [ debbuild: 'gz-gazebo7' , branch: 'gz-sim7' ], + 'launch' : [ debbuild: 'gz-launch6' , branch: 'gz-launch6' ], + 'garden' : [ debbuild: 'gz-garden' , branch: 'main' ], ], ], ] From f42f1c00df8a9bf1032914dc952d73dca304a92f Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 24 Aug 2022 23:28:44 +0200 Subject: [PATCH 201/540] Adapt release.py to work with Garden packages --- release.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/release.py b/release.py index 0ccdfd000..df6f8dbb1 100755 --- a/release.py +++ b/release.py @@ -41,6 +41,22 @@ IGNORE_DRY_RUN = True +GARDEN_IGN_PACKAGES = ['ign-cmake3', + 'ign-common5', + 'ign-fuel-tools8', + 'ign-sim7', + 'ign-gui7', + 'ign-launch6', + 'ign-math7', + 'ign-msgs9', + 'ign-physics6', + 'ign-plugin2', + 'ign-rendering7', + 'ign-sensors7', + 'ign-tools2', + 'ign-transport12', + 'ign-utils2'] + class ErrorNoPermsRepo(Exception): pass @@ -141,6 +157,10 @@ def parse_args(argv): args = parser.parse_args() + if args.package in GARDEN_IGN_PACKAGES: + print(f"Garden packages start with gz- prefix, changing {args.package} to {args.package.replace('ign-','gz-')}",) + args.package = args.package.replace('ign-','gz-') + args.package_alias = args.package # If ignition auto is enabled, replace ign- with ignition- at the beginning if not args.no_ignition_auto and args.package.startswith('ign-'): From 6214e1d4dd95108f6cef588385272992453df1f3 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 26 Aug 2022 00:07:06 +0200 Subject: [PATCH 202/540] Fix gz-sim nightly name in ignition_collection.dsl --- jenkins-scripts/dsl/ignition_collection.dsl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index 008d70264..d6b0514bb 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -26,13 +26,13 @@ gz_collections = [ 'common' : [ debbuild: 'gz-common5' , branch: 'gz-common5' ], 'msgs' : [ debbuild: 'gz-msgs9' , branch: 'gz-msgs9' ], 'rendering' : [ debbuild: 'gz-rendering7' , branch: 'gz-rendering7' ], - 'sdformat' : [ debbuild: 'sdformat13' , branch: 'sdf13' ], + 'sdformat' : [ debbuild: 'sdformat13' , branch: 'sdf13' ], 'fuel-tools': [ debbuild: 'gz-fuel-tools8' , branch: 'gz-fuel-tools8' ], 'transport' : [ debbuild: 'gz-transport12' , branch: 'gz-transport12' ], 'gui' : [ debbuild: 'gz-gui7' , branch: 'gz-gui7' ], 'sensors' : [ debbuild: 'gz-sensors7' , branch: 'gz-sensors7' ], 'physics' : [ debbuild: 'gz-physics6' , branch: 'gz-physics6' ], - 'gazebo' : [ debbuild: 'gz-gazebo7' , branch: 'gz-sim7' ], + 'gazebo' : [ debbuild: 'gz-sim7' , branch: 'gz-sim7' ], 'launch' : [ debbuild: 'gz-launch6' , branch: 'gz-launch6' ], 'garden' : [ debbuild: 'gz-garden' , branch: 'main' ], ], From 6073baffefee03d1e572fe853424162277c59812 Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Fri, 26 Aug 2022 09:10:25 -0700 Subject: [PATCH 203/540] brew CI: set PYTHONPATH (#803) Set PYTHONPATH to /usr/local/lib/python so we can find our python bindings. Signed-off-by: Steve Peters --- jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash | 2 ++ 1 file changed, 2 insertions(+) diff --git a/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash b/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash index 72421907d..62c18bed9 100644 --- a/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash +++ b/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash @@ -34,6 +34,8 @@ export HOMEBREW_PREFIX=/usr/local export HOMEBREW_CELLAR=${HOMEBREW_PREFIX}/Cellar export PATH=${HOMEBREW_PREFIX}/bin:$PATH +export PYTHONPATH=$PYTHONPATH:/usr/local/lib/python + # make verbose mode? MAKE_VERBOSE_STR="" if [[ ${MAKE_VERBOSE} ]]; then From f660988bc7d50674ffaee59d94c2e74bc2f201fe Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 26 Aug 2022 18:12:05 +0200 Subject: [PATCH 204/540] Replace gz-gazebo7 debbuilder by gz-sim7 --- jenkins-scripts/dsl/ignition.dsl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index c6a75f0f8..e9b0cfb27 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -229,6 +229,8 @@ ArrayList all_debbuilders() branches.add("ign-${software_name}${major_version}") } } + if (software_name == 'gazebo') + software_name = 'sim' supported_gz_branches("${software_name}").each { major_version -> if (major_version) { // No 1-debbuild versions, they use the unversioned job From 5d669df32c58dee48f5fcbc592eedfd99e7f238e Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Sat, 27 Aug 2022 11:55:16 -0700 Subject: [PATCH 205/540] Fix gz-sim nightly branch Signed-off-by: Steve Peters --- jenkins-scripts/dsl/ignition_collection.dsl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index d6b0514bb..0a0d0ed8d 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -558,6 +558,8 @@ nightly_scheduler_job.with src_branch="${sensors_branch}" elif [[ "\${n}" != "\${n/sdformat/}" ]]; then src_branch="${sdformat_branch}" + elif [[ "\${n}" != "\${n/sim/}" ]]; then + src_branch="${gazebo_branch}" elif [[ "\${n}" != "\${n/transport/}" ]]; then src_branch="${transport_branch}" elif [[ "\${n}" != "\${n/tools/}" ]]; then From 3bda6d65bdbd0392a45862865a4294db254f8ec0 Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Sat, 27 Aug 2022 11:57:53 -0700 Subject: [PATCH 206/540] garden view: use sdf13 CI jobs This was missed during #793. Signed-off-by: Steve Peters --- jenkins-scripts/dsl/ignition_collection.dsl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index 0a0d0ed8d..9e73740b0 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -282,9 +282,9 @@ gz_collection_jobs = 'ignition_utils-ci-gz-utils2-homebrew-amd64', 'ignition_utils-install-pkg-focal-amd64', 'ignition_utils1-install_bottle-homebrew-amd64', - 'sdformat-ci-main-focal-amd64', - 'sdformat-ci-main-homebrew-amd64', - 'sdformat-ci-main-windows7-amd64', + 'sdformat-ci-sdformat13-focal-amd64', + 'sdformat-ci-sdformat13-homebrew-amd64', + 'sdformat-ci-sdformat13-windows7-amd64', 'sdformat-install-sdformat12_pkg-focal-amd64' ], ] From 4ec0590c1114a9feb323e7bd54a3a27ea83209e9 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 25 Aug 2022 18:32:44 +0200 Subject: [PATCH 207/540] Geneate gz- -install- jobs for testing gz- pkgs --- jenkins-scripts/dsl/ignition.dsl | 63 ++++++++++++++++++-------------- 1 file changed, 36 insertions(+), 27 deletions(-) diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index e9b0cfb27..926e7b65b 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -418,6 +418,37 @@ gz_software.each { gz_sw -> } } +void generate_install_job(prefix, gz_sw, major_version, distro, arch) +{ + def install_default_job = job("${prefix}_${gz_sw}${major_version}-install-pkg-${distro}-${arch}") + OSRFLinuxInstall.create(install_default_job) + include_gpu_label_if_needed(install_default_job, gz_sw) + + install_default_job.with + { + triggers { + cron(Globals.CRON_EVERY_THREE_DAYS) + } + + def dev_package = "lib${prefix}-${gz_sw}${major_version}-dev" + def gzdev_project = "${prefix}-${gz_sw}${major_version}" + + steps { + shell("""\ + #!/bin/bash -xe + + ${GLOBAL_SHELL_CMD} + + export DISTRO=${distro} + export ARCH=${arch} + export INSTALL_JOB_PKG=${dev_package} + export GZDEV_PROJECT_NAME="${gzdev_project}" + /bin/bash -x ./scripts/jenkins-scripts/docker/generic-install-test-job.bash + """.stripIndent()) + } + } +} + // INSTALL PACKAGE ALL PLATFORMS / DAILY gz_software.each { gz_sw -> // Exclusion list @@ -439,33 +470,11 @@ gz_software.each { gz_sw -> major_version = "" // -------------------------------------------------------------- - def install_default_job = job("ignition_${gz_sw}${major_version}-install-pkg-${distro}-${arch}") - OSRFLinuxInstall.create(install_default_job) - include_gpu_label_if_needed(install_default_job, gz_sw) - - install_default_job.with - { - triggers { - cron(Globals.CRON_EVERY_THREE_DAYS) - } - - def dev_package = "libignition-${gz_sw}${major_version}-dev" - def gzdev_project = "ignition-${gz_sw}${major_version}" - - steps { - shell("""\ - #!/bin/bash -xe - - ${GLOBAL_SHELL_CMD} - - export DISTRO=${distro} - export ARCH=${arch} - export INSTALL_JOB_PKG=${dev_package} - export GZDEV_PROJECT_NAME="${gzdev_project}" - /bin/bash -x ./scripts/jenkins-scripts/docker/generic-install-test-job.bash - """.stripIndent()) - } - } + // ignition_ prefix packages: + generate_install_job("ignition", gz_sw, major_version, distro, arch) + // -------------------------------------------------------------- + // gz_ prefix packages: + generate_install_job("gz", gz_sw, major_version, distro, arch) } } } From 64e97de92d89e58b1172febd6cfcdc4ff7f8161a Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 25 Aug 2022 20:28:23 +0200 Subject: [PATCH 208/540] Fix gz-gazebo to gz-sim --- jenkins-scripts/dsl/ignition.dsl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index 926e7b65b..5071a6ba0 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -474,7 +474,7 @@ gz_software.each { gz_sw -> generate_install_job("ignition", gz_sw, major_version, distro, arch) // -------------------------------------------------------------- // gz_ prefix packages: - generate_install_job("gz", gz_sw, major_version, distro, arch) + generate_install_job("gz", gz_sw.replace('gazebo', 'sim'), major_version, distro, arch) } } } From 3c1ad39ffa2cd82a2e6a680e643a573e0e1c3198 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 25 Aug 2022 20:35:15 +0200 Subject: [PATCH 209/540] Add gz- jobs for collections --- jenkins-scripts/dsl/ignition_collection.dsl | 59 +++++++++++---------- 1 file changed, 32 insertions(+), 27 deletions(-) diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index 9e73740b0..c7d0caad5 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -291,6 +291,36 @@ gz_collection_jobs = def DISABLE_TESTS = false +void generate_install_job(prefix, gz_collection_name, distro, arch) +{ + def install_default_job = job("${prefix}_${gz_collection_name}-install-pkg-${distro}-${arch}") + OSRFLinuxInstall.create(install_default_job) + + install_default_job.with + { + triggers { + cron(Globals.CRON_EVERY_THREE_DAYS) + } + + def dev_package = "${prefix}-${gz_collection_name}" + def job_name = 'ign_launch-install-test-job.bash' + + label "gpu-reliable" + + steps { + shell("""\ + #!/bin/bash -xe + + export DISTRO=${distro} + export ARCH=${arch} + export INSTALL_JOB_PKG=${dev_package} + export GZDEV_PROJECT_NAME="${dev_package}" + /bin/bash -x ./scripts/jenkins-scripts/docker/${job_name} + """.stripIndent()) + } + } +} + // Testing compilation from source gz_collections.each { gz_collection -> // COLCON - Windows @@ -313,33 +343,8 @@ gz_collections.each { gz_collection -> gz_collection.get('distros').each { distro -> // INSTALL JOBS: // -------------------------------------------------------------- - def install_default_job = job("ignition_${gz_collection_name}-install-pkg-${distro}-${arch}") - OSRFLinuxInstall.create(install_default_job) - - install_default_job.with - { - triggers { - cron(Globals.CRON_EVERY_THREE_DAYS) - } - - def dev_package = "ignition-${gz_collection_name}" - - label "gpu-reliable" - - def job_name = 'ign_launch-install-test-job.bash' - - steps { - shell("""\ - #!/bin/bash -xe - - export DISTRO=${distro} - export ARCH=${arch} - export INSTALL_JOB_PKG=${dev_package} - export GZDEV_PROJECT_NAME="${dev_package}" - /bin/bash -x ./scripts/jenkins-scripts/docker/${job_name} - """.stripIndent()) - } - } + generate_install_job('ignition', gz_collection_name, distro, arch) + generate_install_job('gz', gz_collection_name, distro, arch) } // MAC Brew CI job From 96d51b21b04f32c7186ba97d5dc12808ee67151f Mon Sep 17 00:00:00 2001 From: Michael Carroll Date: Mon, 29 Aug 2022 20:55:54 -0500 Subject: [PATCH 210/540] Fix OGRE paths for the ABI checker (#809) * Fix OGRE paths for the ABI checker * Disable weird logic for garden Signed-off-by: Michael Carroll Signed-off-by: Steve Peters Co-authored-by: Steve Peters --- .../docker/ignition-abichecker.bash | 30 +++++++++++++------ 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/jenkins-scripts/docker/ignition-abichecker.bash b/jenkins-scripts/docker/ignition-abichecker.bash index 2dd18646a..bb5a2cf93 100644 --- a/jenkins-scripts/docker/ignition-abichecker.bash +++ b/jenkins-scripts/docker/ignition-abichecker.bash @@ -48,17 +48,29 @@ then fi # check if OGRE-2.2 include paths are needed -if [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-gazebo" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -ge 6 ]] || \ - [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-gui" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -ge 6 ]] || \ - [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-launch" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -ge 5 ]] || \ - [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-sensors" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -ge 6 ]] || \ - [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-rendering" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -ge 6 ]] +if [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-gazebo" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -eq 6 ]] || \ + [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-gui" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -eq 6 ]] || \ + [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-launch" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -eq 5 ]] || \ + [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-sensors" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -eq 6 ]] || \ + [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-rendering" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -eq 6 ]] then - export EXTRA_INCLUDES=""" - - /usr/include/OGRE-2.2/Hlms/Common - + + # OGRE 2.2 is packaged as "Ogre-Next" on jammy + if [[ "${DISTRO}" == "jammy" ]] + then + export EXTRA_INCLUDES=""" + + /usr/include/OGRE-Next/Hlms/Common + +""" + elif [[ "${DISTRO}" == "focal" ]] + then + export EXTRA_INCLUDES=""" + + /usr/include/OGRE-2.2/Hlms/Common + """ + fi fi # default to use stable repos From 2c51df5c8f03d16ed58a8784e7be212e17061f22 Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Tue, 30 Aug 2022 10:04:26 -0700 Subject: [PATCH 211/540] release collection: stable CI jobs, update view (#807) Signed-off-by: Steve Peters Signed-off-by: Steve Peters --- .github/ISSUE_TEMPLATE/release_collection.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/ISSUE_TEMPLATE/release_collection.md b/.github/ISSUE_TEMPLATE/release_collection.md index 7465d0152..77e5367e9 100644 --- a/.github/ISSUE_TEMPLATE/release_collection.md +++ b/.github/ISSUE_TEMPLATE/release_collection.md @@ -24,6 +24,9 @@ When opening PRs, add a link back to this issue for easier tracking. - [ ] Merge each library forward from previous versions * - [ ] Create stable branches off of `main` + - [ ] `New branch` button on branches page ([example](https://github.com/gazebosim/sdformat/branches)) + - [ ] Enable CI for stable branches ([example](https://github.com/gazebo-tooling/release-tools/pull/787)) + - [ ] Use stable branch jobs in Jenkins view ([example part 1](https://github.com/gazebo-tooling/release-tools/pull/793) and [part 2](https://github.com/gazebo-tooling/release-tools/pull/806)) - [ ] Pull requests previously targeted at `main`: - Bug fixes: - Retarget to the new stable branch. From b03bbd34f887cd15ebb320ec20b879463b674b5c Mon Sep 17 00:00:00 2001 From: Louise Poubel Date: Tue, 30 Aug 2022 11:53:11 -0700 Subject: [PATCH 212/540] Dashboard for Garden pre-release Signed-off-by: Louise Poubel --- terminal-dashboard/_dashboard_lib.sh | 32 ++++++++++++++-------------- terminal-dashboard/table.bash | 3 ++- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/terminal-dashboard/_dashboard_lib.sh b/terminal-dashboard/_dashboard_lib.sh index 6ec8f1d64..375d7e9a0 100644 --- a/terminal-dashboard/_dashboard_lib.sh +++ b/terminal-dashboard/_dashboard_lib.sh @@ -45,23 +45,23 @@ get_libraries_by_collection() ignition-fortress" elif [ "$COLLECTION" = "garden" ]; then LIBS=" - ignition-cmake2 - ignition-math7 - ignition-utils1 - ignition-tools - ignition-common5 - ignition-msgs9 - ignition-transport12 - ignition-fuel-tools8 - ignition-plugin - ignition-rendering7 + gz-cmake3 + gz-math7 + gz-utils2 + gz-tools2 + gz-common5 + gz-msgs9 + gz-transport12 + gz-fuel-tools8 + gz-plugin2 + gz-rendering7 sdformat13 - ignition-physics6 - ignition-sensors7 - ignition-gui7 - ignition-gazebo7 - ignition-launch6 - ignition-garden" + gz-physics6 + gz-sensors7 + gz-gui7 + gz-gazebo7 + gz-launch6 + gz-garden" else return 1 fi diff --git a/terminal-dashboard/table.bash b/terminal-dashboard/table.bash index bb319c340..e47d412d5 100755 --- a/terminal-dashboard/table.bash +++ b/terminal-dashboard/table.bash @@ -30,7 +30,8 @@ RED="\e[101m" ARCHS=( "amd64") DISTROS=( "ubuntu" ) -if [[ $PACKAGE_REPO != "nightly" ]]; then +# No nightlies or pre-releases for arm +if [[ $PACKAGE_REPO == "stable" ]]; then ARCHS+=( "i386" "arm64" "armhf") # No debian version supported across the stack right now # DISTROS+=( "debian" ) From a3a9c65651bd78247cf95ede709158668f5f0448 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 30 Aug 2022 19:12:20 +0200 Subject: [PATCH 213/540] Migrate ign-cmake to use colcon --- jenkins-scripts/dsl/ignition.dsl | 3 ++- jenkins-scripts/dsl/ignition_collection.dsl | 6 +++--- jenkins-scripts/ign_cmake-default-devel-windows-amd64.bat | 4 ++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index 5071a6ba0..d12871450 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -92,7 +92,8 @@ gz_prerelease_pkgs = [ 'placeholder' : [ ] // packages using colcon for windows compilation while migrating all them to // this solution -gz_colcon_win = [ 'common', +gz_colcon_win = [ 'cmake', + 'common', 'fuel-tools', 'gazebo', 'gui', diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index c7d0caad5..eeafa81c4 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -42,6 +42,7 @@ gz_collections = [ gz_collection_jobs = [ 'citadel' : [ + 'ign_cmake-ign-2-win', 'ign_common-ign-3-win', 'ign_fuel-tools-ign-4-win', 'ign_gazebo-ign-3-win', @@ -59,7 +60,6 @@ gz_collection_jobs = 'ignition_citadel-install_bottle-homebrew-amd64', 'ignition_cmake-ci-ign-cmake2-focal-amd64', 'ignition_cmake-ci-ign-cmake2-homebrew-amd64', - 'ignition_cmake-ci-ign-cmake2-windows7-amd64', 'ignition_cmake2-install-pkg-focal-amd64', 'ignition_cmake2-install_bottle-homebrew-amd64', 'ignition_common-ci-ign-common3-focal-amd64', @@ -120,6 +120,7 @@ gz_collection_jobs = 'sdformat-install-sdformat9_pkg-focal-amd64' ], 'fortress' : [ + 'ign_cmake-ign-2-win', 'ign_common-ign-4-win', 'ign_fuel-tools-ign-7-win', 'ign_gazebo-ign-6-win', @@ -136,7 +137,6 @@ gz_collection_jobs = 'ign_utils-ign-1-win', 'ignition_cmake-ci-ign-cmake2-focal-amd64', 'ignition_cmake-ci-ign-cmake2-homebrew-amd64', - 'ignition_cmake-ci-ign-cmake2-windows7-amd64', 'ignition_cmake2-install-pkg-focal-amd64', 'ignition_cmake2-install_bottle-homebrew-amd64', 'ignition_common-ci-ign-common4-focal-amd64', @@ -204,6 +204,7 @@ gz_collection_jobs = 'sdformat-install-sdformat12_pkg-focal-amd64' ], 'garden' : [ + 'ign_cmake-gz-3-win', 'ign_common-gz-5-win', 'ign_fuel-tools-gz-8-win', 'ign_gazebo-ci-win', @@ -220,7 +221,6 @@ gz_collection_jobs = 'ign_utils-gz-2-win', 'ignition_cmake-ci-gz-cmake3-focal-amd64', 'ignition_cmake-ci-gz-cmake3-homebrew-amd64', - 'ignition_cmake-ci-gz-cmake3-windows7-amd64', 'ignition_cmake2-install-pkg-focal-amd64', 'ignition_cmake2-install_bottle-homebrew-amd64', 'ignition_common-ci-gz-common5-focal-amd64', diff --git a/jenkins-scripts/ign_cmake-default-devel-windows-amd64.bat b/jenkins-scripts/ign_cmake-default-devel-windows-amd64.bat index b8ca4929c..2a6f74c5f 100644 --- a/jenkins-scripts/ign_cmake-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_cmake-default-devel-windows-amd64.bat @@ -4,6 +4,6 @@ set VCS_DIRECTORY=gz-cmake set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true -set EXTRA_CMAKE_TEST_ARGS=-DBUILDSYSTEM_TESTING:BOOL=True +set COLCON_EXTRA_CMAKE_ARGS=-DBUILDSYSTEM_TESTING:BOOL=True -call "%SCRIPT_DIR%/lib/generic-default-devel-windows.bat" +call "%SCRIPT_DIR%\lib\colcon-default-devel-windows.bat" From 3e61ba8c96fb1c1b0f9d8f96e3e463f935551b34 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 30 Aug 2022 19:47:01 +0200 Subject: [PATCH 214/540] Need colcon package name --- jenkins-scripts/ign_cmake-default-devel-windows-amd64.bat | 3 +++ 1 file changed, 3 insertions(+) diff --git a/jenkins-scripts/ign_cmake-default-devel-windows-amd64.bat b/jenkins-scripts/ign_cmake-default-devel-windows-amd64.bat index 2a6f74c5f..ff6c1a15d 100644 --- a/jenkins-scripts/ign_cmake-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_cmake-default-devel-windows-amd64.bat @@ -4,6 +4,9 @@ set VCS_DIRECTORY=gz-cmake set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true +:: This needs to be migrated to DSL to get multi-major versions correctly +set COLCON_PACKAGE=gz-cmake +set COLCON_AUTO_MAJOR_VERSION=true set COLCON_EXTRA_CMAKE_ARGS=-DBUILDSYSTEM_TESTING:BOOL=True call "%SCRIPT_DIR%\lib\colcon-default-devel-windows.bat" From 738d6c96adaeb5ee41fee9fab3fd9167fbd4c244 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 30 Aug 2022 19:56:03 +0200 Subject: [PATCH 215/540] Do not ignore global COLCON_EXTRA_CMAKE_ARGS --- jenkins-scripts/lib/windows_library.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/lib/windows_library.bat b/jenkins-scripts/lib/windows_library.bat index 1b5b90990..fb2836026 100644 --- a/jenkins-scripts/lib/windows_library.bat +++ b/jenkins-scripts/lib/windows_library.bat @@ -221,7 +221,7 @@ echo # BEGIN SECTION: colcon compilation without test for dependencies of !COLCO call :_colcon_build_cmd --packages-skip !COLCON_PACKAGE! "-DBUILD_TESTING=0" "-DCMAKE_CXX_FLAGS=-w" echo # END SECTION echo # BEGIN SECTION: colcon compilation with tests for !COLCON_PACKAGE! -call :_colcon_build_cmd --packages-select !COLCON_PACKAGE! " -DBUILD_TESTING=1" +call :_colcon_build_cmd --packages-select !COLCON_PACKAGE! " %COLCON_EXTRA_CMAKE_ARGS%" " -DBUILD_TESTING=1" echo # END SECTION goto :EOF From 7ee2f4fbada3a7af3738808115cb05237836b5fd Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 30 Aug 2022 20:02:36 +0200 Subject: [PATCH 216/540] Fix variable expansion --- jenkins-scripts/lib/windows_library.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/lib/windows_library.bat b/jenkins-scripts/lib/windows_library.bat index fb2836026..c7514812c 100644 --- a/jenkins-scripts/lib/windows_library.bat +++ b/jenkins-scripts/lib/windows_library.bat @@ -221,7 +221,7 @@ echo # BEGIN SECTION: colcon compilation without test for dependencies of !COLCO call :_colcon_build_cmd --packages-skip !COLCON_PACKAGE! "-DBUILD_TESTING=0" "-DCMAKE_CXX_FLAGS=-w" echo # END SECTION echo # BEGIN SECTION: colcon compilation with tests for !COLCON_PACKAGE! -call :_colcon_build_cmd --packages-select !COLCON_PACKAGE! " %COLCON_EXTRA_CMAKE_ARGS%" " -DBUILD_TESTING=1" +call :_colcon_build_cmd --packages-select !COLCON_PACKAGE! !COLCON_EXTRA_CMAKE_ARGS! " -DBUILD_TESTING=1" echo # END SECTION goto :EOF From 90cd25466c2e2b82a9adcaed8a8dabada2780023 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 31 Aug 2022 01:34:26 +0200 Subject: [PATCH 217/540] Real passing of parameter --- jenkins-scripts/ign_cmake-default-devel-windows-amd64.bat | 2 +- jenkins-scripts/lib/colcon-default-devel-windows.bat | 3 ++- jenkins-scripts/lib/windows_library.bat | 4 +++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/jenkins-scripts/ign_cmake-default-devel-windows-amd64.bat b/jenkins-scripts/ign_cmake-default-devel-windows-amd64.bat index ff6c1a15d..bc8c5c810 100644 --- a/jenkins-scripts/ign_cmake-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_cmake-default-devel-windows-amd64.bat @@ -7,6 +7,6 @@ set IGN_CLEAN_WORKSPACE=true :: This needs to be migrated to DSL to get multi-major versions correctly set COLCON_PACKAGE=gz-cmake set COLCON_AUTO_MAJOR_VERSION=true -set COLCON_EXTRA_CMAKE_ARGS=-DBUILDSYSTEM_TESTING:BOOL=True +set COLCON_PACKAGE_EXTRA_CMAKE_ARGS=-DBUILDSYSTEM_TESTING:BOOL=True call "%SCRIPT_DIR%\lib\colcon-default-devel-windows.bat" diff --git a/jenkins-scripts/lib/colcon-default-devel-windows.bat b/jenkins-scripts/lib/colcon-default-devel-windows.bat index 7c2aac7a7..9a47b3ab3 100644 --- a/jenkins-scripts/lib/colcon-default-devel-windows.bat +++ b/jenkins-scripts/lib/colcon-default-devel-windows.bat @@ -5,6 +5,7 @@ :: - GAZEBODISTRO_FILE : (optional) vcs yaml file in the gazebodistro repository :: - COLCON_PACKAGE : package name to test in colcon ws :: - COLCON_AUTO_MAJOR_VERSION (default false): auto detect major version from CMakeLists +:: - COLCON_PACKAGE_EXTRA_CMAKE_ARGS : (optional) CMake arg to inject into colcon :: - BUILD_TYPE : (default Release) [ Release | Debug ] Build type to use :: - DEPEN_PKGS : (optional) list of dependencies (separted by spaces) :: - KEEP_WORKSPACE : (optional) true | false. Clean workspace at the end @@ -123,7 +124,7 @@ if exist %LOCAL_WS_SOFTWARE_DIR%\configure.bat ( echo # BEGIN SECTION: compiling %VCS_DIRECTORY% cd %LOCAL_WS% -call %win_lib% :build_workspace !COLCON_PACKAGE! || goto :error +call %win_lib% :build_workspace !COLCON_PACKAGE! !COLCON_PACKAGE_EXTRA_CMAKE_ARGS! || goto :error echo # END SECTION if "%ENABLE_TESTS%" == "TRUE" ( diff --git a/jenkins-scripts/lib/windows_library.bat b/jenkins-scripts/lib/windows_library.bat index c7514812c..228bc4365 100644 --- a/jenkins-scripts/lib/windows_library.bat +++ b/jenkins-scripts/lib/windows_library.bat @@ -206,9 +206,11 @@ goto :EOF :: Build all the workspaces packages except the package provided in arg1 :: :: arg1: name of the colcon package excluded from building +:: arg2: extra cmake parameter to pass to the target package COLCON_PACKAGE :build_workspace set COLCON_PACKAGE=%1 +set _COLCON_EXTRA_CMAKE_ARGS=%2 :: Check if package is in colcon workspace echo # BEGIN SECTION Packages in workspace: @@ -221,7 +223,7 @@ echo # BEGIN SECTION: colcon compilation without test for dependencies of !COLCO call :_colcon_build_cmd --packages-skip !COLCON_PACKAGE! "-DBUILD_TESTING=0" "-DCMAKE_CXX_FLAGS=-w" echo # END SECTION echo # BEGIN SECTION: colcon compilation with tests for !COLCON_PACKAGE! -call :_colcon_build_cmd --packages-select !COLCON_PACKAGE! !COLCON_EXTRA_CMAKE_ARGS! " -DBUILD_TESTING=1" +call :_colcon_build_cmd --packages-select !COLCON_PACKAGE! %_COLCON_EXTRA_CMAKE_ARGS% " -DBUILD_TESTING=1" echo # END SECTION goto :EOF From 5ebf79125ad74c5ad3d0ee1694bbec5fe67caf32 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 31 Aug 2022 01:38:08 +0200 Subject: [PATCH 218/540] Use quotes --- jenkins-scripts/ign_cmake-default-devel-windows-amd64.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/ign_cmake-default-devel-windows-amd64.bat b/jenkins-scripts/ign_cmake-default-devel-windows-amd64.bat index bc8c5c810..99e0e8373 100644 --- a/jenkins-scripts/ign_cmake-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_cmake-default-devel-windows-amd64.bat @@ -7,6 +7,6 @@ set IGN_CLEAN_WORKSPACE=true :: This needs to be migrated to DSL to get multi-major versions correctly set COLCON_PACKAGE=gz-cmake set COLCON_AUTO_MAJOR_VERSION=true -set COLCON_PACKAGE_EXTRA_CMAKE_ARGS=-DBUILDSYSTEM_TESTING:BOOL=True +set COLCON_PACKAGE_EXTRA_CMAKE_ARGS="-DBUILDSYSTEM_TESTING:BOOL=True" call "%SCRIPT_DIR%\lib\colcon-default-devel-windows.bat" From 51591aec6ef9ee2f50b1c09ec0bc7af9ec0f3aee Mon Sep 17 00:00:00 2001 From: Louise Poubel Date: Wed, 31 Aug 2022 16:38:16 -0700 Subject: [PATCH 219/540] table.sh: gz-gazebo -> gz-sim Signed-off-by: Louise Poubel --- terminal-dashboard/_dashboard_lib.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terminal-dashboard/_dashboard_lib.sh b/terminal-dashboard/_dashboard_lib.sh index 375d7e9a0..92d393246 100644 --- a/terminal-dashboard/_dashboard_lib.sh +++ b/terminal-dashboard/_dashboard_lib.sh @@ -59,7 +59,7 @@ get_libraries_by_collection() gz-physics6 gz-sensors7 gz-gui7 - gz-gazebo7 + gz-sim7 gz-launch6 gz-garden" else From 5ddddd0668e3592e3a479954fa79defab2412b7b Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 31 Aug 2022 23:46:05 +0200 Subject: [PATCH 220/540] Fix bug for node labels in gz-physics when using release.py --- release.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/release.py b/release.py index df6f8dbb1..d862aeb00 100755 --- a/release.py +++ b/release.py @@ -641,7 +641,8 @@ def go(argv): # Need to use JENKINS_NODE_TAG parameter for large memory nodes # since it runs qemu emulation linux_platform_params['JENKINS_NODE_TAG'] = 'linux-' + a - elif ('ignition-physics' in args.package_alias): + elif ('ignition-physics' in args.package_alias) or \ + ('gz-physics' in args.package_alias): linux_platform_params['JENKINS_NODE_TAG'] = 'large-memory' if (NIGHTLY and a == 'i386'): From e6d158b01c72cbd523add51dd15c2729e8eba4aa Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Thu, 1 Sep 2022 08:27:46 -0700 Subject: [PATCH 221/540] Use ign-garden debbuilder name (#810) Signed-off-by: Steve Peters Signed-off-by: Steve Peters --- jenkins-scripts/dsl/ignition_collection.dsl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index eeafa81c4..049f58617 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -34,7 +34,7 @@ gz_collections = [ 'physics' : [ debbuild: 'gz-physics6' , branch: 'gz-physics6' ], 'gazebo' : [ debbuild: 'gz-sim7' , branch: 'gz-sim7' ], 'launch' : [ debbuild: 'gz-launch6' , branch: 'gz-launch6' ], - 'garden' : [ debbuild: 'gz-garden' , branch: 'main' ], + 'garden' : [ debbuild: 'ign-garden' , branch: 'main' ], ], ], ] From 96674dedf1638635b52a05ee83eeb01ae424e26d Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Thu, 1 Sep 2022 08:30:46 -0700 Subject: [PATCH 222/540] release.py: fix tarball name for fuel_tools (#817) Signed-off-by: Steve Peters Signed-off-by: Steve Peters --- release.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/release.py b/release.py index d862aeb00..521289c3d 100755 --- a/release.py +++ b/release.py @@ -438,6 +438,8 @@ def create_tarball_path(tarball_name, version, builddir, dry_run): if (not dry_run): if not os.path.isfile(alt_tarball_path): error("Can not find a tarball at: " + tarball_path + " or at " + alt_tarball_path) + else: + tarball_fname = alt_tarball_fname tarball_path = alt_tarball_path out, err = check_call(['shasum', '--algorithm', '256', tarball_path]) From 694d401c09118cfa0e87d22b8a960128ce61e5e9 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 30 Aug 2022 19:43:00 +0200 Subject: [PATCH 223/540] Use short names for Windows sdformat DSL jobs --- jenkins-scripts/dsl/ignition_collection.dsl | 9 +++++---- jenkins-scripts/dsl/sdformat.dsl | 5 +++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index 049f58617..11fd1c87d 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -118,6 +118,7 @@ gz_collection_jobs = 'sdformat-ci-sdformat9-homebrew-amd64', 'sdformat-ci-sdformat9-windows7-amd64', 'sdformat-install-sdformat9_pkg-focal-amd64' + 'sdformat-sdf9-win' ], 'fortress' : [ 'ign_cmake-ign-2-win', @@ -200,8 +201,8 @@ gz_collection_jobs = 'ignition_utils1-install_bottle-homebrew-amd64', 'sdformat-ci-sdformat12-focal-amd64', 'sdformat-ci-sdformat12-homebrew-amd64', - 'sdformat-ci-sdformat12-windows7-amd64', - 'sdformat-install-sdformat12_pkg-focal-amd64' + 'sdformat-install-sdformat12_pkg-focal-amd64', + 'sdformat-sdf12-win.xml' ], 'garden' : [ 'ign_cmake-gz-3-win', @@ -284,8 +285,8 @@ gz_collection_jobs = 'ignition_utils1-install_bottle-homebrew-amd64', 'sdformat-ci-sdformat13-focal-amd64', 'sdformat-ci-sdformat13-homebrew-amd64', - 'sdformat-ci-sdformat13-windows7-amd64', - 'sdformat-install-sdformat12_pkg-focal-amd64' + 'sdformat-install-sdformat12_pkg-focal-amd64', + 'sdformat-sdf13-win' ], ] diff --git a/jenkins-scripts/dsl/sdformat.dsl b/jenkins-scripts/dsl/sdformat.dsl index 33507ff5e..49b748ab2 100644 --- a/jenkins-scripts/dsl/sdformat.dsl +++ b/jenkins-scripts/dsl/sdformat.dsl @@ -353,7 +353,7 @@ sdformat_supported_versions.each { version -> // WINDOWS: CI job // 1. any - String ci_build_any_job_name_win7 = "sdformat-ci-pr_any-windows7-amd64" + String ci_build_any_job_name_win7 = "sdformat-pr-win" def sdformat_win_ci_any_job = job(ci_build_any_job_name_win7) OSRFWinCompilationAnyGitHub.create(sdformat_win_ci_any_job, "gazebosim/sdformat") @@ -369,7 +369,8 @@ sdformat_supported_versions.each { version -> // 2. main / @ SCM/Daily all_versions = sdformat_supported_versions + 'main' all_versions.each { version -> - def sdformat_win_ci_job = job("sdformat-ci-${version}-windows7-amd64") + // Use replace to get branch names to sync with ignition packages + def sdformat_win_ci_job = job("sdformat-" + version.replace('sdformat', 'sdf') + "-win") OSRFWinCompilation.create(sdformat_win_ci_job) OSRFGitHub.create(sdformat_win_ci_job, "gazebosim/sdformat", get_sdformat_branch_name(version)) From 156cf9382dc30afe5cff435f2f13231a79dba275 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 31 Aug 2022 02:19:41 +0200 Subject: [PATCH 224/540] Fix naming scheme to match docs --- jenkins-scripts/dsl/sdformat.dsl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/jenkins-scripts/dsl/sdformat.dsl b/jenkins-scripts/dsl/sdformat.dsl index 49b748ab2..ed376199d 100644 --- a/jenkins-scripts/dsl/sdformat.dsl +++ b/jenkins-scripts/dsl/sdformat.dsl @@ -369,8 +369,9 @@ sdformat_supported_versions.each { version -> // 2. main / @ SCM/Daily all_versions = sdformat_supported_versions + 'main' all_versions.each { version -> - // Use replace to get branch names to sync with ignition packages - def sdformat_win_ci_job = job("sdformat-" + version.replace('sdformat', 'sdf') + "-win") + // Use replace to get branch names to sync with ignition packages and the + // format of $branch-$version + def sdformat_win_ci_job = job("sdformat-" + version.replace('sdformat', 'sdf-') + "-win") OSRFWinCompilation.create(sdformat_win_ci_job) OSRFGitHub.create(sdformat_win_ci_job, "gazebosim/sdformat", get_sdformat_branch_name(version)) From bff54fc8e5804346406ec255394ddb1642eb864d Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 31 Aug 2022 19:08:26 +0200 Subject: [PATCH 225/540] Fix syntax --- jenkins-scripts/dsl/ignition_collection.dsl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index 11fd1c87d..d0aa77973 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -117,7 +117,7 @@ gz_collection_jobs = 'sdformat-ci-sdformat9-focal-amd64', 'sdformat-ci-sdformat9-homebrew-amd64', 'sdformat-ci-sdformat9-windows7-amd64', - 'sdformat-install-sdformat9_pkg-focal-amd64' + 'sdformat-install-sdformat9_pkg-focal-amd64', 'sdformat-sdf9-win' ], 'fortress' : [ From 1360574b7e6a386569d55aa51d9c197124069d28 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 1 Sep 2022 17:12:57 +0200 Subject: [PATCH 226/540] Use sdf-version to mimic gz- jobs --- jenkins-scripts/dsl/sdformat.dsl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/sdformat.dsl b/jenkins-scripts/dsl/sdformat.dsl index ed376199d..10fb0d5b5 100644 --- a/jenkins-scripts/dsl/sdformat.dsl +++ b/jenkins-scripts/dsl/sdformat.dsl @@ -371,7 +371,7 @@ all_versions = sdformat_supported_versions + 'main' all_versions.each { version -> // Use replace to get branch names to sync with ignition packages and the // format of $branch-$version - def sdformat_win_ci_job = job("sdformat-" + version.replace('sdformat', 'sdf-') + "-win") + def sdformat_win_ci_job = job("sdformat-sdf-" + version.replace('sdformat','') + "-win") OSRFWinCompilation.create(sdformat_win_ci_job) OSRFGitHub.create(sdformat_win_ci_job, "gazebosim/sdformat", get_sdformat_branch_name(version)) From dfe5743103c1ae726f1f60e0e75464b754bab929 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 1 Sep 2022 17:14:05 +0200 Subject: [PATCH 227/540] Update collection --- jenkins-scripts/dsl/ignition_collection.dsl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index d0aa77973..84a36d5c2 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -118,7 +118,7 @@ gz_collection_jobs = 'sdformat-ci-sdformat9-homebrew-amd64', 'sdformat-ci-sdformat9-windows7-amd64', 'sdformat-install-sdformat9_pkg-focal-amd64', - 'sdformat-sdf9-win' + 'sdformat-sdf-9-win' ], 'fortress' : [ 'ign_cmake-ign-2-win', @@ -202,7 +202,7 @@ gz_collection_jobs = 'sdformat-ci-sdformat12-focal-amd64', 'sdformat-ci-sdformat12-homebrew-amd64', 'sdformat-install-sdformat12_pkg-focal-amd64', - 'sdformat-sdf12-win.xml' + 'sdformat-sdf-12-win.xml' ], 'garden' : [ 'ign_cmake-gz-3-win', @@ -286,7 +286,7 @@ gz_collection_jobs = 'sdformat-ci-sdformat13-focal-amd64', 'sdformat-ci-sdformat13-homebrew-amd64', 'sdformat-install-sdformat12_pkg-focal-amd64', - 'sdformat-sdf13-win' + 'sdformat-sdf-13-win' ], ] From 70f293eb9d6da9b046d50b0dfb8d2b4eaeef3772 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 1 Sep 2022 17:38:32 +0200 Subject: [PATCH 228/540] Use gz- debbuilders in collections --- jenkins-scripts/dsl/ignition_collection.dsl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index 84a36d5c2..c1aae7979 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -34,7 +34,7 @@ gz_collections = [ 'physics' : [ debbuild: 'gz-physics6' , branch: 'gz-physics6' ], 'gazebo' : [ debbuild: 'gz-sim7' , branch: 'gz-sim7' ], 'launch' : [ debbuild: 'gz-launch6' , branch: 'gz-launch6' ], - 'garden' : [ debbuild: 'ign-garden' , branch: 'main' ], + 'garden' : [ debbuild: 'gz-garden' , branch: 'main' ], ], ], ] @@ -403,7 +403,7 @@ gz_collections.each { gz_collection -> // DEBBUILD: linux package builder // -------------------------------------------------------------- - def build_pkg_job = job("ign-${gz_collection_name}-debbuilder") + def build_pkg_job = job("gz-${gz_collection_name}-debbuilder") OSRFLinuxBuildPkg.create(build_pkg_job) build_pkg_job.with { From 29617ca3a55e91e2164c4ca6e10434f015801561 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 2 Sep 2022 20:16:17 +0200 Subject: [PATCH 229/540] Fixes for DSL related to gz transition --- jenkins-scripts/dsl/ignition.dsl | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index d12871450..6e6f3bbee 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -21,6 +21,7 @@ gz_software = [ 'cmake', gz_gpu = [ 'gazebo', 'gui', 'rendering', + 'sim', 'sensors' ] // DESC: software does not support cmake warnings enabled gz_no_cmake_warnings = [ 'cmake', @@ -81,8 +82,7 @@ gz_prerelease_branches = [] // DESC: versioned names to generate debbuild jobs for special cases that // don't appear in gz_branches (like nightly builders or 0-debbuild // jobs for the special cases of foo0 packages) -gz_extra_debbuild = [ 'gazebo7', - 'utils1' ] // see comment https://github.com/gazebo-tooling/release-tools/pull/431#issuecomment-815099918 +gz_extra_debbuild = [ 'utils1' ] // see comment https://github.com/gazebo-tooling/release-tools/pull/431#issuecomment-815099918 // DESC: exclude ignition from generate any install testing job gz_no_pkg_yet = [ ] // DESC: major versions that has a package in the prerelease repo. Should @@ -201,17 +201,19 @@ ArrayList all_branches(String software_name) branches.add("ign-${software_name}${major_version}") } } + prerelease_branches("${software_name}").each { branch -> + if ("${branch}") { + branches.add(branch) + } + } + if (software_name == 'gazebo') + software_name = 'sim' supported_gz_branches("${software_name}").each { major_version -> if ("${major_version}") { branches.add("gz-${software_name}${major_version}") } } branches.add('main') - prerelease_branches("${software_name}").each { branch -> - if ("${branch}") { - branches.add(branch) - } - } return branches } From 966c79597c1a65b840d17b7f09c0724fdf7b8b2d Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Sat, 3 Sep 2022 21:54:16 -0700 Subject: [PATCH 230/540] Use -DGZ_SANITIZER to fix deprecation warning (#776) Signed-off-by: Steve Peters --- jenkins-scripts/dsl/ignition.dsl | 2 +- jenkins-scripts/dsl/sdformat.dsl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index 6e6f3bbee..48e14b9f9 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -486,7 +486,7 @@ gz_software.each { gz_sw -> void generate_asan_ci_job(gz_ci_job, gz_sw, branch, distro, arch) { generate_ci_job(gz_ci_job, gz_sw, branch, distro, arch, - '-DIGN_SANITIZER=Address', + '-DGZ_SANITIZER=Address', Globals.MAKETEST_SKIP_IGN) } diff --git a/jenkins-scripts/dsl/sdformat.dsl b/jenkins-scripts/dsl/sdformat.dsl index 10fb0d5b5..334f018de 100644 --- a/jenkins-scripts/dsl/sdformat.dsl +++ b/jenkins-scripts/dsl/sdformat.dsl @@ -139,7 +139,7 @@ other_supported_distros.each { distro -> void generate_asan_ci_job(sdformat_ci_job, version, distro, arch) { generate_ci_job(sdformat_ci_job, version, distro, arch, - '-DIGN_SANITIZER=Address', + '-DGZ_SANITIZER=Address', Globals.MAKETEST_SKIP_IGN) } From a32c867fe45c7b86af5602a7c35ee37aa08abca6 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 8 Sep 2022 00:26:51 +0200 Subject: [PATCH 231/540] Import changes from 797 --- .../rename_ignition_to_gazebo.bash | 40 +++++++++++++------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/release-repo-scripts/rename_ignition_to_gazebo.bash b/release-repo-scripts/rename_ignition_to_gazebo.bash index 8e3790a83..fdb391b27 100755 --- a/release-repo-scripts/rename_ignition_to_gazebo.bash +++ b/release-repo-scripts/rename_ignition_to_gazebo.bash @@ -24,17 +24,23 @@ find . -name control -type f | while IFS= read -r f; do echo "Processing control file: ${f}" ignition_pkgs=$(grep-dctrl -sPackage -n '' ${f}) echo " - Replace ignition ocurrences" - # Rename urls and packages - sed -i -e 's:ignitionrobotics:gazebosim:g' "${f}" - sed -i -e 's:libignition:libgz:g' "${f}" - sed -i -e 's:ignition:gz:g' "${f}" - sed -i -e 's:Ignition:Gazebo:g' "${f}" - sed -i -e 's:ign-:gz-:g' "${f}" + # Rename packages and sources + sed -i -e 's/Package: python3-ignition-gazebo/Package: python3-gz-sim/g' "${f}" + sed -i -e 's/Package: ignition-gazebo/Package: gz-sim/g' "${f}" + sed -i -e 's/Package: libignition-gazebo/Package: libgz-sim/g' "${f}" + sed -i -e 's/Source: ignition-gazebo/Source: gz-sim/g' "${f}" + + sed -i -e 's/Package: python3-ignition/Package: python3-gz/g' "${f}" + sed -i -e 's/Package: ruby-ignition/Package: ruby-gz/g' "${f}" + sed -i -e 's/Package: ignition/Package: gz/g' "${f}" + sed -i -e 's/Package: libignition/Package: libgz/g' "${f}" + sed -i -e 's/Source: ignition/Source: gz/g' "${f}" # Inject transitional (alias) packages echo " - Transitional packages in control file" for pkg in ${ignition_pkgs}; do echo " * Processing transitional package for ${pkg}" new_pkg_name=${pkg/ignition/gz} + new_pkg_name=${new_pkg_name/gazebo/sim} cat << EOF >> "${f}" Package: ${pkg} @@ -96,6 +102,11 @@ done echo # 6. Rename all ignition filenames not symlinks +find . -name '*ignition-gazebo*' -type f | while IFS= read -r f; do + echo "Renaming ${f} -> ${f/ignition-gazebo/gz-sim}" + git mv "${f}" "${f/ignition-gazebo/gz-sim}" +done + find . -name '*ignition*' -type f | while IFS= read -r f; do echo "Renaming ${f} -> ${f/ignition-/gz-}" git mv "${f}" "${f/ignition-/gz-}" @@ -111,9 +122,14 @@ done echo -# 7. Revert Provides, Breaks, Replaces -find . -name control -type f | while IFS= read -r f; do - sed -i -e 's:\(Breaks\:.*\)gz-\(.*\):\1ignition-\2:' "${f}" - sed -i -e 's:\(Provides\:.*\)gz-\(.*\):\1ignition-\2:' "${f}" - sed -i -e 's:\(Replaces\:.*\)gz-\(.*\):\1ignition-\2:' "${f}" -done +# # 7. Revert Provides, Breaks, Replaces +# find . -name control -type f | while IFS= read -r f; do +# sed -i -e 's:\(Breaks\:.*\)gz-\(.*\):\1ignition-\2:' "${f}" +# sed -i -e 's:\(Provides\:.*\)gz-\(.*\):\1ignition-\2:' "${f}" +# sed -i -e 's:\(Replaces\:.*\)gz-\(.*\):\1ignition-\2:' "${f}" +# done + +echo "Do manually:" +echo " * Update depends from the same library" +echo " * Add breaks / replaces for previous nightly" +echo " * Update rules" From 657d23bda9dc22bc01612a1753112901a8f6c75a Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 8 Sep 2022 00:30:15 +0200 Subject: [PATCH 232/540] Custom fixes for changes --- .../rename_ignition_to_gazebo.bash | 22 +++++++++++++------ 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/release-repo-scripts/rename_ignition_to_gazebo.bash b/release-repo-scripts/rename_ignition_to_gazebo.bash index fdb391b27..fdbfb9f3f 100755 --- a/release-repo-scripts/rename_ignition_to_gazebo.bash +++ b/release-repo-scripts/rename_ignition_to_gazebo.bash @@ -55,6 +55,14 @@ EOF echo done +# 2.0 Rename all ignition symlinks and destinations +find . -name '*ignition-gazebo*' -type l | while IFS= read -r f; do + echo "Renaming ${f} -> ${f/ignition-gazebo/gz-sim}" + link=$(readlink "$f") + ln -sfT "${link//ignition-gazebo/gz-sim}" "${f}" + git mv "${f}" "${f/ignition-gazebo/gz-sim}" +done + # 2. Copyright files find . -name '*copyright*' -type f | while IFS= read -r f; do echo "Procesing copyright file: ${f}" @@ -122,14 +130,14 @@ done echo -# # 7. Revert Provides, Breaks, Replaces -# find . -name control -type f | while IFS= read -r f; do -# sed -i -e 's:\(Breaks\:.*\)gz-\(.*\):\1ignition-\2:' "${f}" -# sed -i -e 's:\(Provides\:.*\)gz-\(.*\):\1ignition-\2:' "${f}" -# sed -i -e 's:\(Replaces\:.*\)gz-\(.*\):\1ignition-\2:' "${f}" -# done + # 7. Revert Provides, Breaks, Replaces + find . -name control -type f | while IFS= read -r f; do + sed -i -e 's:\(Breaks\:.*\)gz-\(.*\):\1ignition-\2:' "${f}" + sed -i -e 's:\(Provides\:.*\)gz-\(.*\):\1ignition-\2:' "${f}" + sed -i -e 's:\(Replaces\:.*\)gz-\(.*\):\1ignition-\2:' "${f}" +done echo "Do manually:" echo " * Update depends from the same library" -echo " * Add breaks / replaces for previous nightly" +echo " * Check breaks / replaces for previous nightly" echo " * Update rules" From dc4c430af0daef08161f31a383a3b4f9087b2358 Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Sun, 11 Sep 2022 23:06:06 -0700 Subject: [PATCH 233/540] release_collection: prerelease Changelog updates Update the release_collection issue template to remind us to update the library Changelogs for prereleases. Signed-off-by: Steve Peters --- .github/ISSUE_TEMPLATE/release_collection.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/release_collection.md b/.github/ISSUE_TEMPLATE/release_collection.md index 77e5367e9..5ff1f75ab 100644 --- a/.github/ISSUE_TEMPLATE/release_collection.md +++ b/.github/ISSUE_TEMPLATE/release_collection.md @@ -48,7 +48,7 @@ When opening PRs, add a link back to this issue for easier tracking. - [ ] Enable nightlies for all `main` branches on `gzdev` ([example](https://github.com/gazebo-tooling/gzdev/pull/50)) - [ ] Execute the tick-tock's "tock" for deprecations ([example](https://github.com/gazebosim/gz-sim/pull/875)) * -- [ ] Pre-release libraries as all `beta` labels are merged. +- [ ] Update Changelog and pre-release libraries as all `beta` labels are merged. * - [ ] Make collection pre-release after all libraries are pre-released. * From b6fed158610463ab78646e416c8b0c48c99c2608 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Mon, 12 Sep 2022 18:47:11 +0200 Subject: [PATCH 234/540] Add gz- install jobs for Linux for Garden dashboard --- jenkins-scripts/dsl/ignition_collection.dsl | 22 ++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index c1aae7979..431439870 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -205,6 +205,22 @@ gz_collection_jobs = 'sdformat-sdf-12-win.xml' ], 'garden' : [ + 'gz_cmake2-install-pkg-focal-amd64', + 'gz_common4-install-pkg-focal-amd64', + 'gz_fuel-tools7-install-pkg-focal-amd64', + 'gz_garden-install-pkg-focal-amd64', + 'gz_gazebo6-install-pkg-focal-amd64', + 'gz_gui6-install-pkg-focal-amd64', + 'gz_launch5-install-pkg-focal-amd64', + 'gz_math6-install-pkg-focal-amd64', + 'gz_msgs8-install-pkg-focal-amd64', + 'gz_physics5-install-pkg-focal-amd64', + 'gz_plugin-install-pkg-focal-amd64', + 'gz_rendering6-install-pkg-focal-amd64', + 'gz_sensors6-install-pkg-focal-amd64', + 'gz_tools-install-pkg-focal-amd64', + 'gz_transport11-install-pkg-focal-amd64', + 'gz_utils-install-pkg-focal-amd64', 'ign_cmake-gz-3-win', 'ign_common-gz-5-win', 'ign_fuel-tools-gz-8-win', @@ -228,13 +244,13 @@ gz_collection_jobs = 'ignition_common-ci-gz-common5-homebrew-amd64', 'ignition_common4-install-pkg-focal-amd64', 'ignition_common4-install_bottle-homebrew-amd64', - 'ignition_garden-ci-main-homebrew-amd64', - 'ignition_garden-install-pkg-focal-amd64', - 'ignition_garden-install_bottle-homebrew-amd64', 'ignition_fuel-tools-ci-gz-fuel-tools8-focal-amd64', 'ignition_fuel-tools-ci-gz-fuel-tools8-homebrew-amd64', 'ignition_fuel-tools7-install-pkg-focal-amd64', 'ignition_fuel-tools7-install_bottle-homebrew-amd64', + 'ignition_garden-ci-main-homebrew-amd64', + 'ignition_garden-install-pkg-focal-amd64', + 'ignition_garden-install_bottle-homebrew-amd64', 'ignition_gazebo-ci-main-focal-amd64', 'ignition_gazebo-ci-main-homebrew-amd64', 'ignition_gazebo6-install-pkg-focal-amd64', From 82ec1a8e4d73f5f5a0940b4722b9a90d226cc0b5 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Mon, 19 Sep 2022 23:26:49 +0200 Subject: [PATCH 235/540] Add release documents to update in release_collection template --- .github/ISSUE_TEMPLATE/release_collection.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/ISSUE_TEMPLATE/release_collection.md b/.github/ISSUE_TEMPLATE/release_collection.md index 5ff1f75ab..61d58e520 100644 --- a/.github/ISSUE_TEMPLATE/release_collection.md +++ b/.github/ISSUE_TEMPLATE/release_collection.md @@ -63,6 +63,10 @@ When opening PRs, add a link back to this issue for easier tracking. * - [ ] Hold tutorial party ([example](https://community.gazebosim.org/t/ignition-edifice-tutorial-party-support-open-source-and-get-t-shirts/866)) * +- [ ] Update references to the release install instructions + * [ ] Versions doc https://github.com/gazebosim/docs/blob/master/tools/versions.md + * [ ] ROS2 integration https://github.com/gazebosim/docs/blob/master/\/ros2_integration.md + * [ ] ros_gz README https://github.com/gazebosim/ros_gz/blob/ros2/README.md - [ ] PRs fixing documentation and critical bugs can be merged and pre-released - As libraries have all their documentation reviewed: - [ ] Make stable releases From 1f00ca20da235c4938a564841a32faf8337cfd4f Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Tue, 20 Sep 2022 16:35:22 -0700 Subject: [PATCH 236/540] changelog script: improve tag/branch detection (#826) Currently the source_changelog.bash script only works for ignition-* tags, while garden will use gz-* tags. This updates the script to grep for a tag that ends with the specified version number. It also uses the current branch for the comparison to avoid assumptions about ign-* or gz-* branch names. Signed-off-by: Steve Peters --- source-repo-scripts/source_changelog.bash | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/source-repo-scripts/source_changelog.bash b/source-repo-scripts/source_changelog.bash index 4a7ae07ee..eb733dbb4 100755 --- a/source-repo-scripts/source_changelog.bash +++ b/source-repo-scripts/source_changelog.bash @@ -1,23 +1,20 @@ #!/bin/bash # bash source_changelog.bash -PREV_TAG=$1 +PREV_VER=$1 git fetch --tags -REPO=$(basename `git rev-parse --show-toplevel`) -REPO="${REPO/ign-/gz-}" -MAJOR=${PREV_TAG%.*.*} -BRANCH=${REPO/sdformat/sdf}${MAJOR} -TAG="${REPO}${MAJOR}" +ORIGIN_URL=$(git remote get-url origin) +REPO=$(basename ${ORIGIN_URL%.git}) -# TODO(chapulina) Support Garden tags and branches, which will start with gz- -TAG="${TAG/gz-/ignition-}" -TAG="${TAG/sim/gazebo}" -BRANCH="${BRANCH/gz-/ign-}" -BRANCH="${BRANCH/sim/gazebo}" +# Find tag that ends with _$PREV_VER +PREV_TAG=$(git tag | grep "_${PREV_VER}$") -COMMITS=$(git log ${BRANCH}...${TAG}_${PREV_TAG} --pretty=format:"%h") +# Compare current branch to PREV_TAG +BRANCH=$(git rev-parse --abbrev-ref HEAD) + +COMMITS=$(git log ${BRANCH}...${PREV_TAG} --pretty=format:"%h") for COMMIT in $COMMITS do From f9bcef80da86fcbcaeeaa10ce63ad380aed9c52a Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 21 Sep 2022 07:33:34 +0200 Subject: [PATCH 237/540] Ignore ALSA lib false positive in warnings inside Jenkins (#828) --- jenkins-scripts/dsl/_configs_/OSRFLinuxCompilation.groovy | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/_configs_/OSRFLinuxCompilation.groovy b/jenkins-scripts/dsl/_configs_/OSRFLinuxCompilation.groovy index a8baa6677..07fc5c23f 100644 --- a/jenkins-scripts/dsl/_configs_/OSRFLinuxCompilation.groovy +++ b/jenkins-scripts/dsl/_configs_/OSRFLinuxCompilation.groovy @@ -77,7 +77,11 @@ class OSRFLinuxCompilation extends OSRFLinuxBase minimumSeverity { name('LOW') } - filters { } + filters { + 'io.jenkins.plugins.analysis.core.filter.ExcludeFile' { + pattern('.*ALSA lib.*') + } + } isEnabledForFailure(false) isAggregatingResults(false) isBlameDisabled(false) From ad303d2ac747f7243699625fde81e7839f511e3f Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 21 Sep 2022 21:34:25 +0200 Subject: [PATCH 238/540] Test gz-tools subcommands expected in collection test jobs --- .../docker/ign_launch-install-test-job.bash | 2 ++ jenkins-scripts/docker/lib/_gazebo_utils.sh | 20 ++++++++++++++++++- jenkins-scripts/dsl/ignition_collection.dsl | 3 +++ 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/jenkins-scripts/docker/ign_launch-install-test-job.bash b/jenkins-scripts/docker/ign_launch-install-test-job.bash index 6b91dbe00..a10ba2908 100644 --- a/jenkins-scripts/docker/ign_launch-install-test-job.bash +++ b/jenkins-scripts/docker/ign_launch-install-test-job.bash @@ -10,6 +10,8 @@ export GPU_SUPPORT_NEEDED=true . ${SCRIPT_DIR}/lib/_gazebo_utils.sh INSTALL_JOB_POSTINSTALL_HOOK=""" +GZ_SIM_RUNTIME_TEST_USE_IGN=${GZ_SIM_RUNTIME_TEST_USE_IGN:-false} + ${GZ_SIM_RUNTIME_TEST} """ # Need bc to proper testing and parsing the time diff --git a/jenkins-scripts/docker/lib/_gazebo_utils.sh b/jenkins-scripts/docker/lib/_gazebo_utils.sh index e31c349d6..2f0bf0168 100644 --- a/jenkins-scripts/docker/lib/_gazebo_utils.sh +++ b/jenkins-scripts/docker/lib/_gazebo_utils.sh @@ -24,9 +24,26 @@ tar -xf /tmp/master.tar.gz -C ~/.gazebo/models --strip 1 >/dev/null 2>&1 rm /tmp/master.tar.gz""" GZ_SIM_RUNTIME_TEST=""" +if \${GZ_SIM_RUNTIME_TEST_USE_IGN}; then + CMD_TOOLS='ign' + CMD_TOOLS_SIMULATOR='ign gazebo -v' + CMD_TOOLS_MODULES='gui fuel topic service gazebo sdf model launch plugin msg log' +else + CMD_TOOLS='gz' + CMD_TOOLS_SIMULATOR='gz sim --verbose 4' + CMD_TOOLS_MODULES='gui fuel topic service sim sdf model launch plugin msg log' +fi + +echo '# BEGIN SECTION: check gz subcommands available' + for module in \${CMD_TOOLS_MODULES}; do + echo \"Testing \${module}\" + \${CMD_TOOLS} \${module} --versions + done +echo '# END SECTION' + echo '# BEGIN SECTION: test the script' TEST_START=\`date +%s\` -timeout --preserve-status 180 ign gazebo -v -r camera_sensor.sdf || true +timeout --preserve-status 180 \${CMD_TOOLS_SIMULATOR} -r camera_sensor.sdf || true TEST_END=\`date +%s\` DIFF=\`echo \"\$TEST_END - \$TEST_START\" | bc\` @@ -34,6 +51,7 @@ if [ \$DIFF -lt 180 ]; then echo 'The test took less than 180s. Something bad happened' exit 1 fi + echo '# END SECTION' """ diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index 431439870..88fb2a9ae 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -332,6 +332,9 @@ void generate_install_job(prefix, gz_collection_name, distro, arch) export ARCH=${arch} export INSTALL_JOB_PKG=${dev_package} export GZDEV_PROJECT_NAME="${dev_package}" + if [[ ${gz_collection_name} == 'citadel' || ${gz_collection_name} == 'fortress' ]]; then + export GZ_SIM_RUNTIME_TEST_USE_IGN=true + fi /bin/bash -x ./scripts/jenkins-scripts/docker/${job_name} """.stripIndent()) } From fce2471641c695ae323effa0d2934632e4d4ec37 Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Fri, 30 Sep 2022 11:10:04 -0700 Subject: [PATCH 239/540] dsl: disable ignition_garden-install-* jobs There is no stable release of an ignition-garden debian package, so don't create a CI job for it. Signed-off-by: Steve Peters --- jenkins-scripts/dsl/ignition_collection.dsl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index 431439870..0f3ed7d13 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -360,7 +360,9 @@ gz_collections.each { gz_collection -> gz_collection.get('distros').each { distro -> // INSTALL JOBS: // -------------------------------------------------------------- - generate_install_job('ignition', gz_collection_name, distro, arch) + if ((gz_collection_name == "citadel") || (gz_collection_name == "fortress")) { + generate_install_job('ignition', gz_collection_name, distro, arch) + } generate_install_job('gz', gz_collection_name, distro, arch) } From 0b74ebadfb3218c563f4ac9c659a15029fa865c1 Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Sat, 1 Oct 2022 14:51:10 -0700 Subject: [PATCH 240/540] asan dsl: skip _gz_TEST tests also The _ign_TEST CLI tests are skipped for asan due to an LD_PRELOAD error, but the _gz_TESTs on garden should be skipped as well. Signed-off-by: Steve Peters --- jenkins-scripts/dsl/_configs_/Globals.groovy | 2 +- jenkins-scripts/dsl/ignition.dsl | 2 +- jenkins-scripts/dsl/sdformat.dsl | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/jenkins-scripts/dsl/_configs_/Globals.groovy b/jenkins-scripts/dsl/_configs_/Globals.groovy index d7e2e2a3b..fd04b67f3 100644 --- a/jenkins-scripts/dsl/_configs_/Globals.groovy +++ b/jenkins-scripts/dsl/_configs_/Globals.groovy @@ -13,7 +13,7 @@ class Globals static CRON_EVERY_THREE_DAYS = 'H H * * H/3' static CRON_ON_WEEKEND = 'H H * * 6-7' - static MAKETEST_SKIP_IGN = "-E _ign_TEST" + static MAKETEST_SKIP_GZ = "-E _gz_TEST -E _ign_TEST" static gpu_by_distro = [ bionic : [ 'nvidia' ]] diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index 48e14b9f9..27ea61888 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -487,7 +487,7 @@ void generate_asan_ci_job(gz_ci_job, gz_sw, branch, distro, arch) { generate_ci_job(gz_ci_job, gz_sw, branch, distro, arch, '-DGZ_SANITIZER=Address', - Globals.MAKETEST_SKIP_IGN) + Globals.MAKETEST_SKIP_GZ) } void generate_ci_job(gz_ci_job, gz_sw, branch, distro, arch, diff --git a/jenkins-scripts/dsl/sdformat.dsl b/jenkins-scripts/dsl/sdformat.dsl index 334f018de..27a9a474a 100644 --- a/jenkins-scripts/dsl/sdformat.dsl +++ b/jenkins-scripts/dsl/sdformat.dsl @@ -140,7 +140,7 @@ void generate_asan_ci_job(sdformat_ci_job, version, distro, arch) { generate_ci_job(sdformat_ci_job, version, distro, arch, '-DGZ_SANITIZER=Address', - Globals.MAKETEST_SKIP_IGN) + Globals.MAKETEST_SKIP_GZ) } void generate_ci_job(sdformat_ci_job, version, distro, arch, From 6fde14661fef11abf5bccd79a5177d4c4910c5e7 Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Mon, 3 Oct 2022 00:13:18 -0700 Subject: [PATCH 241/540] Combine regex into one -E arg Signed-off-by: Steve Peters --- jenkins-scripts/dsl/_configs_/Globals.groovy | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/_configs_/Globals.groovy b/jenkins-scripts/dsl/_configs_/Globals.groovy index fd04b67f3..89c47d42e 100644 --- a/jenkins-scripts/dsl/_configs_/Globals.groovy +++ b/jenkins-scripts/dsl/_configs_/Globals.groovy @@ -13,7 +13,9 @@ class Globals static CRON_EVERY_THREE_DAYS = 'H H * * H/3' static CRON_ON_WEEKEND = 'H H * * 6-7' - static MAKETEST_SKIP_GZ = "-E _gz_TEST -E _ign_TEST" + // Only one -E regex can be passed, so make a regex that matches both + // _ign_TEST and _gz_TEST + static MAKETEST_SKIP_GZ = "-E _i?g[nz]_TEST" static gpu_by_distro = [ bionic : [ 'nvidia' ]] From de9165236cf4360aaf2b2a88347a70e36af1ea79 Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Tue, 11 Oct 2022 21:12:32 -0700 Subject: [PATCH 242/540] brew CI: use python3 explicitly python 2.X is no longer available in macOS Monterey Signed-off-by: Steve Peters --- jenkins-scripts/gazebo-default-devel-homebrew-amd64.bash | 2 +- jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/jenkins-scripts/gazebo-default-devel-homebrew-amd64.bash b/jenkins-scripts/gazebo-default-devel-homebrew-amd64.bash index 637f81235..b154d713d 100644 --- a/jenkins-scripts/gazebo-default-devel-homebrew-amd64.bash +++ b/jenkins-scripts/gazebo-default-devel-homebrew-amd64.bash @@ -6,7 +6,7 @@ SCRIPT_DIR="${SCRIPT_DIR%/*}" # Identify GAZEBO_MAJOR_VERSION to help with dependency resolution GAZEBO_MAJOR_VERSION=$(\ - python ${SCRIPT_DIR}/tools/detect_cmake_major_version.py \ + python3 ${SCRIPT_DIR}/tools/detect_cmake_major_version.py \ ${WORKSPACE}/gazebo/CMakeLists.txt) if [ $GAZEBO_MAJOR_VERSION -ge 7 ]; then diff --git a/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash b/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash index 62c18bed9..eb8c0f227 100644 --- a/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash +++ b/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash @@ -27,7 +27,7 @@ PROJECT_PATH=${PROJECT_PATH/ign-sim/ign-gazebo} # Check for major version number # the PROJECT_FORMULA variable is only used for dependency resolution PROJECT_FORMULA=${PROJECT//[0-9]}$(\ - python ${SCRIPT_DIR}/tools/detect_cmake_major_version.py \ + python3 ${SCRIPT_DIR}/tools/detect_cmake_major_version.py \ ${WORKSPACE}/${PROJECT_PATH}/CMakeLists.txt || true) export HOMEBREW_PREFIX=/usr/local From 51d03f535d89a5b26f7dcf1db42c8e1ee881a01e Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Wed, 12 Oct 2022 09:16:44 -0700 Subject: [PATCH 243/540] brew_release.dsl: build monterey bottles (#840) Signed-off-by: Steve Peters Signed-off-by: Steve Peters --- jenkins-scripts/dsl/brew_release.dsl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/brew_release.dsl b/jenkins-scripts/dsl/brew_release.dsl index 5f1592741..56bd7d166 100644 --- a/jenkins-scripts/dsl/brew_release.dsl +++ b/jenkins-scripts/dsl/brew_release.dsl @@ -4,7 +4,7 @@ import javaposse.jobdsl.dsl.Job Globals.default_emails = "jrivero@osrfoundation.org, scpeters@osrfoundation.org" // first distro in list is used as touchstone -brew_supported_distros = [ "catalina", "bigsur" ] +brew_supported_distros = [ "catalina", "bigsur", "monterey" ] bottle_hash_updater_job_name = 'generic-release-homebrew_pr_bottle_hash_updater' bottle_builder_job_name = 'generic-release-homebrew_triggered_bottle_builder' directory_for_bottles = 'pkgs' From 3120a8c48173d6fc2be261531c95de3598593f0d Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Thu, 20 Oct 2022 17:00:20 -0700 Subject: [PATCH 244/540] Add script for opening release PR (#834) Signed-off-by: Steve Peters --- .../merge_forward_pull_request.bash | 2 +- source-repo-scripts/release_pull_request.bash | 76 +++++++++++++++++++ 2 files changed, 77 insertions(+), 1 deletion(-) create mode 100755 source-repo-scripts/release_pull_request.bash diff --git a/source-repo-scripts/merge_forward_pull_request.bash b/source-repo-scripts/merge_forward_pull_request.bash index a785a4d04..7b5ff7091 100755 --- a/source-repo-scripts/merge_forward_pull_request.bash +++ b/source-repo-scripts/merge_forward_pull_request.bash @@ -49,7 +49,7 @@ BODY="# ➡️ Forward port Port \`${FROM_BRANCH} \` ➡️ \`${TO_BRANCH}\` -Branch comparision: https://github.com/${ORIGIN_ORG_REPO}/compare/${TO_BRANCH}...${FROM_BRANCH} +Branch comparison: https://github.com/${ORIGIN_ORG_REPO}/compare/${TO_BRANCH}...${FROM_BRANCH} **Note to maintainers**: Remember to **Merge** with commit (not squash-merge or rebase)" diff --git a/source-repo-scripts/release_pull_request.bash b/source-repo-scripts/release_pull_request.bash new file mode 100755 index 000000000..f3800c8fd --- /dev/null +++ b/source-repo-scripts/release_pull_request.bash @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +# Copyright (C) 2022 Open Source Robotics Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The script will open a pull request for a forward port. +# +# Requires the 'gh' CLI to be installed. +# +# Usage: +# $ ./merge_forward_pull_request.bash +# +# For example, to merge `ign-rendering6` forward to `main`: +# +# ./merge_forward_pull_request.bash ign-rendering6 main + +VERSION=${1} +TO_BRANCH=${2} +PREV_VER=${3} + +if [[ $# -ne 3 ]]; then + echo "./release_pull_request.bash " + exit 1 +fi + +set -e + +CURRENT_BRANCH=$(git branch | sed -n -e 's/^\* \(.*\)/\1/p') + +ORIGIN_URL=$(git remote get-url origin) +ORIGIN_ORG_REPO=$(echo ${ORIGIN_URL} | sed -e 's@.*github\.com.@@' | sed -e 's/\.git//g') + +PREV_TAG=$(git tag | grep "_${PREV_VER}$") + +TITLE="Prepare for ${VERSION} Release" + +BODY="# 🎈 Release + +Preparation for ${VERSION} release. + +Comparison to ${PREV_VER}: https://github.com/${ORIGIN_ORG_REPO}/compare/${PREV_TAG}...${TO_BRANCH} + + +Needed by + +## Checklist +- [ ] Asked team if this is a good time for a release +- [ ] There are no changes to be ported from the previous major version +- [ ] No PRs targeted at this major version are close to getting in +- [ ] Bumped minor for new features, patch for bug fixes +- [ ] Updated changelog +- [ ] Updated migration guide (as needed) +- [ ] Link to PR updating dependency versions in appropriate repository in [gazebo-release](https://github.com/gazebo-release) (as needed): + + + +**Note to maintainers**: Remember to use **Squash-Merge** and edit the commit message to match the pull request summary while retaining \`Signed-off-by\` messages." + + +gh pr create \ + --title "$TITLE" \ + --repo "$ORIGIN_ORG_REPO" \ + --base "$TO_BRANCH" \ + --body "$BODY" \ + --head "$CURRENT_BRANCH" From 356a9789cd414502c8c7d7cba9cb18e9ef44af22 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 26 Oct 2022 16:37:10 +0200 Subject: [PATCH 245/540] Add script to convert a gazebodistro file to vcs file with -release repos (#798) --- release-repo-scripts/README.md | 22 +++++++++ .../convert_gazebodistro_to_release.py | 46 +++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100755 release-repo-scripts/convert_gazebodistro_to_release.py diff --git a/release-repo-scripts/README.md b/release-repo-scripts/README.md index 941f67c83..f88766910 100644 --- a/release-repo-scripts/README.md +++ b/release-repo-scripts/README.md @@ -72,6 +72,28 @@ git clone https://github.com/gz-release/gz-rendering7-release cd gz-rendering7-release ./path/to/release-tools/release-repo-scripts/changelog_spawn.sh 7.1.0-1 ``` +### convert_gazebodistro_to_release.py + +Convert a gazebodistro file into a vcs compatible file with the -release repositories +associated to the repositories in the input file. + +### Usage + +Output will printed to stdout: + +```bash +./path/to/release-tools/release-repo-scripts/convert_gazebodistro_to_release.py +``` + +### Example + +To import all release repositories related to gz Garden collection: + +```bash +./path/to/release-tools/release-repo-scripts/convert_gazebodistro_to_release.py ~/code/gazebodistro/collection-garden.yaml > collection-garden-release.yaml +vcs import < collection-garden-release.yaml +``` + ### new_ubuntu_distribution.bash The script will create a new directory to host a new Ubuntu distribution inside diff --git a/release-repo-scripts/convert_gazebodistro_to_release.py b/release-repo-scripts/convert_gazebodistro_to_release.py new file mode 100755 index 000000000..aba484e49 --- /dev/null +++ b/release-repo-scripts/convert_gazebodistro_to_release.py @@ -0,0 +1,46 @@ +#!/usr/bin/python3 + +import argparse +import sys + +import yaml + + +def _load_gazebodistro_file(filepath) -> dict: + with open(filepath, 'r') as file: + gazebodistro = yaml.load(file, Loader=yaml.FullLoader) + + return gazebodistro + + +def _convert_to_release(input_yaml) -> None: + repos = {} + for repo, repo_info in input_yaml['repositories'].items(): + branch = repo_info['version'].replace('sdf', 'sdformat') + repo_name = f'{branch}-release' + repos[repo_name] = {'type': repo_info['type'], + 'url': f'https://github.com/gazebo-release/{repo_name}', + 'version': 'main'} + repos = {'repositories': repos} + yaml.dump(repos, sys.stdout) + + +def _init_argparse() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + usage='%(prog)s GAZEBODISTRO_FILE', + description='Transform gazebodistro files into vcs files for release repos' + ) + parser.add_argument('input_file', nargs=1, type=str) + return parser + + +def main() -> None: + """Convert gazebodistro files into vcs files for release repositories in the input file.""" + parser = _init_argparse() + args = parser.parse_args() + input_yaml = _load_gazebodistro_file(args.input_file[0]) + _convert_to_release(input_yaml) + + +if __name__ == '__main__': + main() From 202e6b7a12302872c9de81518ca15f16e1ecf6cc Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 26 Oct 2022 16:37:43 +0200 Subject: [PATCH 246/540] Support for testing gz collections using ros_bootstrap repository (#801) * Support ros_boostrap installation of gz collections --- .../docker/ign_launch-install-test-job.bash | 9 ++-- .../docker/lib/boilerplate_prepare.sh | 4 ++ .../lib/docker_generate_dockerfile.bash | 6 +++ jenkins-scripts/dsl/ignition_collection.dsl | 47 +++++++++++++++++++ 4 files changed, 63 insertions(+), 3 deletions(-) diff --git a/jenkins-scripts/docker/ign_launch-install-test-job.bash b/jenkins-scripts/docker/ign_launch-install-test-job.bash index a10ba2908..ec5abeb2f 100644 --- a/jenkins-scripts/docker/ign_launch-install-test-job.bash +++ b/jenkins-scripts/docker/ign_launch-install-test-job.bash @@ -4,16 +4,19 @@ [[ -L ${0} ]] && SCRIPT_DIR=$(readlink ${0}) || SCRIPT_DIR=${0} SCRIPT_DIR="${SCRIPT_DIR%/*}" -export GPU_SUPPORT_NEEDED=true +[[ -z ${ENABLE_GZ_SIM_RUNTIME_TEST} ]] && ENABLE_GZ_SIM_RUNTIME_TEST=true # Import library . ${SCRIPT_DIR}/lib/_gazebo_utils.sh -INSTALL_JOB_POSTINSTALL_HOOK=""" +if ${ENABLE_GZ_SIM_RUNTIME_TEST}; then + export GPU_SUPPORT_NEEDED=true + export INSTALL_JOB_POSTINSTALL_HOOK=""" GZ_SIM_RUNTIME_TEST_USE_IGN=${GZ_SIM_RUNTIME_TEST_USE_IGN:-false} - ${GZ_SIM_RUNTIME_TEST} """ +fi + # Need bc to proper testing and parsing the time export DEPENDENCY_PKGS="${DEPENDENCY_PKGS} wget bc" diff --git a/jenkins-scripts/docker/lib/boilerplate_prepare.sh b/jenkins-scripts/docker/lib/boilerplate_prepare.sh index 6a67fdce1..81510e945 100644 --- a/jenkins-scripts/docker/lib/boilerplate_prepare.sh +++ b/jenkins-scripts/docker/lib/boilerplate_prepare.sh @@ -91,6 +91,10 @@ if [ -z "${ROS2}" ]; then export ROS2=false fi +if [ -z "${ROS_BOOTSTRAP}" ]; then + export ROS_BOOTSTRAP=false +fi + # Define making jobs by default if not present if [ -z ${MAKE_JOBS} ]; then MAKE_JOBS=1 diff --git a/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash b/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash index 867ec1283..74c349cc2 100644 --- a/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash +++ b/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash @@ -227,6 +227,12 @@ RUN echo "deb [arch=amd64,arm64 signed-by=/usr/share/keyrings/ros-archive-keyrin RUN echo "deb [arch=amd64,arm64 signed-by=/usr/share/keyrings/ros-archive-keyring.gpg] http://repo.ros2.org/ubuntu/testing ${DISTRO} main" > \\ /etc/apt/sources.list.d/ros2-testing.list RUN curl -sSL https://raw.githubusercontent.com/ros/rosdistro/master/ros.key -o /usr/share/keyrings/ros-archive-keyring.gpg +DELIM_ROS_REPO + elif ${ROS_BOOTSTRAP}; then +cat >> Dockerfile << DELIM_ROS_REPO +RUN echo "deb http://repos.ros.org/repos/ros_bootstrap/ ${DISTRO} main" > \\ + /etc/apt/sources.list.d/ros_bootstrap.list +RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 8EDB2EF661FC880E DELIM_ROS_REPO else cat >> Dockerfile << DELIM_ROS_REPO diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index 1a95e8afa..bb79cf2d6 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -367,6 +367,53 @@ gz_collections.each { gz_collection -> generate_install_job('ignition', gz_collection_name, distro, arch) } generate_install_job('gz', gz_collection_name, distro, arch) + + // ROS BOOTSTRAP INSTALL JOBS: + // -------------------------------------------------------------- + def install_ros_bootstrap_job = + job("ignition_${gz_collection_name}-install-pkg_ros_bootstrap-any-manual") + OSRFLinuxInstall.create(install_ros_bootstrap_job) + install_ros_bootstrap_job.with + { + parameters { + stringParam("LINUX_DISTRO", 'ubuntu', "Linux distribution to build packages for") + stringParam("DISTRO", distro, "Linux release inside LINUX_DISTRO to build packages for") + stringParam("ARCH", arch, "Architecture to build packages for") + labelParam('JENKINS_NODE_TAG') { + description('Jenkins node or group to run the build on') + defaultValue('gpu-reliable') + } + } + + // Designed to be run manually. No triggers. + label "gpu-reliable" + + steps { + systemGroovyCommand("""\ + build.setDescription( + '' + build.buildVariableResolver.resolve('LINUX_DISTRO') + '/' + + build.buildVariableResolver.resolve('DISTRO') + '::' + + build.buildVariableResolver.resolve('ARCH') + '' + + '
' + + 'RTOOLS_BRANCH: ' + build.buildVariableResolver.resolve('RTOOLS_BRANCH')); + """.stripIndent()) + + shell("""\ + #!/bin/bash -xe + + + export INSTALL_JOB_PKG=ignition-${gz_collection_name} + export USE_ROS_REPO=true + export ROS_BOOTSTRAP=true + # needed for arm64 machines and other arch tests + export ENABLE_GZ_SIM_RUNTIME_TEST=false + if [[ \${JENKINS_NODE_TAG} == 'gpu-reliable' ]]; then + export ENABLE_GZ_SIM_RUNTIME_TEST=true + fi + /bin/bash -x ./scripts/jenkins-scripts/docker/ign_launch-install-test-job.bash + """.stripIndent()) + } + } } // MAC Brew CI job From ae3ca4adbc3c4a75cb1a8944029626200aef039c Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 26 Oct 2022 17:26:48 +0200 Subject: [PATCH 247/540] Fix terminal-dashboard to avoid confusions with arch=all packages (#835) * Fix terminal-dashboard to avoid confusions with arch=all packages * Use libgzfoo/libgzfoo-dev for searching successful builds Signed-off-by: Jose Luis Rivero --- terminal-dashboard/table.bash | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/terminal-dashboard/table.bash b/terminal-dashboard/table.bash index e47d412d5..2fe6c14eb 100755 --- a/terminal-dashboard/table.bash +++ b/terminal-dashboard/table.bash @@ -32,12 +32,28 @@ ARCHS=( "amd64") DISTROS=( "ubuntu" ) # No nightlies or pre-releases for arm if [[ $PACKAGE_REPO == "stable" ]]; then - ARCHS+=( "i386" "arm64" "armhf") + if [[ $COLLECTION == "citadel" || $COLLECTION == "fortress" ]]; then + ARCHS+=( "i386" ) + fi + + ARCHS+=( "arm64" "armhf") # No debian version supported across the stack right now # DISTROS+=( "debian" ) fi +# Search heuristics used and context: +# We are assuming that all the gz- libraries have a package named libgz${LIB} or +# libgz${LIB}-dev except for the collection packages starting with Garden. +# Note that relying on the fact of having a packages available in a given +# arch does not imply that build is successful since arch=all debs are built +# once for amd64 and appear in all the arches. +# The Source field (for source packages) is not mandatory and it is probably +# not present when the binary package has the same name than the source +# package. for LIB in $(get_libraries_by_collection "${COLLECTION}" ); do + if [[ "${LIB}" != "gz-${COLLECTION}" && "${LIB}" != "ignition-${COLLECTION}" ]]; then + LIB=lib${LIB} + fi echo -e "\e[107m\e[90m${LIB}\e[49m\e[39m" LIB_VER="" @@ -70,10 +86,8 @@ for LIB in $(get_libraries_by_collection "${COLLECTION}" ); do if [[ $ARCH == "i386" && $VER != "bionic" && $VER != "buster" ]]; then PKG_VERSION="disabled" else - # The Source field is not mandatory and it is probably not present when - # the binary package has the same name than the source package PKG_VERSION=$(wget -qO- http://packages.osrfoundation.org/gazebo/${DISTRO}-${PACKAGE_REPO}/dists/${VER}/main/binary-${ARCH}/Packages | \ - grep -2 -m 1 -e "Source: ${LIB}" -e "Package: ${LIB}" | \ + grep -2 -m 1 -e "Package: ${LIB}$" -e "Package: ${LIB}-dev$"| \ sed -n 's/^Version: \(.*\)/\1/p' | uniq) fi From 84b559856162c41278ff9728892804ab3d4f6015 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 28 Oct 2022 01:01:19 +0200 Subject: [PATCH 248/540] Support for check_initialization_order in asan builds (#714) * Support for ASAN_OPTIONS in asan builds Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/ignition.dsl | 7 +++++-- jenkins-scripts/dsl/sdformat.dsl | 7 +++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index 27ea61888..4576eb52b 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -487,11 +487,13 @@ void generate_asan_ci_job(gz_ci_job, gz_sw, branch, distro, arch) { generate_ci_job(gz_ci_job, gz_sw, branch, distro, arch, '-DGZ_SANITIZER=Address', - Globals.MAKETEST_SKIP_GZ) + Globals.MAKETEST_SKIP_GZ, + 'export ASAN_OPTIONS=check_initialization_order=true') } + void generate_ci_job(gz_ci_job, gz_sw, branch, distro, arch, - extra_cmake = '', extra_test = '') + extra_cmake = '', extra_test = '', extra_cmd = '') { OSRFLinuxCompilation.create(gz_ci_job, enable_testing(software_name)) OSRFGitHub.create(gz_ci_job, @@ -509,6 +511,7 @@ void generate_ci_job(gz_ci_job, gz_sw, branch, distro, arch, #!/bin/bash -xe ${GLOBAL_SHELL_CMD} + ${extra_cmd} export BUILDING_EXTRA_CMAKE_PARAMS="${extra_cmake}" export BUILDING_EXTRA_MAKETEST_PARAMS="${extra_test}" export DISTRO=${distro} diff --git a/jenkins-scripts/dsl/sdformat.dsl b/jenkins-scripts/dsl/sdformat.dsl index 27a9a474a..2d07bf5e3 100644 --- a/jenkins-scripts/dsl/sdformat.dsl +++ b/jenkins-scripts/dsl/sdformat.dsl @@ -140,11 +140,13 @@ void generate_asan_ci_job(sdformat_ci_job, version, distro, arch) { generate_ci_job(sdformat_ci_job, version, distro, arch, '-DGZ_SANITIZER=Address', - Globals.MAKETEST_SKIP_GZ) + Globals.MAKETEST_SKIP_GZ, + 'export ASAN_OPTIONS=check_initialization_order=true') } + void generate_ci_job(sdformat_ci_job, version, distro, arch, - extra_cmake = '', extra_test = '') + extra_cmake = '', extra_test = '', extra_cmd = '') { OSRFLinuxCompilation.create(sdformat_ci_job) OSRFGitHub.create(sdformat_ci_job, "gazebosim/sdformat", @@ -155,6 +157,7 @@ void generate_ci_job(sdformat_ci_job, version, distro, arch, shell("""\ #!/bin/bash -xe + ${extra_cmd} export BUILDING_EXTRA_CMAKE_PARAMS="${extra_cmake}" export BUILDING_EXTRA_MAKETEST_PARAMS="${extra_test}" export DISTRO=${distro} From cf59042daaa2a45f8896c152a92552a46f63c740 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 28 Oct 2022 01:15:57 +0200 Subject: [PATCH 249/540] Support for strict_init_order=true in ASAN builds Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/ignition.dsl | 2 +- jenkins-scripts/dsl/sdformat.dsl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index 4576eb52b..7fc5c4f8f 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -488,7 +488,7 @@ void generate_asan_ci_job(gz_ci_job, gz_sw, branch, distro, arch) generate_ci_job(gz_ci_job, gz_sw, branch, distro, arch, '-DGZ_SANITIZER=Address', Globals.MAKETEST_SKIP_GZ, - 'export ASAN_OPTIONS=check_initialization_order=true') + 'export ASAN_OPTIONS=check_initialization_order=true:strict_init_order=true') } diff --git a/jenkins-scripts/dsl/sdformat.dsl b/jenkins-scripts/dsl/sdformat.dsl index 2d07bf5e3..f8dd47848 100644 --- a/jenkins-scripts/dsl/sdformat.dsl +++ b/jenkins-scripts/dsl/sdformat.dsl @@ -141,7 +141,7 @@ void generate_asan_ci_job(sdformat_ci_job, version, distro, arch) generate_ci_job(sdformat_ci_job, version, distro, arch, '-DGZ_SANITIZER=Address', Globals.MAKETEST_SKIP_GZ, - 'export ASAN_OPTIONS=check_initialization_order=true') + 'export ASAN_OPTIONS=check_initialization_order=true:strict_init_order=true') } From f32e9ce39376c435109e7368dd0a75583ac5677a Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Fri, 28 Oct 2022 04:07:39 -0700 Subject: [PATCH 250/540] gazebo.dsl: use new GitHub URL (#843) Change osrf/gazebo to gazebosim/gazebo-classic. Signed-off-by: Steve Peters Signed-off-by: Steve Peters --- jenkins-scripts/dsl/gazebo.dsl | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/jenkins-scripts/dsl/gazebo.dsl b/jenkins-scripts/dsl/gazebo.dsl index d947b587a..f56167d04 100644 --- a/jenkins-scripts/dsl/gazebo.dsl +++ b/jenkins-scripts/dsl/gazebo.dsl @@ -8,7 +8,7 @@ def ubuntu_official_packages_distros = [ 'bionic' : 'gazebo9', 'focal' : 'gazebo9', 'jammy' : 'gazebo11'] // Main platform using for quick CI -// Using bionic until https://github.com/osrf/gazebo/issues/3151 is fixed +// Using bionic until https://github.com/gazebosim/gazebo-classic/issues/3151 is fixed def ci_distro = [ 'bionic' ] def ci_gpu = Globals.get_ci_gpu() def abi_distro = Globals.get_abi_distro() @@ -89,7 +89,7 @@ abi_distro.each { distro -> abi_job_name = "gazebo-abichecker-any_to_any-ubuntu_auto-${arch}" def abi_job = job(abi_job_name) OSRFLinuxABIGitHub.create(abi_job) - GenericAnyJobGitHub.create(abi_job, 'osrf/gazebo', gazebo_supported_branches) + GenericAnyJobGitHub.create(abi_job, 'gazebosim/gazebo-classic', gazebo_supported_branches) abi_job.with { label "large-memory" @@ -120,7 +120,7 @@ ci_distro.each { distro -> // 1. Create the any job def gazebo_ci_any_job_name = "gazebo-ci-pr_any-ubuntu_auto-${arch}-gpu-${gpu}" def gazebo_ci_any_job = job(gazebo_ci_any_job_name) - OSRFLinuxCompilationAnyGitHub.create(gazebo_ci_any_job, "osrf/gazebo") + OSRFLinuxCompilationAnyGitHub.create(gazebo_ci_any_job, "gazebosim/gazebo-classic") gazebo_ci_any_job.with { label "gpu-reliable && large-memory" @@ -158,7 +158,7 @@ other_supported_distros.each { distro -> // gazebo11 job for the rest of arches / scm@daily def gazebo_ci_job = job("gazebo-ci-gazebo11-${distro}-${arch}-gpu-${gpu}") OSRFLinuxCompilation.create(gazebo_ci_job) - OSRFGitHub.create(gazebo_ci_job, "osrf/gazebo") + OSRFGitHub.create(gazebo_ci_job, "gazebosim/gazebo-classic") gazebo_ci_job.with { @@ -196,7 +196,7 @@ ci_distro.each { distro -> supported_arches.each { arch -> ci_gpu.each { gpu -> def multi_any_job = job("gazebo-ci-pr_any+sdformat_any+ign_any-${distro}-${arch}-gpu-${gpu}") - OSRFLinuxCompilationAnyGitHub.create(multi_any_job, "osrf/gazebo", true, true, [], DISABLE_GITHUB_INTEGRATION) + OSRFLinuxCompilationAnyGitHub.create(multi_any_job, "gazebosim/gazebo-classic", true, true, [], DISABLE_GITHUB_INTEGRATION) multi_any_job.with { parameters @@ -239,7 +239,7 @@ gazebo_supported_branches.each { branch -> if (is_watched_by_buildcop(branch, distro, gpu)) Globals.extra_emails = Globals.build_cop_email OSRFLinuxCompilation.create(gazebo_ci_job) - OSRFGitHub.create(gazebo_ci_job, "osrf/gazebo", branch) + OSRFGitHub.create(gazebo_ci_job, "gazebosim/gazebo-classic", branch) gazebo_ci_job.with { @@ -273,7 +273,7 @@ ci_distro.each { distro -> experimental_arches.each { arch -> def gazebo_ci_job = job("gazebo-ci-gazebo11-${distro}-${arch}-gpu-none") OSRFLinuxCompilation.create(gazebo_ci_job) - OSRFGitHub.create(gazebo_ci_job, "osrf/gazebo", "gazebo11") + OSRFGitHub.create(gazebo_ci_job, "gazebosim/gazebo-classic", "gazebo11") gazebo_ci_job.with { @@ -303,7 +303,7 @@ ci_distro.each { distro -> ci_gpu.each { gpu -> def gazebo_ci_job = job("gazebo-ci-coverage-${distro}-${arch}-gpu-${gpu}") OSRFLinuxCompilation.create(gazebo_ci_job) - OSRFGitHub.create(gazebo_ci_job, "osrf/gazebo", "gazebo11") + OSRFGitHub.create(gazebo_ci_job, "gazebosim/gazebo-classic", "gazebo11") gazebo_ci_job.with { triggers { @@ -338,7 +338,7 @@ ci_distro.each { distro -> gazebo_supported_build_types.each { build_type -> def gazebo_ci_job = job("gazebo-ci_BT${build_type}-gazebo11-${distro}-${arch}-gpu-none") OSRFLinuxCompilation.create(gazebo_ci_job) - OSRFGitHub.create(gazebo_ci_job, "osrf/gazebo", "gazebo11") + OSRFGitHub.create(gazebo_ci_job, "gazebosim/gazebo-classic", "gazebo11") gazebo_ci_job.with { @@ -445,7 +445,7 @@ all_debbuild_branches.each { branch -> // 1. ANY job @ SCM/5min String ci_build_any_job_name_brew = "gazebo-ci-pr_any-homebrew-amd64" def gazebo_brew_ci_any_job = job(ci_build_any_job_name_brew) -OSRFBrewCompilationAnyGitHub.create(gazebo_brew_ci_any_job, "osrf/gazebo") +OSRFBrewCompilationAnyGitHub.create(gazebo_brew_ci_any_job, "gazebosim/gazebo-classic") gazebo_brew_ci_any_job.with { label "osx_gazebo" @@ -487,7 +487,7 @@ all_branches.each { branch -> def gazebo_brew_ci_job = job("gazebo-ci-${branch}-homebrew-amd64") OSRFBrewCompilation.create(gazebo_brew_ci_job) - OSRFGitHub.create(gazebo_brew_ci_job, "osrf/gazebo", branch) + OSRFGitHub.create(gazebo_brew_ci_job, "gazebosim/gazebo-classic", branch) gazebo_brew_ci_job.with { @@ -516,7 +516,7 @@ all_branches.each { branch -> // 1. any String ci_build_any_job_name_win7 = "gazebo-ci-pr_any-windows7-amd64" def gazebo_win_ci_any_job = job(ci_build_any_job_name_win7) - OSRFWinCompilationAnyGitHub.create(gazebo_win_ci_any_job, "osrf/gazebo") + OSRFWinCompilationAnyGitHub.create(gazebo_win_ci_any_job, "gazebosim/gazebo-classic") gazebo_win_ci_any_job.with { steps { @@ -531,7 +531,7 @@ all_branches = gazebo_supported_branches all_branches.each { branch -> def gazebo_win_ci_job = job("gazebo-ci-${branch}-windows7-amd64") OSRFWinCompilation.create(gazebo_win_ci_job) - OSRFGitHub.create(gazebo_win_ci_job, "osrf/gazebo", branch) + OSRFGitHub.create(gazebo_win_ci_job, "gazebosim/gazebo-classic", branch) gazebo_win_ci_job.with { From 6cdea0cf184bdf095f5d66ed953b97efa6b98eb7 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 28 Oct 2022 13:12:01 +0200 Subject: [PATCH 251/540] Implement the gz_*-install jobs for Garden (#838) * Implement the gz_*-install jobs for Garden Signed-off-by: Jose Luis Rivero Co-authored-by: Steve Peters --- jenkins-scripts/dsl/ignition.dsl | 36 ++++++++++++++------- jenkins-scripts/dsl/ignition_collection.dsl | 30 ++++++++--------- 2 files changed, 40 insertions(+), 26 deletions(-) diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index 4576eb52b..9d64c7861 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -14,6 +14,7 @@ gz_software = [ 'cmake', 'plugin', 'rendering', 'sensors', + 'sim', 'tools', 'transport', 'utils' ] @@ -161,6 +162,10 @@ Map merge_maps(Map[] sources) { // -dev package. ArrayList supported_ign_branches(String ign_software) { + // sim was not used in ignition + if (ign_software == 'sim') + return [''] + major_versions_registered = ignition_branches["${ign_software}"] if (major_versions_registered == null) @@ -173,6 +178,10 @@ ArrayList supported_ign_branches(String ign_software) // -dev package. ArrayList supported_gz_branches(String gz_software) { + // sim was not used in ignition + if (gz_software == 'gazebo') + return [''] + major_versions_registered = gz_branches["${gz_software}"] if (major_versions_registered == null) @@ -255,7 +264,6 @@ ArrayList all_debbuilders() // return all ci branch names // Map with the form of: major versions as keys. // Lists of distros supported as values -// @TODO[scpeters] update this to support gz_branches after garden release Map supported_install_pkg_branches(String gz_software) { major_versions_prerelease = gz_prerelease_pkgs["${gz_software}"] @@ -264,14 +272,16 @@ Map supported_install_pkg_branches(String gz_software) // all_supported_distros map_of_stable_versions = [:] map_of_stable_versions[gz_software] = [:] - supported_ign_branches(gz_software).each { major_version -> + (supported_ign_branches(gz_software) + supported_gz_branches(gz_software)).each { major_version -> + if (! major_version) + return false // keep looping new_relation = [:] new_relation[major_version] = all_supported_distros map_of_stable_versions[gz_software] << new_relation } if (major_versions_prerelease == null) - return map_of_stable_versions[gz_software]; + return map_of_stable_versions[gz_software] return merge_maps(map_of_stable_versions[gz_software], major_versions_prerelease) @@ -417,7 +427,8 @@ gz_software.each { gz_sw -> } // end of ci_any_job // add ci-pr_any to the list for CIWorkflow - ci_pr_any_list[software_name] << gz_ci_job_name + if (gz_sw != 'sim') + ci_pr_any_list[software_name] << gz_ci_job_name } } @@ -469,15 +480,16 @@ gz_software.each { gz_sw -> // No 1-dev or 0-dev packages (except special cases see // gz_debbuild variable), unversioned + major_version_in_pkgname = major_version if ("${major_version}" == "0" || "${major_version}" == "1") - major_version = "" + major_version_in_pkgname = "" - // -------------------------------------------------------------- - // ignition_ prefix packages: - generate_install_job("ignition", gz_sw, major_version, distro, arch) - // -------------------------------------------------------------- - // gz_ prefix packages: + // 1. gz_ prefix packages. All but not gazebo (replaced by sim) generate_install_job("gz", gz_sw.replace('gazebo', 'sim'), major_version, distro, arch) + + // 2. ignition_ prefix packages. gz software does not have ignition packages + if (major_version in supported_ign_branches(gz_sw)) + generate_install_job("ignition", gz_sw, major_version_in_pkgname, distro, arch) } } } @@ -632,8 +644,10 @@ gz_software.each { gz_sw -> } } + // do not add gazebo twice // add ci-pr_any to the list for CIWorkflow - ci_pr_any_list[software_name] << gz_brew_ci_any_job_name + if (gz_sw != 'sim') + ci_pr_any_list[software_name] << gz_brew_ci_any_job_name // 2. main, release branches all_branches("${software_name}").each { branch -> diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index bb79cf2d6..024465dfa 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -205,22 +205,22 @@ gz_collection_jobs = 'sdformat-sdf-12-win.xml' ], 'garden' : [ - 'gz_cmake2-install-pkg-focal-amd64', - 'gz_common4-install-pkg-focal-amd64', - 'gz_fuel-tools7-install-pkg-focal-amd64', + 'gz_cmake3-install-pkg-focal-amd64', + 'gz_common5-install-pkg-focal-amd64', + 'gz_fuel-tools8-install-pkg-focal-amd64', 'gz_garden-install-pkg-focal-amd64', - 'gz_gazebo6-install-pkg-focal-amd64', - 'gz_gui6-install-pkg-focal-amd64', - 'gz_launch5-install-pkg-focal-amd64', - 'gz_math6-install-pkg-focal-amd64', - 'gz_msgs8-install-pkg-focal-amd64', - 'gz_physics5-install-pkg-focal-amd64', - 'gz_plugin-install-pkg-focal-amd64', - 'gz_rendering6-install-pkg-focal-amd64', - 'gz_sensors6-install-pkg-focal-amd64', - 'gz_tools-install-pkg-focal-amd64', - 'gz_transport11-install-pkg-focal-amd64', - 'gz_utils-install-pkg-focal-amd64', + 'gz_gazebo7-install-pkg-focal-amd64', + 'gz_gui7-install-pkg-focal-amd64', + 'gz_launch6-install-pkg-focal-amd64', + 'gz_math7-install-pkg-focal-amd64', + 'gz_msgs9-install-pkg-focal-amd64', + 'gz_physics6-install-pkg-focal-amd64', + 'gz_plugin2-install-pkg-focal-amd64', + 'gz_rendering7-install-pkg-focal-amd64', + 'gz_sensors7-install-pkg-focal-amd64', + 'gz_tools2-install-pkg-focal-amd64', + 'gz_transport12-install-pkg-focal-amd64', + 'gz_utils2-install-pkg-focal-amd64', 'ign_cmake-gz-3-win', 'ign_common-gz-5-win', 'ign_fuel-tools-gz-8-win', From 440a5920b829a2d186d1f53e633f83f48bce4c2e Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 3 Nov 2022 00:31:37 +0100 Subject: [PATCH 252/540] Workaround to enable rendering on Win only in selected nodes (#847) * Workaround to enable rendering on Win only in selected nodes * Extend win_rendering to gui/sensors/sim Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/ignition.dsl | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index 9d64c7861..2c7e53686 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -741,9 +741,10 @@ gz_software.each { gz_sw -> gz_software_name = gz_sw // Necessary substitution. gz_sw won't overwrite // ign-gazebo only support Windows from ign-gazebo5 - if (gz_sw == 'gazebo') + if (gz_sw == 'gazebo') { supported_branches = [ 'ign-gazebo6', 'gz-sim7', 'main' ] gz_software_name = "sim" + } // ign-launch only support Windows from ign-launch5 if (gz_sw == 'launch') @@ -758,6 +759,13 @@ gz_software.each { gz_sw -> enable_cmake_warnings(gz_sw)) gz_win_ci_any_job.with { + if (gz_sw == 'gui' || + gz_sw == 'rendering' || + gz_sw == 'sensors' || + gz_software_name == 'sim') + label('win_rendering') + + steps { batchFile("""\ call "./scripts/jenkins-scripts/ign_${gz_sw}-default-devel-windows-amd64.bat" @@ -809,6 +817,12 @@ gz_software.each { gz_sw -> call "./scripts/jenkins-scripts/ign_${gz_sw}-default-devel-windows-amd64.bat" """.stripIndent()) } + + if (gz_sw == 'gui' || + gz_sw == 'rendering' || + gz_sw == 'sensors' || + gz_software_name == 'sim') + label('win_rendering') } } } From 0faf09102f342eb64cfa59743d8f79461db7f939 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 3 Nov 2022 16:12:39 +0100 Subject: [PATCH 253/540] Do not generate ignition_sim-* jobs (#845) Signed-off-by: Jose Luis Rivero Signed-off-by: Steve Peters Co-authored-by: Steve Peters --- jenkins-scripts/dsl/ignition.dsl | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index 2c7e53686..b3e78518a 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -499,7 +499,7 @@ void generate_asan_ci_job(gz_ci_job, gz_sw, branch, distro, arch) { generate_ci_job(gz_ci_job, gz_sw, branch, distro, arch, '-DGZ_SANITIZER=Address', - Globals.MAKETEST_SKIP_GZ, + Globals.MAKETEST_SKIP_GZ, 'export ASAN_OPTIONS=check_initialization_order=true') } @@ -727,6 +727,9 @@ gz_software.each { gz_sw -> // 1. any gz_software.each { gz_sw -> + if (gz_sw == 'sim') + return + if (is_a_colcon_package(gz_sw)) { // colcon uses long paths and windows has a hard limit of 260 chars. Keep // names minimal @@ -738,12 +741,10 @@ gz_software.each { gz_sw -> } supported_branches = [] - gz_software_name = gz_sw // Necessary substitution. gz_sw won't overwrite // ign-gazebo only support Windows from ign-gazebo5 if (gz_sw == 'gazebo') { supported_branches = [ 'ign-gazebo6', 'gz-sim7', 'main' ] - gz_software_name = "sim" } // ign-launch only support Windows from ign-launch5 @@ -762,7 +763,7 @@ gz_software.each { gz_sw -> if (gz_sw == 'gui' || gz_sw == 'rendering' || gz_sw == 'sensors' || - gz_software_name == 'sim') + gz_sw == 'gazebo') label('win_rendering') @@ -821,7 +822,7 @@ gz_software.each { gz_sw -> if (gz_sw == 'gui' || gz_sw == 'rendering' || gz_sw == 'sensors' || - gz_software_name == 'sim') + gz_sw == 'gazebo') label('win_rendering') } } @@ -829,6 +830,8 @@ gz_software.each { gz_sw -> // Main CI workflow gz_software.each { gz_sw -> + if (gz_sw == 'sim') + return def String ci_main_name = "ignition_${gz_sw}-ci-manual_any" def gz_ci_main = pipelineJob(ci_main_name) OSRFCIWorkFlowMultiAnyGitHub.create(gz_ci_main, ci_pr_any_list[gz_sw]) From a72a739593dc426991e8badc5bfad4c837c92f70 Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Wed, 2 Nov 2022 12:18:49 -0700 Subject: [PATCH 254/540] Remove references to GAZEBO_EXPERIMENTAL_BUILD The GAZEBO_EXPERIMENTAL_BUILD is very outdated and not supported, so remove references to it from CI scripts. Signed-off-by: Steve Peters --- .../docker/lib/_gazebo_version_hook.bash | 11 +- .../docker/lib/gazebo-base-default.bash | 3 - jenkins-scripts/lib/dependencies_archive.sh | 126 ++++++++---------- 3 files changed, 58 insertions(+), 82 deletions(-) diff --git a/jenkins-scripts/docker/lib/_gazebo_version_hook.bash b/jenkins-scripts/docker/lib/_gazebo_version_hook.bash index 7e59d241b..9a7695266 100644 --- a/jenkins-scripts/docker/lib/_gazebo_version_hook.bash +++ b/jenkins-scripts/docker/lib/_gazebo_version_hook.bash @@ -1,12 +1,7 @@ # Identify GAZEBO_MAJOR_VERSION to help with dependency resolution -if [ "${GAZEBO_EXPERIMENTAL_BUILD}" = true ]; then - # Identify GAZEBO_MAJOR_VERSION to help with dependency resolution - GAZEBO_MAJOR_VERSION=0 -else - GAZEBO_MAJOR_VERSION=$(\ - python3 ${SCRIPT_DIR}/../tools/detect_cmake_major_version.py \ - ${WORKSPACE}/gazebo/CMakeLists.txt) -fi +GAZEBO_MAJOR_VERSION=$(\ + python3 ${SCRIPT_DIR}/../tools/detect_cmake_major_version.py \ + ${WORKSPACE}/gazebo/CMakeLists.txt) # Check gazebo version is integer if ! [[ ${GAZEBO_MAJOR_VERSION} =~ ^-?[0-9]+$ ]]; then diff --git a/jenkins-scripts/docker/lib/gazebo-base-default.bash b/jenkins-scripts/docker/lib/gazebo-base-default.bash index db2441ce8..74ac702df 100644 --- a/jenkins-scripts/docker/lib/gazebo-base-default.bash +++ b/jenkins-scripts/docker/lib/gazebo-base-default.bash @@ -49,9 +49,6 @@ if [[ $GAZEBO_MAJOR_VERSION -lt 8 ]]; then fi SOFTWARE_DIR="gazebo" -if [ "${GAZEBO_EXPERIMENTAL_BUILD}" = true ]; then - SOFTWARE_DIR="${SOFTWARE_DIR}_experimental" -fi cat > build.sh << DELIM_DART ################################################### diff --git a/jenkins-scripts/lib/dependencies_archive.sh b/jenkins-scripts/lib/dependencies_archive.sh index 1ae78f86c..2f8ed4c02 100644 --- a/jenkins-scripts/lib/dependencies_archive.sh +++ b/jenkins-scripts/lib/dependencies_archive.sh @@ -142,79 +142,63 @@ if $DART_FROM_PKGS; then fi fi -# By default, the stable version of gazebo -[[ -z ${GAZEBO_EXPERIMENTAL_BUILD} ]] && GAZEBO_EXPERIMENTAL_BUILD=false -if ! ${GAZEBO_EXPERIMENTAL_BUILD}; then - # -------------------------------------- - # GAZEBO - current version - # -------------------------------------- - - GAZEBO_BASE_DEPENDENCIES_NO_SDFORMAT="libfreeimage-dev \\ - libprotoc-dev \\ - libprotobuf-dev \\ - protobuf-compiler \\ - freeglut3-dev \\ - libcurl4-openssl-dev \\ - libtinyxml-dev \\ - libtinyxml2-dev \\ - libtar-dev \\ - libtbb-dev \\ - ${ogre_pkg} \\ - libxml2-dev \\ - pkg-config \\ - qtbase5-dev \\ - libqwt-qt5-dev \\ - libltdl-dev \\ - libgts-dev \\ - libboost-thread-dev \\ - libboost-system-dev \\ - libboost-filesystem-dev \\ - libboost-program-options-dev \\ - libboost-regex-dev \\ - libboost-iostreams-dev \\ - ${bullet_pkg} \\ - libsimbody-dev \\ - ${dart_pkgs}" - - if [[ ${GAZEBO_MAJOR_VERSION} -ge 11 ]]; then - GAZEBO_BASE_DEPENDENCIES_NO_SDFORMAT="${GAZEBO_BASE_DEPENDENCIES_NO_SDFORMAT} \\ - libignition-common3-dev \\ - libignition-fuel-tools4-dev \\ - libignition-transport8-dev \\ - libignition-math6-dev \\ - libignition-msgs5-dev" - elif [[ ${GAZEBO_MAJOR_VERSION} -ge 9 ]]; then - GAZEBO_BASE_DEPENDENCIES_NO_SDFORMAT="${GAZEBO_BASE_DEPENDENCIES_NO_SDFORMAT} \\ - libignition-common-dev \\ - libignition-fuel-tools1-dev \\ - libignition-transport4-dev \\ - libignition-math4-dev \\ - libignition-msgs-dev" - fi +# -------------------------------------- +# GAZEBO classic +# -------------------------------------- + +GAZEBO_BASE_DEPENDENCIES_NO_SDFORMAT="libfreeimage-dev \\ + libprotoc-dev \\ + libprotobuf-dev \\ + protobuf-compiler \\ + freeglut3-dev \\ + libcurl4-openssl-dev \\ + libtinyxml-dev \\ + libtinyxml2-dev \\ + libtar-dev \\ + libtbb-dev \\ + ${ogre_pkg} \\ + libxml2-dev \\ + pkg-config \\ + qtbase5-dev \\ + libqwt-qt5-dev \\ + libltdl-dev \\ + libgts-dev \\ + libboost-thread-dev \\ + libboost-system-dev \\ + libboost-filesystem-dev \\ + libboost-program-options-dev \\ + libboost-regex-dev \\ + libboost-iostreams-dev \\ + ${bullet_pkg} \\ + libsimbody-dev \\ + ${dart_pkgs}" + +if [[ ${GAZEBO_MAJOR_VERSION} -ge 11 ]]; then + GAZEBO_BASE_DEPENDENCIES_NO_SDFORMAT="${GAZEBO_BASE_DEPENDENCIES_NO_SDFORMAT} \\ + libignition-common3-dev \\ + libignition-fuel-tools4-dev \\ + libignition-transport8-dev \\ + libignition-math6-dev \\ + libignition-msgs5-dev" +elif [[ ${GAZEBO_MAJOR_VERSION} -ge 9 ]]; then + GAZEBO_BASE_DEPENDENCIES_NO_SDFORMAT="${GAZEBO_BASE_DEPENDENCIES_NO_SDFORMAT} \\ + libignition-common-dev \\ + libignition-fuel-tools1-dev \\ + libignition-transport4-dev \\ + libignition-math4-dev \\ + libignition-msgs-dev" +fi - GAZEBO_BASE_DEPENDENCIES="${GAZEBO_BASE_DEPENDENCIES_NO_SDFORMAT} \\ - ${sdformat_pkg}" +GAZEBO_BASE_DEPENDENCIES="${GAZEBO_BASE_DEPENDENCIES_NO_SDFORMAT} \\ + ${sdformat_pkg}" - GAZEBO_EXTRA_DEPENDENCIES="libavformat-dev \\ - libavcodec-dev \\ - libgdal-dev \\ - libgraphviz-dev \\ - libswscale-dev \\ - libavdevice-dev \\ - ruby-ronn" -else - # -------------------------------------- - # GAZEBO - experimental version - # -------------------------------------- - GAZEBO_BASE_DEPENDENCIES="libgflags-dev \\ - pkg-config \\ - libprotoc-dev \\ - libprotobuf-dev \\ - protobuf-compiler \\ - ${pythonv}-protobuf \\ - libignition-common-dev \\ - libignition-msgs-dev" -fi +GAZEBO_EXTRA_DEPENDENCIES="libavformat-dev \\ + libavcodec-dev \\ + libgdal-dev \\ + libgraphviz-dev \\ + libswscale-dev \\ + libavdevice-dev \\ + ruby-ronn" if [[ -z $ROS_DISTRO ]]; then From 28811cfd81227c0d7d755eca2ce34363caa6a7bf Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Wed, 2 Nov 2022 12:30:44 -0700 Subject: [PATCH 255/540] Remove outdated gazebo-classic version checks Signed-off-by: Steve Peters --- jenkins-scripts/docker/gazebo-abichecker.bash | 3 --- .../docker/lib/gazebo-base-default.bash | 16 +++++----------- .../gazebo-default-devel-homebrew-amd64.bash | 11 +---------- 3 files changed, 6 insertions(+), 24 deletions(-) diff --git a/jenkins-scripts/docker/gazebo-abichecker.bash b/jenkins-scripts/docker/gazebo-abichecker.bash index d0d830b05..fbce3b0f5 100644 --- a/jenkins-scripts/docker/gazebo-abichecker.bash +++ b/jenkins-scripts/docker/gazebo-abichecker.bash @@ -19,9 +19,6 @@ fi export ABI_JOB_SOFTWARE_NAME="gazebo" export ABI_JOB_REPOS="stable" export ABI_JOB_PKG_DEPENDENCIES_VAR_NAME="GAZEBO_BASE_DEPENDENCIES" -if [[ $GAZEBO_MAJOR_VERSION -lt 8 ]]; then - export ABI_JOB_CMAKE_PARAMS="-DENABLE_TESTS_COMPILATION:BOOL=False" -fi export ABI_JOB_IGNORE_HEADERS="gazebo/GIMPACT gazebo/opcode gazebo/test" . ${SCRIPT_DIR}/lib/generic-abi-base.bash diff --git a/jenkins-scripts/docker/lib/gazebo-base-default.bash b/jenkins-scripts/docker/lib/gazebo-base-default.bash index 74ac702df..d6e5171e1 100644 --- a/jenkins-scripts/docker/lib/gazebo-base-default.bash +++ b/jenkins-scripts/docker/lib/gazebo-base-default.bash @@ -44,10 +44,6 @@ if ${COVERAGE_ENABLED} ; then EXTRA_PACKAGES="${EXTRA_PACKAGES} wget" fi -if [[ $GAZEBO_MAJOR_VERSION -lt 8 ]]; then - GAZEBO_BASE_CMAKE_ARGS="${GAZEBO_BASE_CMAKE_ARGS} -DENABLE_TESTS_COMPILATION=True" -fi - SOFTWARE_DIR="gazebo" cat > build.sh << DELIM_DART @@ -166,13 +162,11 @@ make -j${MAKE_JOBS} stop_stopwatch COMPILATION echo '# END SECTION' -if [[ $GAZEBO_MAJOR_VERSION -ge 8 ]]; then - echo '# BEGIN SECTION: Tests compilation' - init_stopwatch TESTS_COMPILATION - make -j${MAKE_JOBS} tests - stop_stopwatch TESTS_COMPILATION - echo '# END SECTION' -fi +echo '# BEGIN SECTION: Tests compilation' +init_stopwatch TESTS_COMPILATION +make -j${MAKE_JOBS} tests +stop_stopwatch TESTS_COMPILATION +echo '# END SECTION' echo '# BEGIN SECTION: Gazebo installation' sudo make install diff --git a/jenkins-scripts/gazebo-default-devel-homebrew-amd64.bash b/jenkins-scripts/gazebo-default-devel-homebrew-amd64.bash index b154d713d..00a9a5b5f 100644 --- a/jenkins-scripts/gazebo-default-devel-homebrew-amd64.bash +++ b/jenkins-scripts/gazebo-default-devel-homebrew-amd64.bash @@ -4,22 +4,13 @@ [[ -L ${0} ]] && SCRIPT_DIR=$(readlink ${0}) || SCRIPT_DIR=${0} SCRIPT_DIR="${SCRIPT_DIR%/*}" -# Identify GAZEBO_MAJOR_VERSION to help with dependency resolution -GAZEBO_MAJOR_VERSION=$(\ - python3 ${SCRIPT_DIR}/tools/detect_cmake_major_version.py \ - ${WORKSPACE}/gazebo/CMakeLists.txt) +RERUN_FAILED_TESTS=1 -if [ $GAZEBO_MAJOR_VERSION -ge 7 ]; then - RERUN_FAILED_TESTS=1 -fi - -if [[ $GAZEBO_MAJOR_VERSION -ge 8 ]]; then PRE_TESTS_EXECUTION_HOOK=""" echo '#BEGIN SECTION: compile the tests suite' make tests -j\${MAKE_JOBS} \${MAKE_VERBOSE_STR} echo '# END SECTION' """ -fi # clear the heightmap paging cache rm -rf $HOME/.gazebo/paging From 41f319af687ddc888896ebcae94a4017f47b2bd2 Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Wed, 2 Nov 2022 12:31:34 -0700 Subject: [PATCH 256/540] Update gazebo-classic source folder Follow up to the GitHub URL renaming from #843. Signed-off-by: Steve Peters --- jenkins-scripts/docker/lib/_gazebo_version_hook.bash | 2 +- jenkins-scripts/docker/lib/gazebo-base-default.bash | 8 ++++---- jenkins-scripts/gazebo-default-devel-homebrew-amd64.bash | 2 +- jenkins-scripts/gazebo-default-devel-windows7-amd64.bat | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/jenkins-scripts/docker/lib/_gazebo_version_hook.bash b/jenkins-scripts/docker/lib/_gazebo_version_hook.bash index 9a7695266..df2a6bf74 100644 --- a/jenkins-scripts/docker/lib/_gazebo_version_hook.bash +++ b/jenkins-scripts/docker/lib/_gazebo_version_hook.bash @@ -1,7 +1,7 @@ # Identify GAZEBO_MAJOR_VERSION to help with dependency resolution GAZEBO_MAJOR_VERSION=$(\ python3 ${SCRIPT_DIR}/../tools/detect_cmake_major_version.py \ - ${WORKSPACE}/gazebo/CMakeLists.txt) + ${WORKSPACE}/gazebo-classic/CMakeLists.txt) # Check gazebo version is integer if ! [[ ${GAZEBO_MAJOR_VERSION} =~ ^-?[0-9]+$ ]]; then diff --git a/jenkins-scripts/docker/lib/gazebo-base-default.bash b/jenkins-scripts/docker/lib/gazebo-base-default.bash index d6e5171e1..b77fbbd8b 100644 --- a/jenkins-scripts/docker/lib/gazebo-base-default.bash +++ b/jenkins-scripts/docker/lib/gazebo-base-default.bash @@ -44,7 +44,7 @@ if ${COVERAGE_ENABLED} ; then EXTRA_PACKAGES="${EXTRA_PACKAGES} wget" fi -SOFTWARE_DIR="gazebo" +SOFTWARE_DIR="gazebo-classic" cat > build.sh << DELIM_DART ################################################### @@ -79,8 +79,8 @@ if ${COVERAGE_ENABLED} ; then set -x # back to debug # Set up Bullseyes for compiling export PATH=/usr/bullseyes/bin:\$PATH - export COVFILE=$WORKSPACE/gazebo/test.cov - cd $WORKSPACE/gazebo + export COVFILE=$WORKSPACE/${SOFTWARE_DIR}/test.cov + cd $WORKSPACE/${SOFTWARE_DIR} covselect --file test.cov --add . cov01 --on echo '# END SECTION' @@ -205,7 +205,7 @@ if ${COVERAGE_ENABLED} ; then rm -fr $WORKSPACE/bullshtml mkdir -p $WORKSPACE/coverage covselect --add '!$WORKSPACE/build/' '!../build/' '!test/' '!tools/test/' '!deps/' '!/opt/' '!gazebo/rendering/skyx/' '!/tmp/' - covhtml --srcdir $WORKSPACE/gazebo/ $WORKSPACE/coverage + covhtml --srcdir $WORKSPACE/${SOFTWARE_DIR}/ $WORKSPACE/coverage # Generate valid cover.xml file using the bullshtml software # java is needed to run bullshtml apt-get install -y default-jre diff --git a/jenkins-scripts/gazebo-default-devel-homebrew-amd64.bash b/jenkins-scripts/gazebo-default-devel-homebrew-amd64.bash index 00a9a5b5f..36618ae4d 100644 --- a/jenkins-scripts/gazebo-default-devel-homebrew-amd64.bash +++ b/jenkins-scripts/gazebo-default-devel-homebrew-amd64.bash @@ -15,4 +15,4 @@ echo '# END SECTION' # clear the heightmap paging cache rm -rf $HOME/.gazebo/paging -. ${SCRIPT_DIR}/lib/project-default-devel-homebrew-amd64.bash gazebo +. ${SCRIPT_DIR}/lib/project-default-devel-homebrew-amd64.bash gazebo-classic diff --git a/jenkins-scripts/gazebo-default-devel-windows7-amd64.bat b/jenkins-scripts/gazebo-default-devel-windows7-amd64.bat index cf09bcc96..d44d52a33 100644 --- a/jenkins-scripts/gazebo-default-devel-windows7-amd64.bat +++ b/jenkins-scripts/gazebo-default-devel-windows7-amd64.bat @@ -1,6 +1,6 @@ set SCRIPT_DIR=%~dp0 set PLATFORM_TO_BUILD=amd64 -set VCS_DIRECTORY=gazebo +set VCS_DIRECTORY=gazebo-classic for /f %%i in ('python "%SCRIPT_DIR%\tools\detect_cmake_major_version.py" "%WORKSPACE%\%VCS_DIRECTORY%\CMakeLists.txt"') do set GAZEBO_MAJOR_VERSION=%%i From 747dec201aa490afc084d0e7516233cda85d9c0e Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 3 Nov 2022 22:45:42 +0100 Subject: [PATCH 257/540] Add -fPIC to abichecker for qt dependent builds Signed-off-by: Jose Luis Rivero --- jenkins-scripts/docker/ignition-abichecker.bash | 10 ++++++++++ jenkins-scripts/docker/lib/generic-abi-base.bash | 5 +++++ 2 files changed, 15 insertions(+) diff --git a/jenkins-scripts/docker/ignition-abichecker.bash b/jenkins-scripts/docker/ignition-abichecker.bash index bb5a2cf93..253579513 100644 --- a/jenkins-scripts/docker/ignition-abichecker.bash +++ b/jenkins-scripts/docker/ignition-abichecker.bash @@ -73,6 +73,16 @@ then fi fi +if [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-gazebo" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -ge 7 ]] || \ + [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-gui" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -ge 7 ]] || \ + [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-launch" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -ge 6 ]] || \ + [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-sensors" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -ge 7 ]] || \ + [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-rendering" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -ge 7 ]] +then + # -fPIC needed to compile Qt + export ABI_JOB_EXTRA_GCC_OPTIONS="-fPIC" +fi + # default to use stable repos export ABI_JOB_REPOS="stable" diff --git a/jenkins-scripts/docker/lib/generic-abi-base.bash b/jenkins-scripts/docker/lib/generic-abi-base.bash index fabf03b2f..3cb5ba650 100644 --- a/jenkins-scripts/docker/lib/generic-abi-base.bash +++ b/jenkins-scripts/docker/lib/generic-abi-base.bash @@ -11,6 +11,8 @@ # ABI_JOB_CMAKE_PARAMS: (option) cmake parameters to be pased to cmake configuration # ABI_JOB_IGNORE_HEADERS: (optional) relative (to root project path) list (space separated) # of path headers to ignore +# ABI_JOB_EXTRA_GCC_OPTIONS: (optional) inject gcc_options in the descriptor file +# one per line # Jenkins variables: # DEST_BRANCH @@ -100,6 +102,7 @@ sudo perl Makefile.pl -install --prefix=/usr mkdir -p $WORKSPACE/abi_checker cd $WORKSPACE/abi_checker + cat > pkg.xml << CURRENT_DELIM branch: $DEST_BRANCH @@ -126,6 +129,7 @@ cat >> pkg.xml << CURRENT_DELIM_LIBS -std=${ABI_CXX_STANDARD} + ${ABI_JOB_EXTRA_GCC_OPTIONS} CURRENT_DELIM_LIBS @@ -156,6 +160,7 @@ cat >> devel.xml << DEVEL_DELIM_LIBS -std=${ABI_CXX_STANDARD} + ${ABI_JOB_EXTRA_GCC_OPTIONS} DEVEL_DELIM_LIBS echo '# END SECTION' From 191255577a5b7d8d4b71dbf1f60e1e6ac1e1574d Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 4 Nov 2022 16:06:23 +0100 Subject: [PATCH 258/540] Replace release instructions in README by links to gazebosim Signed-off-by: Jose Luis Rivero --- README.md | 88 ++++--------------------------------------------------- 1 file changed, 5 insertions(+), 83 deletions(-) diff --git a/README.md b/README.md index 817f2abbe..f63255889 100644 --- a/README.md +++ b/README.md @@ -12,87 +12,9 @@ Gazebo software. ### Making releases -#### One-time configuration +For the developers officially maintaining Gazebo software that need to run a new release of the software: -1. `sudo apt install devscripts s3cmd` -1. `s3cmd --configure` - * Check with one of the maintainers to get the necessary credentials -1. `export DEBEMAIL="@openrobotics.org"` -1. `export DEBFULLNAME=""` - -**Note:** You may wish to add the two above exported variables to your `.bashrc` - -### For each release - -1. Check if this is a good time for the release - * Ask team if there are any concerns about making the release. - * Check if there are changes to previous versions that need to be forward-ported. - * See if there are open PRs against that release branch that could go into the new release. -1. Choose how to bump the version - * Bump major if the release includes breaking changes - this is usually done for all libraries at around the same time. - * Bump minor if there are new backwards-compatible features, like new APIs. - * Bump patch if there are only bug fixes that won't affect the API. -1. Open pull request updating the version on the source code - * [Example pull request.](https://github.com/gazebosim/gz-physics/pull/132) - * Update the version on `CMakeLists.txt`. - * Include a link comparing the current branch to the latest release. - 1. [Example branch comparison.](https://github.com/gazebosim/gz-sim/compare/ignition-gazebo3_3.5.0...ign-gazebo3) - 1. Substitute the package version and name that are relevant to your release. - * Update the changelog. - 1. Use the [source_changelog.bash](source_changelog.bash) script to generate the Changelog.md entries newer than the most recent tag. - * `cd ` - * `bash /source_changelog.bash ` - * e.g. `bash ../release-tools/source_changelog.bash 3.5.0` - 1. Verify the Changelog.md entries using the GitHub branch comparison. - * Update the migration guide as needed. - * Wait for this pull request to be merged before proceeding. -1. Clone the appropriate release repository from https://github.com/gazebo-release. - * For example, `git clone https://github.com/gazebo-release/ign-physics3-release` -1. If the versions of any dependencies have increased in the source code, open a pull request updating them on the release repo - * [Example pull request.](https://github.com/gazebo-release/gz-sim3-release/pull/4) -1. Wait for all pull requests opened above to be approved and merged. -1. Clone `release-tools` - * `git clone https://github.com/gazebo-tooling/release-tools` -1. Update the release-repo's changelog: - 1. `cd ` - 1. `/release-repo-scripts/changelog_spawn.sh ` - * Where `` is something like `3.1.0-1` or `4.0.0~pre1-1` - * You will need to force run the above command with `bash` (by prefacing the command with `bash`) if you are using `sh`. -1. Trigger release: - 1. `cd ` - 1. Make sure you're in the branch to be released, after the pull request bumping the version has been merged. - 1. Dry-run `release.py` with the appropriate arguments. Some examples: - 1. New Gazebo minor release and Gazebo classic release: - 1. `/release.py -r 1 ign-physics3 3.1.0 secrettoken --dry-run` - 1. `/release.py -r 1 gazebo11 11.2.0 secrettoken --dry-run` - 1. Pre-release: `/release.py -r 1 ign-physics3 3.0.0~pre1 --upload-to-repo=prerelease secrettoken --dry-run` - 1. If the dry run succeeds, run the same command again, now without `--dry-run`. -1. Check that: - * Several `-debbuilder` jobs have been queued in https://build.osrfoundation.org/ and watch those jobs to see if any of them fail or are marked unstable. - While it would be easier to track the status of these jobs if there was a [dashboard](https://github.com/gazebo-tooling/release-tools/issues/295), - you can watch the page for a specific debbuild, such as https://build.osrfoundation.org/job/ign-gui3-debbuilder/. - 1. If you observe a failure of an Gazebo Dome package, check the list of supported - architectures in [issue #297](https://github.com/gazebo-tooling/release-tools/issues/297); - for example Bionic/armhf and Focal/armhf are not supported for Dome. - To disable builds for a specific architecture, add a file to the `-release` repository that starts with `.releasepy_NO_ARCH_` - and append the name of the architecture to be excluded. The file can be added to the - root of the `-release` repository (like [.releasepy_NO_ARCH_ARM64 in ign-gazebo2-release](https://github.com/gazebo-release/ign-gazebo2-release/blob/master/.releasepy_NO_ARCH_ARM64)) - or in a distro-specific sub-folder (like [bionic/.releasepy_NO_ARCH_armhf in ign-launch2-release](https://github.com/gazebo-release/ign-launch2-release/blob/master/bionic/.releasepy_NO_ARCH_armhf)). - The architecture suffix is not case-sensitive. - 1. To check if a debbuild has previously succeeded for a given architecture, you can look at packages.osrfoundation.org - to see the most recent successful builds. For example, you can see the most recent - [Ubuntu builds of ignition-gazebo3](https://packages.osrfoundation.org/gazebo/ubuntu-stable/pool/main/i/ignition-gazebo3/) - or the [Debian builds of ignition-gazebo3](https://packages.osrfoundation.org/gazebo/debian-stable/pool/main/i/ignition-gazebo3/). - 1. If the failure is on a supported architecture, check the source repository for an existing report of this failure and if none - exists, please report the failure (see [gazebosim/gz-math#161](https://github.com/gazebosim/gz-math/issues/161) - for an example). - 1. If a build is unstable, check if it was unstable before this release and if it has already been reported. - A common cause of unstable debbuilds is newly installed files that are not captured by the pattenrs in the `.install` - files in the `-release` repository. This can be checked by searching for `dh_missing` in the console log of the - unstable build and looking for a list of uninstalled files. Example pull requests that fix problems with `dh_missing` - are [gazebo-release/gz-transport11-release#4](https://github.com/gazebo-release/gz-transport11-release/pull/4) - and [gazebo-release/gz-tools-release#4](https://github.com/gazebo-release/gz-tools-release/pull/4). - * A pull request was opened to https://github.com/osrf/homebrew-simulation - 1. This pull request may take a minute or two to open. - 1. Once it is open, make a comment containing the text "build bottle". - For further details, see the [README at osrf/homebrew-simulation](https://github.com/osrf/homebrew-simulation#to-build-bottles). + * The [Gazebo Release Process](https://gazebosim.org/docs/garden/release) + document provides an overview of how releasing works. + * The [Gazebo Release Instructions](https://gazebosim.org/docs/garden/releases-instructions) + document provides step by step instructions to run new releases. From e3e9a8e6ebd47a4298db492347c8f850339ca864 Mon Sep 17 00:00:00 2001 From: Aditya Pande Date: Mon, 14 Nov 2022 10:44:02 -0800 Subject: [PATCH 259/540] Update Changelog instructions The script to generate source changelog lives in the ``source-repo-scripts`` directory, updating that in the PR. --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 817f2abbe..f8a43d6d2 100644 --- a/README.md +++ b/README.md @@ -41,8 +41,8 @@ Gazebo software. * Update the changelog. 1. Use the [source_changelog.bash](source_changelog.bash) script to generate the Changelog.md entries newer than the most recent tag. * `cd ` - * `bash /source_changelog.bash ` - * e.g. `bash ../release-tools/source_changelog.bash 3.5.0` + * `bash /source-repo-scripts/source_changelog.bash ` + * e.g. `bash ../release-tools/source-repo-scripts/source_changelog.bash 3.5.0` 1. Verify the Changelog.md entries using the GitHub branch comparison. * Update the migration guide as needed. * Wait for this pull request to be merged before proceeding. From 1b2168f3ef259390bf7b31f1f8e77593118c8cbf Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Mon, 14 Nov 2022 22:51:27 -0800 Subject: [PATCH 260/540] gazebo.dsl: use 20.04 for pull request CI builds Signed-off-by: Steve Peters --- jenkins-scripts/dsl/gazebo.dsl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/jenkins-scripts/dsl/gazebo.dsl b/jenkins-scripts/dsl/gazebo.dsl index f56167d04..351bcadfc 100644 --- a/jenkins-scripts/dsl/gazebo.dsl +++ b/jenkins-scripts/dsl/gazebo.dsl @@ -8,8 +8,7 @@ def ubuntu_official_packages_distros = [ 'bionic' : 'gazebo9', 'focal' : 'gazebo9', 'jammy' : 'gazebo11'] // Main platform using for quick CI -// Using bionic until https://github.com/gazebosim/gazebo-classic/issues/3151 is fixed -def ci_distro = [ 'bionic' ] +def ci_distro = [ 'focal' ] def ci_gpu = Globals.get_ci_gpu() def abi_distro = Globals.get_abi_distro() // Other supported platform to be checked but no for quick From b03118ba530827702181015b90845481cd437b54 Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Tue, 15 Nov 2022 00:32:01 -0800 Subject: [PATCH 261/540] gazebo-abichecker: use gazebo-classic Signed-off-by: Steve Peters --- jenkins-scripts/docker/gazebo-abichecker.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/docker/gazebo-abichecker.bash b/jenkins-scripts/docker/gazebo-abichecker.bash index fbce3b0f5..e7997f188 100644 --- a/jenkins-scripts/docker/gazebo-abichecker.bash +++ b/jenkins-scripts/docker/gazebo-abichecker.bash @@ -16,7 +16,7 @@ fi . ${SCRIPT_DIR}/lib/_gazebo_version_hook.bash -export ABI_JOB_SOFTWARE_NAME="gazebo" +export ABI_JOB_SOFTWARE_NAME="gazebo-classic" export ABI_JOB_REPOS="stable" export ABI_JOB_PKG_DEPENDENCIES_VAR_NAME="GAZEBO_BASE_DEPENDENCIES" export ABI_JOB_IGNORE_HEADERS="gazebo/GIMPACT gazebo/opcode gazebo/test" From cb3fb0e7db1285713ba47e5aa617f5770807cbfa Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Tue, 15 Nov 2022 00:32:32 -0800 Subject: [PATCH 262/540] gazebo windows scripts: use gazebo-classic Signed-off-by: Steve Peters --- jenkins-scripts/lib/gazebo-base-windows.bat | 9 +++++---- jenkins-scripts/lib/gazebo9_10-base-windows.bat | 9 +++++---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/jenkins-scripts/lib/gazebo-base-windows.bat b/jenkins-scripts/lib/gazebo-base-windows.bat index 58cddc40d..b221ceee4 100644 --- a/jenkins-scripts/lib/gazebo-base-windows.bat +++ b/jenkins-scripts/lib/gazebo-base-windows.bat @@ -58,6 +58,7 @@ echo # END SECTION echo # BEGIN SECTION: reusing workspace :: Remove gazebo copy IF EXIST %LOCAL_WS%\gazebo ( rmdir /s /q %LOCAL_WS%\gazebo ) || goto :error + IF EXIST %LOCAL_WS%\gazebo-classic ( rmdir /s /q %LOCAL_WS%\gazebo-classic ) || goto :error echo # END SECTION ) @@ -86,7 +87,7 @@ echo # END SECTION echo # BEGIN SECTION: compile and install sdformat set SDFORMAT_DIR=%WORKSPACE%\sdformat if EXIST %SDFORMAT_DIR% ( rmdir /s /q %SDFORMAT_DIR% ) -git clone https://github.com/osrf/sdformat %SDFORMAT_DIR% -b sdf9 +git clone https://github.com/gazebosim/sdformat %SDFORMAT_DIR% -b sdf9 set VCS_DIRECTORY=sdformat set KEEP_WORKSPACE=TRUE set ENABLE_TESTS=FALSE @@ -100,9 +101,9 @@ xcopy %WORKSPACE_INSTALL_DIR%\boost_1_67_0\lib64-msvc-14.1 %WORKSPACE_INSTALL_DI echo # END SECTION echo # BEGIN SECTION: copy gazebo sources to workspace -:: Note that your jenkins job should put source in %WORKSPACE%/gazebo -xcopy %WORKSPACE%\gazebo %LOCAL_WS%\gazebo /s /i /e > xcopy.log -cd %LOCAL_WS%\gazebo +:: Note that your jenkins job should put source in %WORKSPACE%/gazebo-classic +xcopy %WORKSPACE%\gazebo-classic %LOCAL_WS%\gazebo-classic /s /i /e > xcopy.log +cd %LOCAL_WS%\gazebo-classic echo # END SECTION echo # BEGIN SECTION: configure gazebo diff --git a/jenkins-scripts/lib/gazebo9_10-base-windows.bat b/jenkins-scripts/lib/gazebo9_10-base-windows.bat index 7f1550c1d..63c9ba678 100644 --- a/jenkins-scripts/lib/gazebo9_10-base-windows.bat +++ b/jenkins-scripts/lib/gazebo9_10-base-windows.bat @@ -49,6 +49,7 @@ echo # END SECTION echo # BEGIN SECTION: reusing workspace :: Remove gazebo copy IF EXIST %LOCAL_WS%\gazebo ( rmdir /s /q %LOCAL_WS%\gazebo ) || goto :error + IF EXIST %LOCAL_WS%\gazebo-classic ( rmdir /s /q %LOCAL_WS%\gazebo-classic ) || goto :error echo # END SECTION ) @@ -66,7 +67,7 @@ echo # END SECTION echo # BEGIN SECTION: compile and install sdformat set SDFORMAT_DIR=%WORKSPACE%\sdformat if EXIST %SDFORMAT_DIR% ( rmdir /s /q %SDFORMAT_DIR% ) -git clone https://github.com/osrf/sdformat %SDFORMAT_DIR% -b sdf6 +git clone https://github.com/gazebosim/sdformat %SDFORMAT_DIR% -b sdf6 set VCS_DIRECTORY=sdformat set KEEP_WORKSPACE=TRUE set ENABLE_TESTS=FALSE @@ -80,9 +81,9 @@ xcopy %WORKSPACE_INSTALL_DIR%\boost_1_67_0\lib64-msvc-14.1 %WORKSPACE_INSTALL_DI echo # END SECTION echo # BEGIN SECTION: copy gazebo sources to workspace -:: Note that your jenkins job should put source in %WORKSPACE%/gazebo -xcopy %WORKSPACE%\gazebo %LOCAL_WS%\gazebo /s /i /e > xcopy.log -cd %LOCAL_WS%\gazebo +:: Note that your jenkins job should put source in %WORKSPACE%/gazebo-classic +xcopy %WORKSPACE%\gazebo-classic %LOCAL_WS%\gazebo-classic /s /i /e > xcopy.log +cd %LOCAL_WS%\gazebo-classic echo # END SECTION echo # BEGIN SECTION: configure gazebo From 09b35ca83a4ff08d9d8d7ee4b82b3aa4e4af4e48 Mon Sep 17 00:00:00 2001 From: "Addisu Z. Taddese" Date: Thu, 17 Nov 2022 17:13:42 -0600 Subject: [PATCH 263/540] Copy garden view to harmonic Signed-off-by: Addisu Z. Taddese --- jenkins-scripts/dsl/ignition_collection.dsl | 105 +++++++++++++++++++- 1 file changed, 104 insertions(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index 024465dfa..daba198b3 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -4,7 +4,7 @@ import javaposse.jobdsl.dsl.Job // GZ COLLECTIONS arch = 'amd64' -gz_nightly = 'garden' +gz_nightly = 'harmonic' gz_collections = [ [ name : 'citadel', @@ -15,6 +15,9 @@ gz_collections = [ ], [ name : 'garden', distros : [ 'focal' ], + ], + [ name : 'harmonic', + distros : [ 'focal' ], // These are the branches currently targeted at the upcoming collection // They're in topological order nightly_jobs: [ @@ -304,6 +307,106 @@ gz_collection_jobs = 'sdformat-install-sdformat12_pkg-focal-amd64', 'sdformat-sdf-13-win' ], + 'harmonic' : [ + 'gz_cmake3-install-pkg-focal-amd64', + 'gz_common5-install-pkg-focal-amd64', + 'gz_fuel-tools8-install-pkg-focal-amd64', + 'gz_harmonic-install-pkg-focal-amd64', + 'gz_gazebo7-install-pkg-focal-amd64', + 'gz_gui7-install-pkg-focal-amd64', + 'gz_launch6-install-pkg-focal-amd64', + 'gz_math7-install-pkg-focal-amd64', + 'gz_msgs9-install-pkg-focal-amd64', + 'gz_physics6-install-pkg-focal-amd64', + 'gz_plugin2-install-pkg-focal-amd64', + 'gz_rendering7-install-pkg-focal-amd64', + 'gz_sensors7-install-pkg-focal-amd64', + 'gz_tools2-install-pkg-focal-amd64', + 'gz_transport12-install-pkg-focal-amd64', + 'gz_utils2-install-pkg-focal-amd64', + 'ign_cmake-gz-3-win', + 'ign_common-gz-5-win', + 'ign_fuel-tools-gz-8-win', + 'ign_gazebo-ci-win', + 'ign_gui-gz-7-win', + 'ign_launch-gz-6-win', + 'ign_math-gz-7-win', + 'ign_msgs-gz-9-win', + 'ign_physics-gz-6-win', + 'ign_plugin-gz-2-win', + 'ign_rendering-gz-7-win', + 'ign_sensors-gz-7-win', + 'ign_tools-gz-2-win', + 'ign_transport-gz-12-win', + 'ign_utils-gz-2-win', + 'ignition_cmake-ci-gz-cmake3-focal-amd64', + 'ignition_cmake-ci-gz-cmake3-homebrew-amd64', + 'ignition_cmake2-install-pkg-focal-amd64', + 'ignition_cmake2-install_bottle-homebrew-amd64', + 'ignition_common-ci-gz-common5-focal-amd64', + 'ignition_common-ci-gz-common5-homebrew-amd64', + 'ignition_common4-install-pkg-focal-amd64', + 'ignition_common4-install_bottle-homebrew-amd64', + 'ignition_fuel-tools-ci-gz-fuel-tools8-focal-amd64', + 'ignition_fuel-tools-ci-gz-fuel-tools8-homebrew-amd64', + 'ignition_fuel-tools7-install-pkg-focal-amd64', + 'ignition_fuel-tools7-install_bottle-homebrew-amd64', + 'ignition_garden-ci-main-homebrew-amd64', + 'ignition_garden-install-pkg-focal-amd64', + 'ignition_garden-install_bottle-homebrew-amd64', + 'ignition_gazebo-ci-main-focal-amd64', + 'ignition_gazebo-ci-main-homebrew-amd64', + 'ignition_gazebo6-install-pkg-focal-amd64', + 'ignition_gazebo6-install_bottle-homebrew-amd64', + 'ignition_gui-ci-gz-gui7-focal-amd64', + 'ignition_gui-ci-gz-gui7-homebrew-amd64', + 'ignition_gui6-install-pkg-focal-amd64', + 'ignition_gui6-install_bottle-homebrew-amd64', + 'ignition_launch-ci-gz-launch6-focal-amd64', + 'ignition_launch-ci-gz-launch6-homebrew-amd64', + 'ignition_launch5-install-pkg-focal-amd64', + 'ignition_launch5-install_bottle-homebrew-amd64', + 'ignition_math-ci-gz-math7-focal-amd64', + 'ignition_math-ci-gz-math7-homebrew-amd64', + 'ignition_math6-install-pkg-focal-amd64', + 'ignition_math6-install_bottle-homebrew-amd64', + 'ignition_msgs-ci-gz-msgs9-focal-amd64', + 'ignition_msgs-ci-gz-msgs9-homebrew-amd64', + 'ignition_msgs8-install-pkg-focal-amd64', + 'ignition_msgs8-install_bottle-homebrew-amd64', + 'ignition_physics-ci-gz-physics6-focal-amd64', + 'ignition_physics-ci-gz-physics6-homebrew-amd64', + 'ignition_physics5-install-pkg-focal-amd64', + 'ignition_physics5-install_bottle-homebrew-amd64', + 'ignition_plugin-ci-gz-plugin2-focal-amd64', + 'ignition_plugin-ci-gz-plugin2-homebrew-amd64', + 'ignition_plugin-install-pkg-focal-amd64', + 'ignition_plugin1-install_bottle-homebrew-amd64', + 'ignition_rendering-ci-gz-rendering7-focal-amd64', + 'ignition_rendering-ci-gz-rendering7-homebrew-amd64', + 'ignition_rendering6-install-pkg-focal-amd64', + 'ignition_rendering6-install_bottle-homebrew-amd64', + 'ignition_sensors-ci-gz-sensors7-focal-amd64', + 'ignition_sensors-ci-gz-sensors7-homebrew-amd64', + 'ignition_sensors6-install-pkg-focal-amd64', + 'ignition_sensors6-install_bottle-homebrew-amd64', + 'ignition_tools-ci-gz-tools2-focal-amd64', + 'ignition_tools-ci-gz-tools2-homebrew-amd64', + 'ignition_tools-install-pkg-focal-amd64', + 'ignition_tools1-install_bottle-homebrew-amd64', + 'ignition_transport-ci-gz-transport12-focal-amd64', + 'ignition_transport-ci-gz-transport12-homebrew-amd64', + 'ignition_transport11-install-pkg-focal-amd64', + 'ignition_transport11-install_bottle-homebrew-amd64', + 'ignition_utils-ci-gz-utils2-focal-amd64', + 'ignition_utils-ci-gz-utils2-homebrew-amd64', + 'ignition_utils-install-pkg-focal-amd64', + 'ignition_utils1-install_bottle-homebrew-amd64', + 'sdformat-ci-sdformat13-focal-amd64', + 'sdformat-ci-sdformat13-homebrew-amd64', + 'sdformat-install-sdformat12_pkg-focal-amd64', + 'sdformat-sdf-13-win' + ], ] def DISABLE_TESTS = false From df176c4314b32a97e37a7435094437bc9d2f5d3e Mon Sep 17 00:00:00 2001 From: "Addisu Z. Taddese" Date: Thu, 17 Nov 2022 17:22:38 -0600 Subject: [PATCH 264/540] Update view to use main branches for bumped libraries Signed-off-by: Addisu Z. Taddese --- jenkins-scripts/dsl/ignition_collection.dsl | 85 +++++---------------- 1 file changed, 18 insertions(+), 67 deletions(-) diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index daba198b3..2fffcca22 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -28,15 +28,15 @@ gz_collections = [ 'plugin' : [ debbuild: 'gz-plugin2' , branch: 'gz-plugin2' ], 'common' : [ debbuild: 'gz-common5' , branch: 'gz-common5' ], 'msgs' : [ debbuild: 'gz-msgs9' , branch: 'gz-msgs9' ], - 'rendering' : [ debbuild: 'gz-rendering7' , branch: 'gz-rendering7' ], + 'rendering' : [ debbuild: 'gz-rendering8' , branch: 'main' ], 'sdformat' : [ debbuild: 'sdformat13' , branch: 'sdf13' ], 'fuel-tools': [ debbuild: 'gz-fuel-tools8' , branch: 'gz-fuel-tools8' ], 'transport' : [ debbuild: 'gz-transport12' , branch: 'gz-transport12' ], - 'gui' : [ debbuild: 'gz-gui7' , branch: 'gz-gui7' ], - 'sensors' : [ debbuild: 'gz-sensors7' , branch: 'gz-sensors7' ], + 'gui' : [ debbuild: 'gz-gui8' , branch: 'main' ], + 'sensors' : [ debbuild: 'gz-sensors8' , branch: 'main' ], 'physics' : [ debbuild: 'gz-physics6' , branch: 'gz-physics6' ], - 'gazebo' : [ debbuild: 'gz-sim7' , branch: 'gz-sim7' ], - 'launch' : [ debbuild: 'gz-launch6' , branch: 'gz-launch6' ], + 'gazebo' : [ debbuild: 'gz-sim8' , branch: 'main' ], + 'launch' : [ debbuild: 'gz-launch7' , branch: 'main' ], 'garden' : [ debbuild: 'gz-garden' , branch: 'main' ], ], ], @@ -308,103 +308,54 @@ gz_collection_jobs = 'sdformat-sdf-13-win' ], 'harmonic' : [ - 'gz_cmake3-install-pkg-focal-amd64', - 'gz_common5-install-pkg-focal-amd64', - 'gz_fuel-tools8-install-pkg-focal-amd64', - 'gz_harmonic-install-pkg-focal-amd64', - 'gz_gazebo7-install-pkg-focal-amd64', - 'gz_gui7-install-pkg-focal-amd64', - 'gz_launch6-install-pkg-focal-amd64', - 'gz_math7-install-pkg-focal-amd64', - 'gz_msgs9-install-pkg-focal-amd64', - 'gz_physics6-install-pkg-focal-amd64', - 'gz_plugin2-install-pkg-focal-amd64', - 'gz_rendering7-install-pkg-focal-amd64', - 'gz_sensors7-install-pkg-focal-amd64', - 'gz_tools2-install-pkg-focal-amd64', - 'gz_transport12-install-pkg-focal-amd64', - 'gz_utils2-install-pkg-focal-amd64', 'ign_cmake-gz-3-win', 'ign_common-gz-5-win', 'ign_fuel-tools-gz-8-win', 'ign_gazebo-ci-win', - 'ign_gui-gz-7-win', - 'ign_launch-gz-6-win', + 'ign_gui-ci-win', + 'ign_launch-ci-win', 'ign_math-gz-7-win', 'ign_msgs-gz-9-win', 'ign_physics-gz-6-win', 'ign_plugin-gz-2-win', - 'ign_rendering-gz-7-win', - 'ign_sensors-gz-7-win', + 'ign_rendering-ci-win', + 'ign_sensors-ci-win', 'ign_tools-gz-2-win', 'ign_transport-gz-12-win', 'ign_utils-gz-2-win', 'ignition_cmake-ci-gz-cmake3-focal-amd64', 'ignition_cmake-ci-gz-cmake3-homebrew-amd64', - 'ignition_cmake2-install-pkg-focal-amd64', - 'ignition_cmake2-install_bottle-homebrew-amd64', 'ignition_common-ci-gz-common5-focal-amd64', 'ignition_common-ci-gz-common5-homebrew-amd64', - 'ignition_common4-install-pkg-focal-amd64', - 'ignition_common4-install_bottle-homebrew-amd64', 'ignition_fuel-tools-ci-gz-fuel-tools8-focal-amd64', 'ignition_fuel-tools-ci-gz-fuel-tools8-homebrew-amd64', - 'ignition_fuel-tools7-install-pkg-focal-amd64', - 'ignition_fuel-tools7-install_bottle-homebrew-amd64', - 'ignition_garden-ci-main-homebrew-amd64', - 'ignition_garden-install-pkg-focal-amd64', - 'ignition_garden-install_bottle-homebrew-amd64', + 'ignition_harmonic-ci-main-homebrew-amd64', 'ignition_gazebo-ci-main-focal-amd64', 'ignition_gazebo-ci-main-homebrew-amd64', - 'ignition_gazebo6-install-pkg-focal-amd64', - 'ignition_gazebo6-install_bottle-homebrew-amd64', - 'ignition_gui-ci-gz-gui7-focal-amd64', - 'ignition_gui-ci-gz-gui7-homebrew-amd64', - 'ignition_gui6-install-pkg-focal-amd64', - 'ignition_gui6-install_bottle-homebrew-amd64', - 'ignition_launch-ci-gz-launch6-focal-amd64', - 'ignition_launch-ci-gz-launch6-homebrew-amd64', - 'ignition_launch5-install-pkg-focal-amd64', - 'ignition_launch5-install_bottle-homebrew-amd64', + 'ignition_gui-ci-main-focal-amd64', + 'ignition_gui-ci-main-homebrew-amd64', + 'ignition_launch-ci-main-focal-amd64', + 'ignition_launch-ci-main-homebrew-amd64', 'ignition_math-ci-gz-math7-focal-amd64', 'ignition_math-ci-gz-math7-homebrew-amd64', - 'ignition_math6-install-pkg-focal-amd64', - 'ignition_math6-install_bottle-homebrew-amd64', 'ignition_msgs-ci-gz-msgs9-focal-amd64', 'ignition_msgs-ci-gz-msgs9-homebrew-amd64', - 'ignition_msgs8-install-pkg-focal-amd64', - 'ignition_msgs8-install_bottle-homebrew-amd64', 'ignition_physics-ci-gz-physics6-focal-amd64', 'ignition_physics-ci-gz-physics6-homebrew-amd64', - 'ignition_physics5-install-pkg-focal-amd64', - 'ignition_physics5-install_bottle-homebrew-amd64', 'ignition_plugin-ci-gz-plugin2-focal-amd64', 'ignition_plugin-ci-gz-plugin2-homebrew-amd64', - 'ignition_plugin-install-pkg-focal-amd64', - 'ignition_plugin1-install_bottle-homebrew-amd64', - 'ignition_rendering-ci-gz-rendering7-focal-amd64', - 'ignition_rendering-ci-gz-rendering7-homebrew-amd64', - 'ignition_rendering6-install-pkg-focal-amd64', - 'ignition_rendering6-install_bottle-homebrew-amd64', - 'ignition_sensors-ci-gz-sensors7-focal-amd64', - 'ignition_sensors-ci-gz-sensors7-homebrew-amd64', - 'ignition_sensors6-install-pkg-focal-amd64', - 'ignition_sensors6-install_bottle-homebrew-amd64', + 'ignition_rendering-ci-main-focal-amd64', + 'ignition_rendering-ci-main-homebrew-amd64', + 'ignition_sensors-ci-main-focal-amd64', + 'ignition_sensors-ci-main-homebrew-amd64', 'ignition_tools-ci-gz-tools2-focal-amd64', 'ignition_tools-ci-gz-tools2-homebrew-amd64', - 'ignition_tools-install-pkg-focal-amd64', - 'ignition_tools1-install_bottle-homebrew-amd64', 'ignition_transport-ci-gz-transport12-focal-amd64', 'ignition_transport-ci-gz-transport12-homebrew-amd64', - 'ignition_transport11-install-pkg-focal-amd64', - 'ignition_transport11-install_bottle-homebrew-amd64', 'ignition_utils-ci-gz-utils2-focal-amd64', 'ignition_utils-ci-gz-utils2-homebrew-amd64', - 'ignition_utils-install-pkg-focal-amd64', - 'ignition_utils1-install_bottle-homebrew-amd64', 'sdformat-ci-sdformat13-focal-amd64', 'sdformat-ci-sdformat13-homebrew-amd64', - 'sdformat-install-sdformat12_pkg-focal-amd64', 'sdformat-sdf-13-win' ], ] From a823a83fb864bb2f5040af465278a6167c0029e5 Mon Sep 17 00:00:00 2001 From: "Addisu Z. Taddese" Date: Thu, 17 Nov 2022 17:22:56 -0600 Subject: [PATCH 265/540] Setup nightlies for bumped branches Signed-off-by: Addisu Z. Taddese --- jenkins-scripts/dsl/ignition.dsl | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index b3e78518a..caee74888 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -83,7 +83,13 @@ gz_prerelease_branches = [] // DESC: versioned names to generate debbuild jobs for special cases that // don't appear in gz_branches (like nightly builders or 0-debbuild // jobs for the special cases of foo0 packages) -gz_extra_debbuild = [ 'utils1' ] // see comment https://github.com/gazebo-tooling/release-tools/pull/431#issuecomment-815099918 +gz_extra_debbuild = [ 'gui8', + 'launch8', + 'rendering8', + 'sensors8', + 'sim8', + 'utils1' // see comment https://github.com/gazebo-tooling/release-tools/pull/431#issuecomment-815099918 + ] // DESC: exclude ignition from generate any install testing job gz_no_pkg_yet = [ ] // DESC: major versions that has a package in the prerelease repo. Should From ddede4bcc609a4c28b5c2eba9be8337dd6bdfe30 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 18 Nov 2022 19:27:55 +0100 Subject: [PATCH 266/540] Support for none in UPLOAD_TO_REPO in DSL generic OSRFLinuxBuildPkg Signed-off-by: Jose Luis Rivero --- .../dsl/_configs_/OSRFLinuxBuildPkg.groovy | 40 +++++++++++-------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/jenkins-scripts/dsl/_configs_/OSRFLinuxBuildPkg.groovy b/jenkins-scripts/dsl/_configs_/OSRFLinuxBuildPkg.groovy index bcb9477d6..97b86114e 100644 --- a/jenkins-scripts/dsl/_configs_/OSRFLinuxBuildPkg.groovy +++ b/jenkins-scripts/dsl/_configs_/OSRFLinuxBuildPkg.groovy @@ -65,7 +65,7 @@ class OSRFLinuxBuildPkg "If not empty, package name to be used instead of PACKAGE") stringParam("UPLOAD_TO_REPO", default_params.find{ it.key == "UPLOAD_TO_REPO"}?.value, - "OSRF repo name to upload the package to") + "OSRF repo name to upload the package to: stable | prerelease | nightly | none (for testing proposes)") stringParam("OSRF_REPOS_TO_USE", default_params.find{ it.key == "OSRF_REPOS_TO_USE"}?.value, "OSRF repos name to use when building the package") @@ -99,23 +99,29 @@ class OSRFLinuxBuildPkg failBuildOnError(false) } - downstreamParameterized { - trigger('repository_uploader_packages') { - condition('UNSTABLE_OR_BETTER') - parameters { - currentBuild() - predefinedProp("PROJECT_NAME_TO_COPY_ARTIFACTS", "\${JOB_NAME}") - // Workaround to avoid problems on repository uploader. Real - // issue: https://issues.jenkins-ci.org/browse/JENKINS-45005 - predefinedProp("JENKINS_NODE_TAG", "master") - } - } + conditionalAction { + condition { + not { + expression('none|None|^$','${ENV,var="UPLOAD_TO_REPO"}') + } + } + downstreamParameterized { + trigger('repository_uploader_packages') { + condition('UNSTABLE_OR_BETTER') + parameters { + currentBuild() + predefinedProp("PROJECT_NAME_TO_COPY_ARTIFACTS", "\${JOB_NAME}") + // Workaround to avoid problems on repository uploader. Real + // issue: https://issues.jenkins-ci.org/browse/JENKINS-45005 + predefinedProp("JENKINS_NODE_TAG", "master") + } + } + } } - } - - configure { project -> - project / 'properties' / 'hudson.plugins.copyartifact.CopyArtifactPermissionProperty' / 'projectNameList' { - 'string' 'repository_uploader_*' + configure { project -> + project / 'properties' / 'hudson.plugins.copyartifact.CopyArtifactPermissionProperty' / 'projectNameList' { + 'string' 'repository_uploader_*' + } } } } // end of job From bad3c51f177f2e7a81efbc00c3cda2b587b3877d Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 18 Nov 2022 19:45:39 +0100 Subject: [PATCH 267/540] Add none to UPLOAD_TO_REPO in repostory loader Signed-off-by: Jose Luis Rivero --- jenkins-scripts/lib/repository_uploader.bash | 58 +++++++++++--------- 1 file changed, 31 insertions(+), 27 deletions(-) diff --git a/jenkins-scripts/lib/repository_uploader.bash b/jenkins-scripts/lib/repository_uploader.bash index e9b942230..d44d985d7 100644 --- a/jenkins-scripts/lib/repository_uploader.bash +++ b/jenkins-scripts/lib/repository_uploader.bash @@ -109,7 +109,7 @@ fi # Check destination repository if [[ -z ${UPLOAD_TO_REPO} ]]; then - echo "No UPLOAD_TO_REPO value was send. Which repository to use? (stable | prerelease | nightly)" + echo "No UPLOAD_TO_REPO value was send. Which repository to use? (stable | prerelease | nightly | none)" echo "" echo "Please check that the jenkins -debbuild job that called this uploader is defining the parameter" echo "UPLOAD_TO_REPO in the job configuration. If it is not, just define it as a string parameter" @@ -117,32 +117,36 @@ if [[ -z ${UPLOAD_TO_REPO} ]]; then fi case ${UPLOAD_TO_REPO} in - "stable") - # Security checks not to upload nightly or prereleases - # No packages with ~git or ~pre - if [[ -n $(ls ${pkgs_path}/*~git*.*) && -n $(ls ${pkgs_path}/*~pre*.*) ]]; then - echo "There are nightly packages in the upload directory. Not uploading to stable repo" - exit 1 - fi - # No source packages with ~git in version - if [[ -n $(cat ${pkgs_path}/*.dsc | grep ^Version: | grep '~git\|~pre') ]]; then - echo "There is a sorce package with nightly or pre in version. Not uploading to stable repo" - exit 1 - fi - ;; - "nightly") - # No uploads for nightly packages - ENABLE_S3_UPLOAD=false - ;; - "only_s3_upload") - # This should be fine, no repo, only s3 upload - ENABLE_S3_UPLOAD=true - ;; - *) - # Here we could find project repositories uploads or error values. - # Error values for UPLOAD_TO_REPO will be get in the next directory check - # some lines below so we do nothing. - ;; + "stable") + # Security checks not to upload nightly or prereleases + # No packages with ~git or ~pre + if [[ -n $(ls ${pkgs_path}/*~git*.*) && -n $(ls ${pkgs_path}/*~pre*.*) ]]; then + echo "There are nightly packages in the upload directory. Not uploading to stable repo" + exit 1 + fi + # No source packages with ~git in version + if [[ -n $(cat ${pkgs_path}/*.dsc | grep ^Version: | grep '~git\|~pre') ]]; then + echo "There is a sorce package with nightly or pre in version. Not uploading to stable repo" + exit 1 + fi + ;; + "nightly") + # No uploads for nightly packages test runs + ENABLE_S3_UPLOAD=false + ;; + "none") + echo "UPLOAD_TO_REPO was set to none. No upload is done." + exit 0 + ;; + "only_s3_upload") + # This should be fine, no repo, only s3 upload + ENABLE_S3_UPLOAD=true + ;; + *) + # Here we could find project repositories uploads or error values. + # Error values for UPLOAD_TO_REPO will be get in the next directory check + # some lines below so we do nothing. + ;; esac # .zip | (mostly) windows packages From eb88107e2c159b8027d57942185e0d7d6c38e868 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 18 Nov 2022 20:24:38 +0100 Subject: [PATCH 268/540] Fix DSL code Signed-off-by: Jose Luis Rivero --- .../dsl/_configs_/OSRFLinuxBuildPkg.groovy | 33 +++++++++++-------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/jenkins-scripts/dsl/_configs_/OSRFLinuxBuildPkg.groovy b/jenkins-scripts/dsl/_configs_/OSRFLinuxBuildPkg.groovy index 97b86114e..3d3caf4db 100644 --- a/jenkins-scripts/dsl/_configs_/OSRFLinuxBuildPkg.groovy +++ b/jenkins-scripts/dsl/_configs_/OSRFLinuxBuildPkg.groovy @@ -99,21 +99,26 @@ class OSRFLinuxBuildPkg failBuildOnError(false) } - conditionalAction { - condition { - not { - expression('none|None|^$','${ENV,var="UPLOAD_TO_REPO"}') + flexiblePublish + { + conditionalAction { + condition { + not { + expression('none|None|^$','${ENV,var="UPLOAD_TO_REPO"}') + } } - } - downstreamParameterized { - trigger('repository_uploader_packages') { - condition('UNSTABLE_OR_BETTER') - parameters { - currentBuild() - predefinedProp("PROJECT_NAME_TO_COPY_ARTIFACTS", "\${JOB_NAME}") - // Workaround to avoid problems on repository uploader. Real - // issue: https://issues.jenkins-ci.org/browse/JENKINS-45005 - predefinedProp("JENKINS_NODE_TAG", "master") + publishers { + downstreamParameterized { + trigger('repository_uploader_packages') { + condition('UNSTABLE_OR_BETTER') + parameters { + currentBuild() + predefinedProp("PROJECT_NAME_TO_COPY_ARTIFACTS", "\${JOB_NAME}") + // Workaround to avoid problems on repository uploader. Real + // issue: https://issues.jenkins-ci.org/browse/JENKINS-45005 + predefinedProp("JENKINS_NODE_TAG", "master") + } + } } } } From 3e97c409a4ad12e8771c3a9c2c89a70291e08258 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Mon, 21 Nov 2022 17:07:04 +0100 Subject: [PATCH 269/540] Do not add restricted repo twice Signed-off-by: Jose Luis Rivero --- .../docker/lib/docker_generate_dockerfile.bash | 8 -------- 1 file changed, 8 deletions(-) diff --git a/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash b/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash index 74c349cc2..02434b758 100644 --- a/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash +++ b/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash @@ -177,14 +177,6 @@ DELIM_DOCKER_ARCH fi fi -# i386 image only have main by default -if [[ ${LINUX_DISTRO} == 'ubuntu' && ${ARCH} == 'i386' ]]; then -cat >> Dockerfile << DELIM_DOCKER_I386_APT -RUN echo "deb ${SOURCE_LIST_URL} ${DISTRO} restricted universe" \\ - >> /etc/apt/sources.list -DELIM_DOCKER_I386_APT -fi - # Workaround for: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=932019 if [[ ${DISTRO} == 'buster' ]]; then cat >> Dockerfile << DELIM_BUSTER_DWZ From d696b9c1154e3632c66bebb803a2aff9141b43f6 Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Tue, 29 Nov 2022 18:54:38 -0800 Subject: [PATCH 270/540] brew_release.dsl: stop catalina bottle builds homebrew-core has stopped building catalina bottles, so we will do the same. Signed-off-by: Steve Peters --- jenkins-scripts/dsl/brew_release.dsl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/brew_release.dsl b/jenkins-scripts/dsl/brew_release.dsl index 56bd7d166..90bd02f8b 100644 --- a/jenkins-scripts/dsl/brew_release.dsl +++ b/jenkins-scripts/dsl/brew_release.dsl @@ -4,7 +4,7 @@ import javaposse.jobdsl.dsl.Job Globals.default_emails = "jrivero@osrfoundation.org, scpeters@osrfoundation.org" // first distro in list is used as touchstone -brew_supported_distros = [ "catalina", "bigsur", "monterey" ] +brew_supported_distros = [ "bigsur", "monterey" ] bottle_hash_updater_job_name = 'generic-release-homebrew_pr_bottle_hash_updater' bottle_builder_job_name = 'generic-release-homebrew_triggered_bottle_builder' directory_for_bottles = 'pkgs' From e4dea4043632076558013a0a1423737e7f5e1f8c Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 8 Dec 2022 18:17:08 +0100 Subject: [PATCH 271/540] Use gz insetead of ign for extra_debbuilders Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/ignition.dsl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index ccf481b10..9764f2372 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -261,7 +261,11 @@ ArrayList all_debbuilders() } // add all extra debbuilders gz_extra_debbuild.each { gz_name -> - branches.add("ign-${gz_name}") + // utils1 is still using the ign preffix + if (gz_name == 'utils1') + branches.add("ign-${gz_name}") + else + branches.add("gz-${gz_name}") } return branches From 6229c4ada949a0633d3fd3a816b9e2ed495e53c4 Mon Sep 17 00:00:00 2001 From: Ian Chen Date: Thu, 8 Dec 2022 11:31:58 -0800 Subject: [PATCH 272/540] launch8 -> launch7 Signed-off-by: Ian Chen --- jenkins-scripts/dsl/ignition.dsl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index 9764f2372..3427d56cb 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -84,7 +84,7 @@ gz_prerelease_branches = [] // don't appear in gz_branches (like nightly builders or 0-debbuild // jobs for the special cases of foo0 packages) gz_extra_debbuild = [ 'gui8', - 'launch8', + 'launch7', 'rendering8', 'sensors8', 'sim8', From 8a5b86c6675d6ba778e2411717c7eca8ba4acc52 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 22 Dec 2022 21:03:28 +0100 Subject: [PATCH 273/540] Rootless in packages.o.o 1/2: do not install packages Signed-off-by: Jose Luis Rivero --- jenkins-scripts/lib/repository_uploader.bash | 7 ------- 1 file changed, 7 deletions(-) diff --git a/jenkins-scripts/lib/repository_uploader.bash b/jenkins-scripts/lib/repository_uploader.bash index d44d985d7..fa68d80aa 100644 --- a/jenkins-scripts/lib/repository_uploader.bash +++ b/jenkins-scripts/lib/repository_uploader.bash @@ -87,13 +87,6 @@ upload_dsc_package() echo "MARK_BUILD_UNSTABLE" } -NEEDED_HOST_PACKAGES="reprepro openssh-client jq gridsite-clients" -QUERY_HOST_PACKAGES=$(dpkg-query -Wf'${db:Status-abbrev}' ${NEEDED_HOST_PACKAGES} 2>&1) || true -if [[ -n ${QUERY_HOST_PACKAGES} ]]; then - sudo apt-get update - sudo apt-get install -y ${NEEDED_HOST_PACKAGES} -fi - # By default, enable s3 upload of packages ENABLE_S3_UPLOAD=true From 0434eaace57c9cbdc59fa48cff1c3f24c072e08a Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 5 Jan 2023 19:52:14 +0100 Subject: [PATCH 274/540] Remove sudo from reprepro calls Signed-off-by: Jose Luis Rivero --- jenkins-scripts/lib/repository_uploader.bash | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/jenkins-scripts/lib/repository_uploader.bash b/jenkins-scripts/lib/repository_uploader.bash index fa68d80aa..e9e451531 100644 --- a/jenkins-scripts/lib/repository_uploader.bash +++ b/jenkins-scripts/lib/repository_uploader.bash @@ -47,7 +47,7 @@ dsc_package_exists_and_equal_or_greater() { local pkg=${1} new_version=${2} distro=${3} - current_dsc_info=$(sudo GNUPGHOME=/var/lib/jenkins/.gnupg/ reprepro -T dsc list "${distro}" "${pkg}" | tail -n 1) + current_dsc_info=$(GNUPGHOME=$HOME/.gnupg/ reprepro -T dsc list "${distro}" "${pkg}" | tail -n 1) if [[ -z ${current_dsc_info} ]]; then return 1 # do not exits, false @@ -68,7 +68,7 @@ upload_package() [[ -z ${pkg} ]] && echo "Bad parameter pkg" && exit 1 [[ -z ${pkg_name} ]] && echo "Bad parameter pkg_name" && exit 1 - sudo GNUPGHOME=$HOME/.gnupg reprepro --nothingiserror includedeb $DISTRO ${pkg} + GNUPGHOME=$HOME/.gnupg reprepro --nothingiserror includedeb $DISTRO ${pkg} # The path will end up being: s3://osrf-distributions/$pkg_root_name/releases/ S3_upload ${pkg} ${pkg_name}/releases/ @@ -82,8 +82,8 @@ upload_dsc_package() # .dsc sometimes does not include priority or section, # try to upload and if failed, specify the values # see: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=768046 - sudo GNUPGHOME=$HOME/.gnupg reprepro --nothingiserror includedsc $DISTRO ${pkg} || \ - sudo GNUPGHOME=$HOME/.gnupg reprepro --nothingiserror --section science --priority extra includedsc $DISTRO ${pkg} || \ + GNUPGHOME=$HOME/.gnupg reprepro --nothingiserror includedsc $DISTRO ${pkg} || \ + GNUPGHOME=$HOME/.gnupg reprepro --nothingiserror --section science --priority extra includedsc $DISTRO ${pkg} || \ echo "MARK_BUILD_UNSTABLE" } @@ -224,7 +224,7 @@ for pkg in `ls $pkgs_path/*.deb`; do all.deb) # Check if the package already exists. i386 and amd64 generates the same binaries. # all should be multiarch, so supposed to work on every platform - existing_version=$(sudo GNUPGHOME=/var/lib/jenkins/.gnupg/ reprepro ls ${pkg_name} | grep ${DISTRO} | awk '{ print $3 }') + existing_version=$(GNUPGHOME=$HOME/.gnupg/ reprepro ls ${pkg_name} | grep ${DISTRO} | awk '{ print $3 }') if $(dpkg --compare-versions ${pkg_version} le ${existing_version}); then echo "${pkg_relative} for ${DISTRO} is already in the repo with same version or greater" echo "SKIP UPLOAD" From ed8e3e07f2d2475eca56242d62ee017d6d24fb69 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 13 Jan 2023 16:15:03 +0100 Subject: [PATCH 275/540] Galactic EOL Signed-off-by: Jose Luis Rivero --- bloom/release-bloom.py | 1 - jenkins-scripts/dsl/_configs_/Globals.groovy | 4 +--- jenkins-scripts/lib/dependencies_archive.sh | 3 --- 3 files changed, 1 insertion(+), 7 deletions(-) diff --git a/bloom/release-bloom.py b/bloom/release-bloom.py index f5afce442..d93c41f2a 100755 --- a/bloom/release-bloom.py +++ b/bloom/release-bloom.py @@ -15,7 +15,6 @@ UBUNTU_DISTROS_IN_ROS = {'melodic': ['bionic'], 'noetic': ['focal']} UBUNTU_DISTROS_IN_ROS2 = {'foxy': ['focal'], - 'galactic': ['focal'], 'rolling': ['focal']} DRY_RUN = False diff --git a/jenkins-scripts/dsl/_configs_/Globals.groovy b/jenkins-scripts/dsl/_configs_/Globals.groovy index 89c47d42e..6ef9859c0 100644 --- a/jenkins-scripts/dsl/_configs_/Globals.groovy +++ b/jenkins-scripts/dsl/_configs_/Globals.groovy @@ -22,14 +22,12 @@ class Globals static ros_ci = [ 'melodic' : ['bionic'] , 'noetic' : ['focal'] , 'foxy' : ['focal'] , - 'galactic' : ['focal'] , 'rolling' : ['jammy']] // This should be in sync with archive_library static gz_version_by_rosdistro = [ 'melodic' : ['9'] , 'noetic' : ['11'] , 'foxy' : ['11'] , - 'galactic' : ['11'] , 'rolling' : ['11']] static String ign2gz(String str) { @@ -115,7 +113,7 @@ class Globals static ArrayList get_ros2_suported_distros() { - return [ 'foxy', 'galactic', 'rolling' ] + return [ 'foxy', 'rolling' ] } static String get_ros2_development_distro() { diff --git a/jenkins-scripts/lib/dependencies_archive.sh b/jenkins-scripts/lib/dependencies_archive.sh index 2f8ed4c02..d52c69bba 100644 --- a/jenkins-scripts/lib/dependencies_archive.sh +++ b/jenkins-scripts/lib/dependencies_archive.sh @@ -220,9 +220,6 @@ else foxy) GAZEBO_VERSION_FOR_ROS="11" ;; - galactic) - GAZEBO_VERSION_FOR_ROS="11" - ;; rolling) GAZEBO_VERSION_FOR_ROS="11" esac From a8ea9f09effc1d18a76eba1c80627826a2cc23d9 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Mon, 23 Jan 2023 20:31:54 +0100 Subject: [PATCH 276/540] Run autopktest only on amd64 Signed-off-by: Jose Luis Rivero --- jenkins-scripts/docker/lib/debbuild-base.bash | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/jenkins-scripts/docker/lib/debbuild-base.bash b/jenkins-scripts/docker/lib/debbuild-base.bash index 8fab580cd..9ba9cc189 100644 --- a/jenkins-scripts/docker/lib/debbuild-base.bash +++ b/jenkins-scripts/docker/lib/debbuild-base.bash @@ -268,7 +268,11 @@ test \$FOUND_PKG -eq 1 || exit 1 echo '# END SECTION' cat /etc/apt/sources.list -${DEBBUILD_AUTOPKGTEST} +# Run only autopkgtest on amd64 +# see https://github.com/gazebosim/gz-common/issues/484 +if [[ ${ARCH} == 'amd64' ]]; then + ${DEBBUILD_AUTOPKGTEST} +fi DELIM OSRF_REPOS_TO_USE=${OSRF_REPOS_TO_USE:=stable} From dcfac323fc48051a2e7e558cd9552dde2789942f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Ag=C3=BCero?= Date: Wed, 25 Jan 2023 17:00:26 +0100 Subject: [PATCH 277/540] Bump nightlies. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Carlos Agüero --- jenkins-scripts/dsl/ignition_collection.dsl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index 2fffcca22..bebb6be42 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -27,11 +27,11 @@ gz_collections = [ 'math' : [ debbuild: 'gz-math7' , branch: 'gz-math7' ], 'plugin' : [ debbuild: 'gz-plugin2' , branch: 'gz-plugin2' ], 'common' : [ debbuild: 'gz-common5' , branch: 'gz-common5' ], - 'msgs' : [ debbuild: 'gz-msgs9' , branch: 'gz-msgs9' ], + 'msgs' : [ debbuild: 'gz-msgs9' , branch: 'gz-msgs10' ], 'rendering' : [ debbuild: 'gz-rendering8' , branch: 'main' ], 'sdformat' : [ debbuild: 'sdformat13' , branch: 'sdf13' ], 'fuel-tools': [ debbuild: 'gz-fuel-tools8' , branch: 'gz-fuel-tools8' ], - 'transport' : [ debbuild: 'gz-transport12' , branch: 'gz-transport12' ], + 'transport' : [ debbuild: 'gz-transport12' , branch: 'gz-transport13' ], 'gui' : [ debbuild: 'gz-gui8' , branch: 'main' ], 'sensors' : [ debbuild: 'gz-sensors8' , branch: 'main' ], 'physics' : [ debbuild: 'gz-physics6' , branch: 'gz-physics6' ], From 4f8ba6fb4fb33a6c899573250f8ed7eafb2386eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Ag=C3=BCero?= Date: Wed, 25 Jan 2023 17:03:46 +0100 Subject: [PATCH 278/540] Bump nightlies. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Carlos Agüero --- jenkins-scripts/dsl/ignition_collection.dsl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index bebb6be42..7063a4c68 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -27,11 +27,11 @@ gz_collections = [ 'math' : [ debbuild: 'gz-math7' , branch: 'gz-math7' ], 'plugin' : [ debbuild: 'gz-plugin2' , branch: 'gz-plugin2' ], 'common' : [ debbuild: 'gz-common5' , branch: 'gz-common5' ], - 'msgs' : [ debbuild: 'gz-msgs9' , branch: 'gz-msgs10' ], + 'msgs' : [ debbuild: 'gz-msgs9' , branch: 'main' ], 'rendering' : [ debbuild: 'gz-rendering8' , branch: 'main' ], 'sdformat' : [ debbuild: 'sdformat13' , branch: 'sdf13' ], 'fuel-tools': [ debbuild: 'gz-fuel-tools8' , branch: 'gz-fuel-tools8' ], - 'transport' : [ debbuild: 'gz-transport12' , branch: 'gz-transport13' ], + 'transport' : [ debbuild: 'gz-transport12' , branch: 'main' ], 'gui' : [ debbuild: 'gz-gui8' , branch: 'main' ], 'sensors' : [ debbuild: 'gz-sensors8' , branch: 'main' ], 'physics' : [ debbuild: 'gz-physics6' , branch: 'gz-physics6' ], From 52029931b12a562c5a320df39cb7f9b8e561faa1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Ag=C3=BCero?= Date: Wed, 25 Jan 2023 17:47:05 +0100 Subject: [PATCH 279/540] Update jenkins-scripts/dsl/ignition_collection.dsl Co-authored-by: Jose Luis Rivero --- jenkins-scripts/dsl/ignition_collection.dsl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index 7063a4c68..9efb50853 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -27,7 +27,7 @@ gz_collections = [ 'math' : [ debbuild: 'gz-math7' , branch: 'gz-math7' ], 'plugin' : [ debbuild: 'gz-plugin2' , branch: 'gz-plugin2' ], 'common' : [ debbuild: 'gz-common5' , branch: 'gz-common5' ], - 'msgs' : [ debbuild: 'gz-msgs9' , branch: 'main' ], + 'msgs' : [ debbuild: 'gz-msgs10' , branch: 'main' ], 'rendering' : [ debbuild: 'gz-rendering8' , branch: 'main' ], 'sdformat' : [ debbuild: 'sdformat13' , branch: 'sdf13' ], 'fuel-tools': [ debbuild: 'gz-fuel-tools8' , branch: 'gz-fuel-tools8' ], From 5538f579fa469ffd66bd7524c36929e8a59b73da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Ag=C3=BCero?= Date: Wed, 25 Jan 2023 17:47:15 +0100 Subject: [PATCH 280/540] Update jenkins-scripts/dsl/ignition_collection.dsl Co-authored-by: Jose Luis Rivero --- jenkins-scripts/dsl/ignition_collection.dsl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index 9efb50853..779179e5a 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -31,7 +31,7 @@ gz_collections = [ 'rendering' : [ debbuild: 'gz-rendering8' , branch: 'main' ], 'sdformat' : [ debbuild: 'sdformat13' , branch: 'sdf13' ], 'fuel-tools': [ debbuild: 'gz-fuel-tools8' , branch: 'gz-fuel-tools8' ], - 'transport' : [ debbuild: 'gz-transport12' , branch: 'main' ], + 'transport' : [ debbuild: 'gz-transport13' , branch: 'main' ], 'gui' : [ debbuild: 'gz-gui8' , branch: 'main' ], 'sensors' : [ debbuild: 'gz-sensors8' , branch: 'main' ], 'physics' : [ debbuild: 'gz-physics6' , branch: 'gz-physics6' ], From e9c6c1d4306a69ce09c7287090ef2bb745ce2c65 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 25 Jan 2023 19:11:30 +0100 Subject: [PATCH 281/540] Add msgs10 and transport12 nightly jobs Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/ignition.dsl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index 3427d56cb..26998dfad 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -85,11 +85,13 @@ gz_prerelease_branches = [] // jobs for the special cases of foo0 packages) gz_extra_debbuild = [ 'gui8', 'launch7', + 'msgs10', 'rendering8', 'sensors8', 'sim8', + 'transport12' 'utils1' // see comment https://github.com/gazebo-tooling/release-tools/pull/431#issuecomment-815099918 - ] + ] // DESC: exclude ignition from generate any install testing job gz_no_pkg_yet = [ ] // DESC: major versions that has a package in the prerelease repo. Should From 536dc339364d8e3f596ff26f0de73de1b3a6dac9 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 26 Jan 2023 16:00:19 +0100 Subject: [PATCH 282/540] Fix syntax --- jenkins-scripts/dsl/ignition.dsl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index 26998dfad..00589a2a2 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -89,7 +89,7 @@ gz_extra_debbuild = [ 'gui8', 'rendering8', 'sensors8', 'sim8', - 'transport12' + 'transport12', 'utils1' // see comment https://github.com/gazebo-tooling/release-tools/pull/431#issuecomment-815099918 ] // DESC: exclude ignition from generate any install testing job From 95563785cb75ca69799e364a8b2bb00441180d2b Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 26 Jan 2023 16:05:21 +0100 Subject: [PATCH 283/540] Use transport13 for new nightlies instead of 12 Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/ignition.dsl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index 00589a2a2..535d1bd36 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -89,7 +89,7 @@ gz_extra_debbuild = [ 'gui8', 'rendering8', 'sensors8', 'sim8', - 'transport12', + 'transport13', 'utils1' // see comment https://github.com/gazebo-tooling/release-tools/pull/431#issuecomment-815099918 ] // DESC: exclude ignition from generate any install testing job From c9ae388fc81dd0781a4c3788b5dd87d34795c0f6 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 26 Jan 2023 20:53:40 +0100 Subject: [PATCH 284/540] Use system s3cmd instead of using 1.5.0 Signed-off-by: Jose Luis Rivero --- jenkins-scripts/lib/repository_uploader.bash | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/jenkins-scripts/lib/repository_uploader.bash b/jenkins-scripts/lib/repository_uploader.bash index e9e451531..2b15375b5 100644 --- a/jenkins-scripts/lib/repository_uploader.bash +++ b/jenkins-scripts/lib/repository_uploader.bash @@ -25,18 +25,7 @@ S3_upload() S3_DIR=$(mktemp -d ${HOME}/s3.XXXX) pushd ${S3_DIR} - # Hack for not failing when github is down - update_done=false - seconds_waiting=0 - while (! $update_done); do - wget https://github.com/s3tools/s3cmd/archive/v1.5.0-rc1.tar.gz -O foo.tar.gz && update_done=true - sleep 1 - seconds_waiting=$((seconds_waiting+1)) - [ $seconds_waiting -gt 60 ] && exit 1 - done - tar xzf foo.tar.gz - cd s3cmd-* - ./s3cmd put $pkg s3://osrf-distributions/${s3_destination_path} + s3cmd put $pkg s3://osrf-distributions/${s3_destination_path} popd rm -fr ${S3_DIR} } From 1bcc4e4b42687160e1523f40f7b6d18b5f86ec4d Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 27 Jan 2023 17:05:29 +0100 Subject: [PATCH 285/540] Make vcpkg_install_package to upgrade to a new version if needed Signed-off-by: Jose Luis Rivero --- jenkins-scripts/lib/windows_library.bat | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/jenkins-scripts/lib/windows_library.bat b/jenkins-scripts/lib/windows_library.bat index 228bc4365..b71ebc9f5 100644 --- a/jenkins-scripts/lib/windows_library.bat +++ b/jenkins-scripts/lib/windows_library.bat @@ -276,6 +276,10 @@ git pull origin master || goto :error popd %VCPKG_CMD% install --recurse "%1" --overlay-ports="%VCPKG_OSRF_DIR%" +:: vcpkg does not upgrade installed packages using the install command +:: since most of the packages are coming from a frozen snapshot, it is +:: not a problem. However upgrading is needed for the osrf port overlay +%VCPKG_CMD% upgrade "%1" --overlay-ports="%VCPKG_OSRF_DIR%" goto :EOF :: ################################## From fb92f026fe52e7c71c8f2f853cb478132aa92686 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Mon, 30 Jan 2023 17:32:30 +0100 Subject: [PATCH 286/540] Add --no-dry-run to do the upgrades Signed-off-by: Jose Luis Rivero --- jenkins-scripts/lib/windows_library.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/lib/windows_library.bat b/jenkins-scripts/lib/windows_library.bat index b71ebc9f5..a0a539376 100644 --- a/jenkins-scripts/lib/windows_library.bat +++ b/jenkins-scripts/lib/windows_library.bat @@ -279,7 +279,7 @@ popd :: vcpkg does not upgrade installed packages using the install command :: since most of the packages are coming from a frozen snapshot, it is :: not a problem. However upgrading is needed for the osrf port overlay -%VCPKG_CMD% upgrade "%1" --overlay-ports="%VCPKG_OSRF_DIR%" +%VCPKG_CMD% upgrade "%1" --no-dry-run --overlay-ports="%VCPKG_OSRF_DIR%" goto :EOF :: ################################## From 935d43dbfa77bbde10994416b8fb541b1af1a65c Mon Sep 17 00:00:00 2001 From: Ian Chen Date: Sat, 11 Feb 2023 00:35:54 +0000 Subject: [PATCH 287/540] fuel-tools9 nightlies Signed-off-by: Ian Chen --- jenkins-scripts/dsl/ignition.dsl | 1 + jenkins-scripts/dsl/ignition_collection.dsl | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index 535d1bd36..cb514fb15 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -84,6 +84,7 @@ gz_prerelease_branches = [] // don't appear in gz_branches (like nightly builders or 0-debbuild // jobs for the special cases of foo0 packages) gz_extra_debbuild = [ 'gui8', + 'fuel-tools9', 'launch7', 'msgs10', 'rendering8', diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index 779179e5a..5a7443b08 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -30,7 +30,7 @@ gz_collections = [ 'msgs' : [ debbuild: 'gz-msgs10' , branch: 'main' ], 'rendering' : [ debbuild: 'gz-rendering8' , branch: 'main' ], 'sdformat' : [ debbuild: 'sdformat13' , branch: 'sdf13' ], - 'fuel-tools': [ debbuild: 'gz-fuel-tools8' , branch: 'gz-fuel-tools8' ], + 'fuel-tools': [ debbuild: 'gz-fuel-tools9' , branch: 'main' ], 'transport' : [ debbuild: 'gz-transport13' , branch: 'main' ], 'gui' : [ debbuild: 'gz-gui8' , branch: 'main' ], 'sensors' : [ debbuild: 'gz-sensors8' , branch: 'main' ], From 6709b713881d4e25df8474143a431c00d9db63ba Mon Sep 17 00:00:00 2001 From: Ian Chen Date: Sat, 11 Feb 2023 01:10:43 +0000 Subject: [PATCH 288/540] update jobs Signed-off-by: Ian Chen --- jenkins-scripts/dsl/ignition_collection.dsl | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index 5a7443b08..32eb5abce 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -310,25 +310,25 @@ gz_collection_jobs = 'harmonic' : [ 'ign_cmake-gz-3-win', 'ign_common-gz-5-win', - 'ign_fuel-tools-gz-8-win', + 'ign_fuel-tools-gz-9-win', 'ign_gazebo-ci-win', 'ign_gui-ci-win', 'ign_launch-ci-win', 'ign_math-gz-7-win', - 'ign_msgs-gz-9-win', + 'ign_msgs-gz-10-win', 'ign_physics-gz-6-win', 'ign_plugin-gz-2-win', 'ign_rendering-ci-win', 'ign_sensors-ci-win', 'ign_tools-gz-2-win', - 'ign_transport-gz-12-win', + 'ign_transport-gz-13-win', 'ign_utils-gz-2-win', 'ignition_cmake-ci-gz-cmake3-focal-amd64', 'ignition_cmake-ci-gz-cmake3-homebrew-amd64', 'ignition_common-ci-gz-common5-focal-amd64', 'ignition_common-ci-gz-common5-homebrew-amd64', - 'ignition_fuel-tools-ci-gz-fuel-tools8-focal-amd64', - 'ignition_fuel-tools-ci-gz-fuel-tools8-homebrew-amd64', + 'ignition_fuel-tools-ci-gz-fuel-tools9-focal-amd64', + 'ignition_fuel-tools-ci-gz-fuel-tools9-homebrew-amd64', 'ignition_harmonic-ci-main-homebrew-amd64', 'ignition_gazebo-ci-main-focal-amd64', 'ignition_gazebo-ci-main-homebrew-amd64', @@ -338,8 +338,8 @@ gz_collection_jobs = 'ignition_launch-ci-main-homebrew-amd64', 'ignition_math-ci-gz-math7-focal-amd64', 'ignition_math-ci-gz-math7-homebrew-amd64', - 'ignition_msgs-ci-gz-msgs9-focal-amd64', - 'ignition_msgs-ci-gz-msgs9-homebrew-amd64', + 'ignition_msgs-ci-gz-msgs10-focal-amd64', + 'ignition_msgs-ci-gz-msgs10-homebrew-amd64', 'ignition_physics-ci-gz-physics6-focal-amd64', 'ignition_physics-ci-gz-physics6-homebrew-amd64', 'ignition_plugin-ci-gz-plugin2-focal-amd64', @@ -350,8 +350,8 @@ gz_collection_jobs = 'ignition_sensors-ci-main-homebrew-amd64', 'ignition_tools-ci-gz-tools2-focal-amd64', 'ignition_tools-ci-gz-tools2-homebrew-amd64', - 'ignition_transport-ci-gz-transport12-focal-amd64', - 'ignition_transport-ci-gz-transport12-homebrew-amd64', + 'ignition_transport-ci-gz-transport13-focal-amd64', + 'ignition_transport-ci-gz-transport13-homebrew-amd64', 'ignition_utils-ci-gz-utils2-focal-amd64', 'ignition_utils-ci-gz-utils2-homebrew-amd64', 'sdformat-ci-sdformat13-focal-amd64', From 4a99b5a6e6b3e28f239731044cb601a2b0c2de33 Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Tue, 21 Feb 2023 10:59:03 -0800 Subject: [PATCH 289/540] Set HOMEBREW_NO_INSTALL_FROM_API for CI jobs (#879) Homebrew 4.0 now installs from an API by default instead of requiring a large git clone, but it doesn't work for all operations. Set an environment variable for our CI jobs to avoid API installs for now. Signed-off-by: Steve Peters --- jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash | 1 + jenkins-scripts/lib/project-install-homebrew.bash | 1 + 2 files changed, 2 insertions(+) diff --git a/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash b/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash index eb8c0f227..6851e13e5 100644 --- a/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash +++ b/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash @@ -44,6 +44,7 @@ fi # Step 1. Set up homebrew echo "# BEGIN SECTION: clean up ${HOMEBREW_PREFIX}" +export HOMEBREW_NO_INSTALL_FROM_API=1 . ${SCRIPT_DIR}/lib/_homebrew_cleanup.bash . ${SCRIPT_DIR}/lib/_homebrew_base_setup.bash brew cleanup || echo "brew cleanup couldn't be run" diff --git a/jenkins-scripts/lib/project-install-homebrew.bash b/jenkins-scripts/lib/project-install-homebrew.bash index 39dc5367c..4f2fc11e6 100644 --- a/jenkins-scripts/lib/project-install-homebrew.bash +++ b/jenkins-scripts/lib/project-install-homebrew.bash @@ -22,6 +22,7 @@ fi # Step 1. Set up homebrew echo "# BEGIN SECTION: clean up ${HOMEBREW_PREFIX}" +export HOMEBREW_NO_INSTALL_FROM_API=1 . ${SCRIPT_DIR}/lib/_homebrew_cleanup.bash . ${SCRIPT_DIR}/lib/_homebrew_base_setup.bash brew cleanup || echo "brew cleanup couldn't be run" From 2b61a02aad5fd16cb2d174e73bec08e1b5f3fe85 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 23 Feb 2023 13:12:42 +0100 Subject: [PATCH 290/540] Support MSVC 2022 Signed-off-by: Jose Luis Rivero --- jenkins-scripts/lib/windows_library.bat | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/jenkins-scripts/lib/windows_library.bat b/jenkins-scripts/lib/windows_library.bat index a0a539376..f66fed21e 100644 --- a/jenkins-scripts/lib/windows_library.bat +++ b/jenkins-scripts/lib/windows_library.bat @@ -37,6 +37,8 @@ IF %PLATFORM_TO_BUILD% == x86 ( ) echo "Configure the VC++ compilation" +set MSVC22_ON_WIN32_C=C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Auxiliary\Build\vcvarsall.bat +:: 2019 versions set MSVC_ON_WIN64_E=C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat set MSVC_ON_WIN32_E=C:\Program Files\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat set MSVC_ON_WIN64_C=C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build\vcvarsall.bat @@ -46,7 +48,9 @@ set LIB_DIR="%~dp0" call %LIB_DIR%\windows_env_vars.bat set PATH=%PATH%;%VCPKG_DIR%\installed\%VCPKG_DEFAULT_TRIPLET%\bin -IF exist "%MSVC_ON_WIN64_E%" ( +IF exist "%MSVC22_ON_WIN32_C%" ( + call "%MSVC22_ON_WIN32_C%" %MSVC_KEYWORD% || goto %win_lib% :error +) ELSE IF exist "%MSVC_ON_WIN64_E%" ( call "%MSVC_ON_WIN64_E%" %MSVC_KEYWORD% || goto %win_lib% :error ) ELSE IF exist "%MSVC_ON_WIN32_E%" ( call "%MSVC_ON_WIN32_E%" %MSVC_KEYWORD% || goto %win_lib% :error From a4ec6f30fba1e3006d4b9da3cc73e34db010473f Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 28 Feb 2023 19:41:03 +0100 Subject: [PATCH 291/540] Fix curl command to get job jar file Signed-off-by: Jose Luis Rivero --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 127b24092..d40944b46 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -41,7 +41,7 @@ jobs: java-version: '11' - name: Download job dsl jar if: steps.dsl_check.outputs.run_job == 'true' - run: curl https://repo.jenkins-ci.org/public/org/jenkins-ci/plugins/job-dsl-core/1.77/job-dsl-core-1.77-standalone.jar -o jobdsl.jar + run: curl -sSL https://repo.jenkins-ci.org/public/org/jenkins-ci/plugins/job-dsl-core/1.77/job-dsl-core-1.77-standalone.jar -o jobdsl.jar - name: Generate all DSL files if: steps.dsl_check.outputs.run_job == 'true' run: | From b68d771985a20c2c465b7e805d8b1912a4cb154b Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Thu, 2 Mar 2023 13:20:20 -0800 Subject: [PATCH 292/540] Set HOMEBREW_NO_INSTALL_FROM_API for bottle builds (#881) Follow up to #879. Signed-off-by: Steve Peters --- jenkins-scripts/lib/homebrew_bottle_creation.bash | 1 + 1 file changed, 1 insertion(+) diff --git a/jenkins-scripts/lib/homebrew_bottle_creation.bash b/jenkins-scripts/lib/homebrew_bottle_creation.bash index e9358e5f0..bf776ad8c 100644 --- a/jenkins-scripts/lib/homebrew_bottle_creation.bash +++ b/jenkins-scripts/lib/homebrew_bottle_creation.bash @@ -54,6 +54,7 @@ fi echo '# END SECTION' echo '# BEGIN SECTION: clean up environment' +export HOMEBREW_NO_INSTALL_FROM_API=1 rm -fr ${PKG_DIR} && mkdir -p ${PKG_DIR} . ${SCRIPT_LIBDIR}/_homebrew_cleanup.bash # don't use HOMEBREW_UPDATE_TO_TAG for bottle builds From d811bac730e732d68d04fa3e292afb0e103abc0c Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Tue, 7 Mar 2023 14:37:04 -0800 Subject: [PATCH 293/540] brew without sudo: part 1 (#882) Remove build folder without sudo. Signed-off-by: Steve Peters --- jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash b/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash index 6851e13e5..8fe8178b0 100644 --- a/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash +++ b/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash @@ -111,8 +111,7 @@ echo '# END SECTION' # Step 3. Manually compile and install ${PROJECT} echo "# BEGIN SECTION: configure ${PROJECT}" cd ${WORKSPACE}/${PROJECT_PATH} -# Need the sudo since the test are running with roots perms to access to GUI -sudo rm -fr ${WORKSPACE}/build +rm -fr ${WORKSPACE}/build mkdir -p ${WORKSPACE}/build cd ${WORKSPACE}/build From 060f80fe8efbdb2c250555903b8451f4b057a949 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 8 Mar 2023 11:10:28 +0100 Subject: [PATCH 294/540] Fix generation of sim7 bottle Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/ignition.dsl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index 535d1bd36..4b33338a3 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -695,8 +695,8 @@ gz_software.each { gz_sw -> } // 3. install jobs to test bottles - supported_install_pkg_branches(software_name).each { major_version, supported_distros -> - def install_default_job = job("ignition_${software_name}${major_version}-install_bottle-homebrew-amd64") + supported_install_pkg_branches(gz_sw).each { major_version, supported_distros -> + def install_default_job = job("ignition_${gz_sw}${major_version}-install_bottle-homebrew-amd64") OSRFBrewInstall.create(install_default_job) install_default_job.with @@ -709,7 +709,7 @@ gz_software.each { gz_sw -> cron('@daily') } - def bottle_name = "ignition-${software_name}${major_version}" + def bottle_name = "ignition-${gz_sw}${major_version}" steps { shell("""\ From e4beb7703e222614665a366b8d9aedb118374626 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 8 Mar 2023 12:11:42 +0100 Subject: [PATCH 295/540] Use right short name for Windows CI jobs using gz-sim branches Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/ignition.dsl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index 26a7a8747..ebca11759 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -799,6 +799,8 @@ gz_software.each { gz_sw -> branch_name = "ci" else branch_name = branch - gz_sw + // Deal with the special case of changing gazebo name by sim + branch_name = branch_name.replace('gz-sim','gz-') gz_win_ci_job_name = "ign_${gz_sw}-${branch_name}-win" } else { gz_win_ci_job_name = "ignition_${gz_sw}-ci-${branch}-windows7-amd64" From 6cd691fd7fce68792519f7b4085f0106b0d33fb5 Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Wed, 8 Mar 2023 13:11:20 -0800 Subject: [PATCH 296/540] brew without sudo: part 2 (#883) Add group write permissions without sudo. Signed-off-by: Steve Peters --- jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash | 4 ++-- jenkins-scripts/lib/project-install-homebrew.bash | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash b/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash index 8fe8178b0..d8fc2a912 100644 --- a/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash +++ b/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash @@ -49,7 +49,7 @@ export HOMEBREW_NO_INSTALL_FROM_API=1 . ${SCRIPT_DIR}/lib/_homebrew_base_setup.bash brew cleanup || echo "brew cleanup couldn't be run" mkdir -p ${HOMEBREW_CELLAR} -sudo chmod -R ug+rwx ${HOMEBREW_CELLAR} +chmod -R ug+rwx ${HOMEBREW_CELLAR} echo '# END SECTION' echo '# BEGIN SECTION: brew information' @@ -214,5 +214,5 @@ rm -fr \$HOME/.gazebo/models test_results* echo '# END SECTION' echo "# BEGIN SECTION: re-add group write permissions" -sudo chmod -R ug+rwx ${HOMEBREW_CELLAR} +chmod -R ug+rwx ${HOMEBREW_CELLAR} echo '# END SECTION' diff --git a/jenkins-scripts/lib/project-install-homebrew.bash b/jenkins-scripts/lib/project-install-homebrew.bash index 4f2fc11e6..cfdedf0d2 100644 --- a/jenkins-scripts/lib/project-install-homebrew.bash +++ b/jenkins-scripts/lib/project-install-homebrew.bash @@ -27,7 +27,7 @@ export HOMEBREW_NO_INSTALL_FROM_API=1 . ${SCRIPT_DIR}/lib/_homebrew_base_setup.bash brew cleanup || echo "brew cleanup couldn't be run" mkdir -p ${HOMEBREW_CELLAR} -sudo chmod -R ug+rwx ${HOMEBREW_CELLAR} +chmod -R ug+rwx ${HOMEBREW_CELLAR} echo '# END SECTION' echo '# BEGIN SECTION: brew information' @@ -81,5 +81,5 @@ brew list | grep '^szip$' || brew doctor || echo MARK_AS_UNSTABLE echo '# END SECTION' echo "# BEGIN SECTION: re-add group write permissions" -sudo chmod -R ug+rwx ${HOMEBREW_CELLAR} +chmod -R ug+rwx ${HOMEBREW_CELLAR} echo '# END SECTION' From a2ecf73a0782d404a66aaadf873218e130cb0147 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 9 Mar 2023 13:01:33 +0100 Subject: [PATCH 297/540] Added the ros2 installation document --- .github/ISSUE_TEMPLATE/release_collection.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/release_collection.md b/.github/ISSUE_TEMPLATE/release_collection.md index 61d58e520..9ec760bd9 100644 --- a/.github/ISSUE_TEMPLATE/release_collection.md +++ b/.github/ISSUE_TEMPLATE/release_collection.md @@ -65,7 +65,8 @@ When opening PRs, add a link back to this issue for easier tracking. * - [ ] Update references to the release install instructions * [ ] Versions doc https://github.com/gazebosim/docs/blob/master/tools/versions.md - * [ ] ROS2 integration https://github.com/gazebosim/docs/blob/master/\/ros2_integration.md + * [ ] ROS2 Installation https://github.com/gazebosim/docs/blob/master/ros_installation.md + * [ ] ROS2 Integration https://github.com/gazebosim/docs/blob/master/\/ros2_integration.md * [ ] ros_gz README https://github.com/gazebosim/ros_gz/blob/ros2/README.md - [ ] PRs fixing documentation and critical bugs can be merged and pre-released - As libraries have all their documentation reviewed: From 9d4cd191fb317c1044934b181922db297647bbf3 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 9 Mar 2023 22:28:13 +0000 Subject: [PATCH 298/540] Use right short name for Windows CI jobs using gz-sim branches (#887) Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/ignition.dsl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index ebca11759..f5d7cca3c 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -711,6 +711,10 @@ gz_software.each { gz_sw -> } def bottle_name = "ignition-${gz_sw}${major_version}" + // For transiting, use always gz-sim new name since new versions won't + // have ign-gazebo aliases + if ("${gz_sw}" == "sim" || "${gz_sw}" == "gazebo") + bottle_name = "gz-sim${major_version}" steps { shell("""\ From 17a157bc571109a5031cbf0282ea3173218fcd56 Mon Sep 17 00:00:00 2001 From: "Addisu Z. Taddese" Date: Sat, 11 Mar 2023 05:14:33 +0000 Subject: [PATCH 299/540] Update the Garden collection/view on Jenkins (#884) Signed-off-by: Addisu Z. Taddese Co-authored-by: Jose Luis Rivero --- jenkins-scripts/dsl/ignition_collection.dsl | 57 ++++++++------------- 1 file changed, 21 insertions(+), 36 deletions(-) diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index 32eb5abce..c2722c01e 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -212,7 +212,7 @@ gz_collection_jobs = 'gz_common5-install-pkg-focal-amd64', 'gz_fuel-tools8-install-pkg-focal-amd64', 'gz_garden-install-pkg-focal-amd64', - 'gz_gazebo7-install-pkg-focal-amd64', + 'gz_sim7-install-pkg-focal-amd64', 'gz_gui7-install-pkg-focal-amd64', 'gz_launch6-install-pkg-focal-amd64', 'gz_math7-install-pkg-focal-amd64', @@ -224,10 +224,11 @@ gz_collection_jobs = 'gz_tools2-install-pkg-focal-amd64', 'gz_transport12-install-pkg-focal-amd64', 'gz_utils2-install-pkg-focal-amd64', + 'sdformat-install-sdformat13_pkg-focal-amd64', 'ign_cmake-gz-3-win', 'ign_common-gz-5-win', 'ign_fuel-tools-gz-8-win', - 'ign_gazebo-ci-win', + 'ign_gazebo-gz-7-win', 'ign_gui-gz-7-win', 'ign_launch-gz-6-win', 'ign_math-gz-7-win', @@ -239,73 +240,57 @@ gz_collection_jobs = 'ign_tools-gz-2-win', 'ign_transport-gz-12-win', 'ign_utils-gz-2-win', + 'sdformat-sdf-13-win', 'ignition_cmake-ci-gz-cmake3-focal-amd64', 'ignition_cmake-ci-gz-cmake3-homebrew-amd64', - 'ignition_cmake2-install-pkg-focal-amd64', - 'ignition_cmake2-install_bottle-homebrew-amd64', + 'ignition_cmake3-install_bottle-homebrew-amd64', 'ignition_common-ci-gz-common5-focal-amd64', 'ignition_common-ci-gz-common5-homebrew-amd64', - 'ignition_common4-install-pkg-focal-amd64', - 'ignition_common4-install_bottle-homebrew-amd64', + 'ignition_common5-install_bottle-homebrew-amd64', 'ignition_fuel-tools-ci-gz-fuel-tools8-focal-amd64', 'ignition_fuel-tools-ci-gz-fuel-tools8-homebrew-amd64', - 'ignition_fuel-tools7-install-pkg-focal-amd64', - 'ignition_fuel-tools7-install_bottle-homebrew-amd64', + 'ignition_fuel-tools8-install_bottle-homebrew-amd64', 'ignition_garden-ci-main-homebrew-amd64', 'ignition_garden-install-pkg-focal-amd64', 'ignition_garden-install_bottle-homebrew-amd64', - 'ignition_gazebo-ci-main-focal-amd64', - 'ignition_gazebo-ci-main-homebrew-amd64', - 'ignition_gazebo6-install-pkg-focal-amd64', - 'ignition_gazebo6-install_bottle-homebrew-amd64', + 'ignition_gazebo-ci-gz-sim7-focal-amd64', + 'ignition_gazebo-ci-gz-sim7-homebrew-amd64', + 'ignition_sim7-install_bottle-homebrew-amd64', 'ignition_gui-ci-gz-gui7-focal-amd64', 'ignition_gui-ci-gz-gui7-homebrew-amd64', - 'ignition_gui6-install-pkg-focal-amd64', 'ignition_gui6-install_bottle-homebrew-amd64', 'ignition_launch-ci-gz-launch6-focal-amd64', 'ignition_launch-ci-gz-launch6-homebrew-amd64', - 'ignition_launch5-install-pkg-focal-amd64', - 'ignition_launch5-install_bottle-homebrew-amd64', + 'ignition_launch6-install_bottle-homebrew-amd64', 'ignition_math-ci-gz-math7-focal-amd64', 'ignition_math-ci-gz-math7-homebrew-amd64', - 'ignition_math6-install-pkg-focal-amd64', - 'ignition_math6-install_bottle-homebrew-amd64', + 'ignition_math7-install_bottle-homebrew-amd64', 'ignition_msgs-ci-gz-msgs9-focal-amd64', 'ignition_msgs-ci-gz-msgs9-homebrew-amd64', - 'ignition_msgs8-install-pkg-focal-amd64', - 'ignition_msgs8-install_bottle-homebrew-amd64', + 'ignition_msgs9-install_bottle-homebrew-amd64', 'ignition_physics-ci-gz-physics6-focal-amd64', 'ignition_physics-ci-gz-physics6-homebrew-amd64', - 'ignition_physics5-install-pkg-focal-amd64', - 'ignition_physics5-install_bottle-homebrew-amd64', + 'ignition_physics6-install_bottle-homebrew-amd64', 'ignition_plugin-ci-gz-plugin2-focal-amd64', 'ignition_plugin-ci-gz-plugin2-homebrew-amd64', - 'ignition_plugin-install-pkg-focal-amd64', - 'ignition_plugin1-install_bottle-homebrew-amd64', + 'ignition_plugin2-install_bottle-homebrew-amd64', 'ignition_rendering-ci-gz-rendering7-focal-amd64', 'ignition_rendering-ci-gz-rendering7-homebrew-amd64', - 'ignition_rendering6-install-pkg-focal-amd64', - 'ignition_rendering6-install_bottle-homebrew-amd64', + 'ignition_rendering7-install_bottle-homebrew-amd64', 'ignition_sensors-ci-gz-sensors7-focal-amd64', 'ignition_sensors-ci-gz-sensors7-homebrew-amd64', - 'ignition_sensors6-install-pkg-focal-amd64', - 'ignition_sensors6-install_bottle-homebrew-amd64', + 'ignition_sensors7-install_bottle-homebrew-amd64', 'ignition_tools-ci-gz-tools2-focal-amd64', 'ignition_tools-ci-gz-tools2-homebrew-amd64', - 'ignition_tools-install-pkg-focal-amd64', - 'ignition_tools1-install_bottle-homebrew-amd64', + 'ignition_tools2-install_bottle-homebrew-amd64', 'ignition_transport-ci-gz-transport12-focal-amd64', 'ignition_transport-ci-gz-transport12-homebrew-amd64', - 'ignition_transport11-install-pkg-focal-amd64', - 'ignition_transport11-install_bottle-homebrew-amd64', + 'ignition_transport12-install_bottle-homebrew-amd64', 'ignition_utils-ci-gz-utils2-focal-amd64', 'ignition_utils-ci-gz-utils2-homebrew-amd64', - 'ignition_utils-install-pkg-focal-amd64', - 'ignition_utils1-install_bottle-homebrew-amd64', + 'ignition_utils2-install_bottle-homebrew-amd64', 'sdformat-ci-sdformat13-focal-amd64', - 'sdformat-ci-sdformat13-homebrew-amd64', - 'sdformat-install-sdformat12_pkg-focal-amd64', - 'sdformat-sdf-13-win' + 'sdformat-ci-sdformat13-homebrew-amd64' ], 'harmonic' : [ 'ign_cmake-gz-3-win', From 7790bb57b6d30d668a5f40b60bf0e41be6985a9c Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Mon, 13 Mar 2023 10:42:54 +0100 Subject: [PATCH 300/540] Not needed after updates on the document Removed custom references in https://github.com/gazebosim/docs/pull/350 --- .github/ISSUE_TEMPLATE/release_collection.md | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/release_collection.md b/.github/ISSUE_TEMPLATE/release_collection.md index 9ec760bd9..bad24737c 100644 --- a/.github/ISSUE_TEMPLATE/release_collection.md +++ b/.github/ISSUE_TEMPLATE/release_collection.md @@ -66,7 +66,6 @@ When opening PRs, add a link back to this issue for easier tracking. - [ ] Update references to the release install instructions * [ ] Versions doc https://github.com/gazebosim/docs/blob/master/tools/versions.md * [ ] ROS2 Installation https://github.com/gazebosim/docs/blob/master/ros_installation.md - * [ ] ROS2 Integration https://github.com/gazebosim/docs/blob/master/\/ros2_integration.md * [ ] ros_gz README https://github.com/gazebosim/ros_gz/blob/ros2/README.md - [ ] PRs fixing documentation and critical bugs can be merged and pre-released - As libraries have all their documentation reviewed: From 7dbba1ef9bc78ce27c469d0887b8de67af0f640f Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Fri, 17 Mar 2023 17:40:02 -0700 Subject: [PATCH 301/540] DSL: less email for scpeters (#888) Signed-off-by: Steve Peters --- jenkins-scripts/dsl/_configs_/Globals.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/_configs_/Globals.groovy b/jenkins-scripts/dsl/_configs_/Globals.groovy index 6ef9859c0..58cbfcd19 100644 --- a/jenkins-scripts/dsl/_configs_/Globals.groovy +++ b/jenkins-scripts/dsl/_configs_/Globals.groovy @@ -3,7 +3,7 @@ package _configs_ class Globals { // Notifications for email ext plugin - static default_emails = '$DEFAULT_RECIPIENTS, scpeters@osrfoundation.org' + static default_emails = '$DEFAULT_RECIPIENTS' static build_cop_email = 'buildcop@osrfoundation.org' static extra_emails = '' From 80c36c55e6f93e75251594c8856e14c4ab1f0bbb Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 29 Mar 2023 16:02:08 +0200 Subject: [PATCH 302/540] Move ros_gazebo_pkgs bloom scripts to a subdirectory Signed-off-by: Jose Luis Rivero --- bloom/{ => ros_gazebo_pkgs}/README.md | 0 .../{ => ros_gazebo_pkgs}/initial_info_for_new_release_repo.bash | 0 bloom/{ => ros_gazebo_pkgs}/release-bloom.py | 0 bloom/{ => ros_gazebo_pkgs}/rename-gazebo-ros-pkgs.bash | 0 bloom/{ => ros_gazebo_pkgs}/ros_gazebo_pkgs-release.py.bash | 0 bloom/{ => ros_gazebo_pkgs}/ros_gz_bridge-release.py.bash | 0 6 files changed, 0 insertions(+), 0 deletions(-) rename bloom/{ => ros_gazebo_pkgs}/README.md (100%) rename bloom/{ => ros_gazebo_pkgs}/initial_info_for_new_release_repo.bash (100%) rename bloom/{ => ros_gazebo_pkgs}/release-bloom.py (100%) rename bloom/{ => ros_gazebo_pkgs}/rename-gazebo-ros-pkgs.bash (100%) rename bloom/{ => ros_gazebo_pkgs}/ros_gazebo_pkgs-release.py.bash (100%) rename bloom/{ => ros_gazebo_pkgs}/ros_gz_bridge-release.py.bash (100%) diff --git a/bloom/README.md b/bloom/ros_gazebo_pkgs/README.md similarity index 100% rename from bloom/README.md rename to bloom/ros_gazebo_pkgs/README.md diff --git a/bloom/initial_info_for_new_release_repo.bash b/bloom/ros_gazebo_pkgs/initial_info_for_new_release_repo.bash similarity index 100% rename from bloom/initial_info_for_new_release_repo.bash rename to bloom/ros_gazebo_pkgs/initial_info_for_new_release_repo.bash diff --git a/bloom/release-bloom.py b/bloom/ros_gazebo_pkgs/release-bloom.py similarity index 100% rename from bloom/release-bloom.py rename to bloom/ros_gazebo_pkgs/release-bloom.py diff --git a/bloom/rename-gazebo-ros-pkgs.bash b/bloom/ros_gazebo_pkgs/rename-gazebo-ros-pkgs.bash similarity index 100% rename from bloom/rename-gazebo-ros-pkgs.bash rename to bloom/ros_gazebo_pkgs/rename-gazebo-ros-pkgs.bash diff --git a/bloom/ros_gazebo_pkgs-release.py.bash b/bloom/ros_gazebo_pkgs/ros_gazebo_pkgs-release.py.bash similarity index 100% rename from bloom/ros_gazebo_pkgs-release.py.bash rename to bloom/ros_gazebo_pkgs/ros_gazebo_pkgs-release.py.bash diff --git a/bloom/ros_gz_bridge-release.py.bash b/bloom/ros_gazebo_pkgs/ros_gz_bridge-release.py.bash similarity index 100% rename from bloom/ros_gz_bridge-release.py.bash rename to bloom/ros_gazebo_pkgs/ros_gz_bridge-release.py.bash From 5b38ad0e62a2a2e6f334daca9a5d9a0007a1036c Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 29 Mar 2023 16:05:38 +0200 Subject: [PATCH 303/540] Add link in README Signed-off-by: Jose Luis Rivero --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 8958df1f2..ef5d4ab61 100644 --- a/README.md +++ b/README.md @@ -5,10 +5,10 @@ Gazebo software. ## Scripts -* [release.py](release.py): Triggers new debian and homebrew releases (major, minor, patch, pre-release...). - -* For -release repository scripts, please see [release-repo-scripts/README.md](release-repo-scripts/README.md) -* For source repository scripts, please see [source-repo-scripts/README.md](source-repo-scripts/README.md) + * [release.py](release.py): Triggers new debian and homebrew releases (major, minor, patch, pre-release...). + * For -release repository scripts, please see [release-repo-scripts/README.md](release-repo-scripts/README.md) + * For source repository scripts, please see [source-repo-scripts/README.md](source-repo-scripts/README.md) + * For scripts related to unofficial ROS packages, please see [bloom scripts](bloom/ros_gazebo_pkgs/README.md) ### Making releases @@ -17,4 +17,4 @@ For the developers officially maintaining Gazebo software that need to run a new * The [Gazebo Release Process](https://gazebosim.org/docs/garden/release) document provides an overview of how releasing works. * The [Gazebo Release Instructions](https://gazebosim.org/docs/garden/releases-instructions) - document provides step by step instructions to run new releases. \ No newline at end of file + document provides step by step instructions to run new releases. From f120797fec4f15ae31ac1c6e62ad3f9974d9029a Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 29 Mar 2023 16:26:19 +0200 Subject: [PATCH 304/540] Remove obsolete job for the ign brdige Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/ros1_ign_bridge.dsl | 89 ------------------------- 1 file changed, 89 deletions(-) delete mode 100644 jenkins-scripts/dsl/ros1_ign_bridge.dsl diff --git a/jenkins-scripts/dsl/ros1_ign_bridge.dsl b/jenkins-scripts/dsl/ros1_ign_bridge.dsl deleted file mode 100644 index 894d8f8de..000000000 --- a/jenkins-scripts/dsl/ros1_ign_bridge.dsl +++ /dev/null @@ -1,89 +0,0 @@ -import _configs_.* -import javaposse.jobdsl.dsl.Job - -def bridge_packages = [ - 'ros_ign_bridge', - 'ros_ign_gazebo_demos', - 'ros_ign_image', - 'ros_ign', - 'ros_ign_gazebo', - 'ros_ign_point_cloud' -] - -// BLOOM PACKAGE BUILDER JOBS -bridge_packages.each { pkg -> - pkg_dashed = pkg.replaceAll("_", "-") - - def build_pkg_job = job("$pkg_dashed-bloom-debbuilder") - - // Use the linux install as base - OSRFLinuxBuildPkgBase.create(build_pkg_job) - GenericRemoteToken.create(build_pkg_job) - - build_pkg_job.with - { - properties { - priority 100 - } - - configure { project -> - project / 'properties' / 'hudson.plugins.copyartifact.CopyArtifactPermissionProperty' / 'projectNameList' { - 'string' 'repository_uploader_*' - } - } - - parameters { - stringParam("PACKAGE","$pkg_dashed","Package name to be built") - stringParam("IGNITION_VERSION", '', 'Gazebo release supported in the binaries') - stringParam("VERSION",null,"Packages version to be built") - stringParam("RELEASE_VERSION", null, "Packages release version") - stringParam("LINUX_DISTRO", 'ubuntu', "Linux distribution to build packages for") - stringParam("DISTRO", "bionic", "Linux release inside LINUX_DISTRO to build packages for") - stringParam("ARCH", "amd64", "Architecture to build packages for") - stringParam('ROS_DISTRO', 'noetic','ROS DISTRO to build pakcages for') - stringParam("UPLOAD_TO_REPO", 'stable', "OSRF repo name to upload the package to") - stringParam('UPSTREAM_RELEASE_REPO', 'https://github.com/gazebo-release/ros1_ign_bridge-release', 'Release repository url') - } - - steps { - systemGroovyCommand("""\ - build.setDescription( - '' + build.buildVariableResolver.resolve('ROS_DISTRO') + '-' - + build.buildVariableResolver.resolve('VERSION') + '-' - + build.buildVariableResolver.resolve('RELEASE_VERSION') + '' + - '(' + build.buildVariableResolver.resolve('LINUX_DISTRO') + '/' + - build.buildVariableResolver.resolve('DISTRO') + '::' + - build.buildVariableResolver.resolve('ARCH') + ')' + - '
' + - 'upload to: ' + build.buildVariableResolver.resolve('UPLOAD_TO_REPO') + - '
' + - 'RTOOLS_BRANCH: ' + build.buildVariableResolver.resolve('RTOOLS_BRANCH')); - """.stripIndent() - ) - } - - publishers { - downstreamParameterized { - trigger('repository_uploader_packages') { - condition('SUCCESS') - parameters { - currentBuild() - predefinedProp("PROJECT_NAME_TO_COPY_ARTIFACTS", "\${JOB_NAME}") - predefinedProp("PACKAGE_ALIAS", "\${JOB_NAME}") - } - } - } - } - - - steps { - shell("""\ - #!/bin/bash -xe - - export OSRF_REPOS_TO_USE='stable' - - /bin/bash -x ./scripts/jenkins-scripts/docker/bloom-debbuild.bash - """.stripIndent()) - } - } -} From f361533ee02795806b19334f84fd018df2e5d7d6 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 29 Mar 2023 16:26:45 +0200 Subject: [PATCH 305/540] Implement the ros_gz debbuilder packages See https://github.com/gazebo-tooling/release-tools/issues/895 Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/ros_gz_bridge.dsl | 156 ++++++++++++++++++++++++++ 1 file changed, 156 insertions(+) create mode 100644 jenkins-scripts/dsl/ros_gz_bridge.dsl diff --git a/jenkins-scripts/dsl/ros_gz_bridge.dsl b/jenkins-scripts/dsl/ros_gz_bridge.dsl new file mode 100644 index 000000000..7f49d0232 --- /dev/null +++ b/jenkins-scripts/dsl/ros_gz_bridge.dsl @@ -0,0 +1,156 @@ +import _configs_.* +import javaposse.jobdsl.dsl.Job + +def bridge_packages = [ + 'ros_gz', + 'ros_gz_bridge', + 'ros_gz_image', + 'ros_gz_interfaces', + 'ros_gz_sim', + 'ros_gz_sim_demos' +] + +// BLOOM PACKAGE BUILDER JOBS +bridge_packages.each { pkg -> + pkg_dashed = pkg.replaceAll("_", "-") + + postfix_job_str = "bloom-debbuilder" + def build_pkg_job = job("${pkg_dashed}-${postfix_job_str}") + + // Use the linux install as base + OSRFLinuxBuildPkgBase.create(build_pkg_job) + GenericRemoteToken.create(build_pkg_job) + + build_pkg_job.with + { + properties { + priority 100 + } + + configure { project -> + project / 'properties' / 'hudson.plugins.copyartifact.CopyArtifactPermissionProperty' / 'projectNameList' { + 'string' 'repository_uploader_*' + } + } + + parameters { + stringParam("PACKAGE","$pkg_dashed","Package name to be built") + stringParam("VERSION",null,"Packages version to be built") + stringParam("RELEASE_VERSION", null, "Packages release version") + stringParam("LINUX_DISTRO", 'ubuntu', "Linux distribution to build packages for") + stringParam("DISTRO", "jammy", "Linux release inside LINUX_DISTRO to build packages for") + stringParam("ARCH", "amd64", "Architecture to build packages for") + stringParam('ROS_DISTRO', 'humble','ROS DISTRO to build pakcages for') + stringParam("UPLOAD_TO_REPO", 'stable', "OSRF repo name to upload the package to") + stringParam('UPSTREAM_RELEASE_REPO', 'https://github.com/j-rivero/ros_ign-release', 'Release repository url') + } + + // Blocks to control dependencies + projects_to_blockon = [] + if ("${pkg}" == 'ros_gz_sim_demos') + projects_to_blockon = ["ros-gz-sim-${postfix_job_str}", + "ros-gz-bridge-${postfix_job_str}", + "ros-gz-image-${postfix_job_str}"] + else if ("${pkg}" == 'ros_gz_image') + projects_to_blockon = ["ros-gz-bridge-${postfix_job_str}"] + else if ("${pkg}" == 'ros_gz_bridge') + projects_to_blockon = ["ros-gz-interfaces-${postfix_job_str}"] + else if ("${pkg}" == 'ros_gz') + projects_to_blockon = ["ros-gz-sim-demos-${postfix_job_str}", + "ros-gz-sim-${postfix_job_str}", + "ros-gz-bridge-${postfix_job_str}", + "ros-gz-image-${postfix_job_str}"] + + if (projects_to_blockon) { + blockOn(projects_to_blockon) { + blockLevel('GLOBAL') + scanQueueFor('ALL') + } + } + + + steps { + systemGroovyCommand("""\ + build.setDescription( + '' + build.buildVariableResolver.resolve('ROS_DISTRO') + '-' + + build.buildVariableResolver.resolve('VERSION') + '-' + + build.buildVariableResolver.resolve('RELEASE_VERSION') + '' + + '(' + build.buildVariableResolver.resolve('LINUX_DISTRO') + '/' + + build.buildVariableResolver.resolve('DISTRO') + '::' + + build.buildVariableResolver.resolve('ARCH') + ')' + + '
' + + 'upload to: ' + build.buildVariableResolver.resolve('UPLOAD_TO_REPO') + + '
' + + 'RTOOLS_BRANCH: ' + build.buildVariableResolver.resolve('RTOOLS_BRANCH')); + """.stripIndent() + ) + } + + publishers { + downstreamParameterized { + trigger('repository_uploader_packages') { + condition('SUCCESS') + parameters { + currentBuild() + predefinedProp("PROJECT_NAME_TO_COPY_ARTIFACTS", "\${JOB_NAME}") + predefinedProp("PACKAGE_ALIAS", "\${JOB_NAME}") + } + } + } + } + + + steps { + shell("""\ + #!/bin/bash -xe + + export OSRF_REPOS_TO_USE='stable' + export ROS2=true + + /bin/bash -x ./scripts/jenkins-scripts/docker/bloom-debbuild.bash + """.stripIndent()) + } + } +} + +def install_test_job = +job("ros_gz_bridge-install-pkg_unofficial-any-manual") +OSRFLinuxInstall.create(install_test_job) +install_test_job.with +{ + parameters { + stringParam("LINUX_DISTRO", 'ubuntu', "Linux distribution to build packages for") + stringParam("DISTRO", '', "Linux release inside LINUX_DISTRO to build packages for") + stringParam("ARCH", '', "Architecture to build packages for") + stringParam("ROS_DISTRO", 'humble', "ROS distribution") + stringParam("GZ_VERSION", 'garden', "Gazebo version") + stringParam("OSRF_REPOS_TO_USE", 'stable prerelease', "Repositories to add to the testing install") + labelParam('JENKINS_NODE_TAG') { + description('Jenkins node or group to run the build on') + defaultValue('gpu-reliable') + } + } + + // Designed to be run manually. No triggers. + label "gpu-reliable" + + steps { + systemGroovyCommand("""\ + build.setDescription( + '' + build.buildVariableResolver.resolve('LINUX_DISTRO') + '/' + + build.buildVariableResolver.resolve('DISTRO') + '::' + + build.buildVariableResolver.resolve('ROS_DISTRO') + '::' + + build.buildVariableResolver.resolve('GZ_VERSION') + '' + + '
' + + 'RTOOLS_BRANCH: ' + build.buildVariableResolver.resolve('RTOOLS_BRANCH')); + """.stripIndent()) + + shell("""\ + #!/bin/bash -xe + + export INSTALL_JOB_PKG=ros-\${ROS_DISTRO}-ros-gz\${GZ_VERSION}-sim + export ROS2=true + /bin/bash -x ./scripts/jenkins-scripts/docker/ign_launch-install-test-job.bash + """.stripIndent()) + } +} From ca3cc4f0049a79e3953fbabbc673331a5d2fdacf Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 29 Mar 2023 16:27:59 +0200 Subject: [PATCH 306/540] Do not hardcode python-openssl Signed-off-by: Jose Luis Rivero --- .../docker/lib/debbuild-bloom-base.bash | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/jenkins-scripts/docker/lib/debbuild-bloom-base.bash b/jenkins-scripts/docker/lib/debbuild-bloom-base.bash index 7fea34302..dee903dd0 100644 --- a/jenkins-scripts/docker/lib/debbuild-bloom-base.bash +++ b/jenkins-scripts/docker/lib/debbuild-bloom-base.bash @@ -23,6 +23,7 @@ export ENABLE_REAPER=false PACKAGE_UNDERSCORE_NAME=${PACKAGE//-/_} . ${SCRIPT_DIR}/lib/boilerplate_prepare.sh +. ${SCRIPT_DIR}/lib/_gazebo_utils.sh cat > build.sh << DELIM ################################################### @@ -118,18 +119,18 @@ DELIM OSRF_REPOS_TO_USE=${OSRF_REPOS_TO_USE:=stable} USE_ROS_REPO=true DEPENDENCY_PKGS="devscripts \ - ubuntu-dev-tools \ - ubuntu-keyring \ - debhelper \ - cdbs \ - wget \ - ca-certificates \ - equivs \ - dh-make \ - git \ - python-openssl \ - ca-certificates \ - ${pythonv}-rosdep" + ubuntu-dev-tools \ + ubuntu-keyring \ + debhelper \ + cdbs \ + wget \ + ca-certificates \ + equivs \ + dh-make \ + git \ + ${pythonv}-openssl \ + ca-certificates \ + ${pythonv}-rosdep" . ${SCRIPT_DIR}/lib/docker_generate_dockerfile.bash . ${SCRIPT_DIR}/lib/docker_run.bash From c8403f77c90ff332dd831d2b606ceea4b2e87e2f Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 29 Mar 2023 16:38:46 +0200 Subject: [PATCH 307/540] Implement a locking system to orchestrate releasing of ros_gz Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/ros_gz_bridge.dsl | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/jenkins-scripts/dsl/ros_gz_bridge.dsl b/jenkins-scripts/dsl/ros_gz_bridge.dsl index 7f49d0232..c90ea2cd9 100644 --- a/jenkins-scripts/dsl/ros_gz_bridge.dsl +++ b/jenkins-scripts/dsl/ros_gz_bridge.dsl @@ -69,6 +69,30 @@ bridge_packages.each { pkg -> } + // Blocks to control dependencies + projects_to_blockon = [] + if ("${pkg}" == 'ros_gz_sim_demos') + projects_to_blockon = ["ros-gz-sim-${postfix_job_str}", + "ros-gz-bridge-${postfix_job_str}", + "ros-gz-image-${postfix_job_str}"] + else if ("${pkg}" == 'ros_gz_image') + projects_to_blockon = ["ros-gz-bridge-${postfix_job_str}"] + else if ("${pkg}" == 'ros_gz_bridge') + projects_to_blockon = ["ros-gz-interfaces-${postfix_job_str}"] + else if ("${pkg}" == 'ros_gz') + projects_to_blockon = ["ros-gz-sim-demos-${postfix_job_str}", + "ros-gz-sim-${postfix_job_str}", + "ros-gz-bridge-${postfix_job_str}", + "ros-gz-image-${postfix_job_str}"] + + if (projects_to_blockon) { + blockOn(projects_to_blockon) { + blockLevel('GLOBAL') + scanQueueFor('ALL') + } + } + + steps { systemGroovyCommand("""\ build.setDescription( From ade206af524e01d1783f5098fad9b09fb5dba424 Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Wed, 29 Mar 2023 22:23:45 -0700 Subject: [PATCH 308/540] Replace osx_gazebo label with osx We don't need a separate label for gazebo-classic CI jobs anymore. Signed-off-by: Steve Peters --- jenkins-scripts/dsl/gazebo.dsl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jenkins-scripts/dsl/gazebo.dsl b/jenkins-scripts/dsl/gazebo.dsl index 351bcadfc..30ec47a3f 100644 --- a/jenkins-scripts/dsl/gazebo.dsl +++ b/jenkins-scripts/dsl/gazebo.dsl @@ -447,7 +447,7 @@ def gazebo_brew_ci_any_job = job(ci_build_any_job_name_brew) OSRFBrewCompilationAnyGitHub.create(gazebo_brew_ci_any_job, "gazebosim/gazebo-classic") gazebo_brew_ci_any_job.with { - label "osx_gazebo" + label "osx" steps { shell("""\ @@ -482,7 +482,7 @@ all_branches.each { branch -> if (is_watched_by_buildcop(branch)) Globals.extra_emails = Globals.build_cop_email - def osx_label = 'osx_gazebo' + def osx_label = 'osx' def gazebo_brew_ci_job = job("gazebo-ci-${branch}-homebrew-amd64") OSRFBrewCompilation.create(gazebo_brew_ci_job) From dbfabb2b1dfe722dff69e7884703156143b55bac Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Mon, 3 Apr 2023 20:34:57 -0700 Subject: [PATCH 309/540] ABI script: get correct header paths Fixes #890. The ABI script currently assumes that the headers are installed in a subfolder that starts with the ABI_JOB_HEADER_PREFIX variable and a -, which only works for sdformat-9 but requires adjustment for most other projects. This adds a ABI_JOB_HEADER_PREFIX variable to help locate the header folder. Signed-off-by: Steve Peters --- jenkins-scripts/docker/gazebo-abichecker.bash | 1 + jenkins-scripts/docker/ignition-abichecker.bash | 2 ++ jenkins-scripts/docker/lib/generic-abi-base.bash | 8 ++++++-- jenkins-scripts/docker/sdformat-abichecker.bash | 4 ++++ 4 files changed, 13 insertions(+), 2 deletions(-) diff --git a/jenkins-scripts/docker/gazebo-abichecker.bash b/jenkins-scripts/docker/gazebo-abichecker.bash index e7997f188..309ca3a31 100644 --- a/jenkins-scripts/docker/gazebo-abichecker.bash +++ b/jenkins-scripts/docker/gazebo-abichecker.bash @@ -17,6 +17,7 @@ fi . ${SCRIPT_DIR}/lib/_gazebo_version_hook.bash export ABI_JOB_SOFTWARE_NAME="gazebo-classic" +export ABI_JOB_HEADER_PREFIX="gazebo-*" export ABI_JOB_REPOS="stable" export ABI_JOB_PKG_DEPENDENCIES_VAR_NAME="GAZEBO_BASE_DEPENDENCIES" export ABI_JOB_IGNORE_HEADERS="gazebo/GIMPACT gazebo/opcode gazebo/test" diff --git a/jenkins-scripts/docker/ignition-abichecker.bash b/jenkins-scripts/docker/ignition-abichecker.bash index 253579513..046dc53b2 100644 --- a/jenkins-scripts/docker/ignition-abichecker.bash +++ b/jenkins-scripts/docker/ignition-abichecker.bash @@ -32,6 +32,8 @@ export GZ_NAME_PREFIX_MAJOR_VERSION=$(\ ${WORKSPACE}/${ABI_JOB_SOFTWARE_NAME}/CMakeLists.txt) export ${GZ_NAME_PREFIX}_MAJOR_VERSION=${GZ_NAME_PREFIX_MAJOR_VERSION} +export ABI_JOB_HEADER_PREFIX=${ABI_JOB_SOFTWARE_NAME/[ignz]*-/}[0-9]* + # check if NEED_C17_COMPILER should be set if [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-gazebo" ]] || \ [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-physics" ]] || \ diff --git a/jenkins-scripts/docker/lib/generic-abi-base.bash b/jenkins-scripts/docker/lib/generic-abi-base.bash index 3cb5ba650..7c4c551fd 100644 --- a/jenkins-scripts/docker/lib/generic-abi-base.bash +++ b/jenkins-scripts/docker/lib/generic-abi-base.bash @@ -9,6 +9,7 @@ # ABI_JOB_PKG_DEPENDENCIES: (optional) list (space separated) of pkg dependencies # ABI_JOB_PKG_DEPENDENCIES_VAR_NAME: (option) variable in archive to get dependencies from # ABI_JOB_CMAKE_PARAMS: (option) cmake parameters to be pased to cmake configuration +# ABI_JOB_HEADER_PREFIX: (optional) hint for identifying header install prefix # ABI_JOB_IGNORE_HEADERS: (optional) relative (to root project path) list (space separated) # of path headers to ignore # ABI_JOB_EXTRA_GCC_OPTIONS: (optional) inject gcc_options in the descriptor file @@ -33,6 +34,9 @@ fi if [[ -n "${DART_FROM_PKGS_VAR_NAME}" ]]; then eval DART_FROM_PKGS="\$${DART_FROM_PKGS_VAR_NAME}" fi +if [[ -z "${ABI_JOB_HEADER_PREFIX}" ]]; then + eval ABI_JOB_HEADER_PREFIX="\$${ABI_JOB_SOFTWARE_NAME}-*" +fi ABI_CXX_STANDARD=c++11 if [[ "${NEED_C17_COMPILER}" == "true" ]]; then @@ -68,7 +72,7 @@ cmake ${ABI_JOB_CMAKE_PARAMS} \\ /tmp/${ABI_JOB_SOFTWARE_NAME} make -j${MAKE_JOBS} sudo make install -DEST_DIR=\$(find /usr/local/destination_branch/include -name ${ABI_JOB_SOFTWARE_NAME}-* -type d | sed -e 's:.*/::') +DEST_DIR=\$(find /usr/local/destination_branch/include -name ${ABI_JOB_HEADER_PREFIX} -type d | sed -e 's:.*/include/::') echo '# END SECTION' echo '# BEGIN SECTION: compile and install branch: ${SRC_BRANCH}' @@ -89,7 +93,7 @@ cmake ${ABI_JOB_CMAKE_PARAMS} \\ /tmp/${ABI_JOB_SOFTWARE_NAME} make -j${MAKE_JOBS} sudo make install -SRC_DIR=\$(find /usr/local/source_branch/include -name ${ABI_JOB_SOFTWARE_NAME}-* -type d | sed -e 's:.*/::') +SRC_DIR=\$(find /usr/local/source_branch/include -name ${ABI_JOB_HEADER_PREFIX} -type d | sed -e 's:.*/include/::') echo '# END SECTION' echo '# BEGIN SECTION: install the ABI checker' diff --git a/jenkins-scripts/docker/sdformat-abichecker.bash b/jenkins-scripts/docker/sdformat-abichecker.bash index 1bcd61f83..ae53713aa 100644 --- a/jenkins-scripts/docker/sdformat-abichecker.bash +++ b/jenkins-scripts/docker/sdformat-abichecker.bash @@ -23,6 +23,10 @@ if [[ ${SDFORMAT_MAJOR_VERSION} -ge 8 ]]; then export NEED_C17_COMPILER=true fi +if [[ ${SDFORMAT_MAJOR_VERSION} -ge 10 ]]; then + export ABI_JOB_HEADER_PREFIX=sdformat[0-9]* +fi + # default to use stable repos export ABI_JOB_REPOS="stable" From 0e33f7c3d4fad12fca257057bb6bcd69803bda9a Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Mon, 3 Apr 2023 21:56:34 -0700 Subject: [PATCH 310/540] Add documentation and logic for gz-sim Signed-off-by: Steve Peters --- .../docker/ignition-abichecker.bash | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/jenkins-scripts/docker/ignition-abichecker.bash b/jenkins-scripts/docker/ignition-abichecker.bash index 046dc53b2..66b24559a 100644 --- a/jenkins-scripts/docker/ignition-abichecker.bash +++ b/jenkins-scripts/docker/ignition-abichecker.bash @@ -32,7 +32,24 @@ export GZ_NAME_PREFIX_MAJOR_VERSION=$(\ ${WORKSPACE}/${ABI_JOB_SOFTWARE_NAME}/CMakeLists.txt) export ${GZ_NAME_PREFIX}_MAJOR_VERSION=${GZ_NAME_PREFIX_MAJOR_VERSION} -export ABI_JOB_HEADER_PREFIX=${ABI_JOB_SOFTWARE_NAME/[ignz]*-/}[0-9]* +# Set the ABI_JOB_HEADER_PREFIX variable to help find the header locations. +# +# Since gz-cmake include paths are like the following: +# /include/ignition/gazebo6/* +# /include/ignition/math6/* +# /include/gz/math7/* +# /include/gz/sim7/* +# +# Set ABI_JOB_HEADER_PREFIX to match the last subfolder: +# gazebo6, math6, math7, sim7 +if [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-gazebo" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -ge 7 ]] +then + # special case for gz-sim7+ + export ABI_JOB_HEADER_PREFIX=sim[0-9]* +else + # otherwise, strip the ign- or gz- prefix and append "[0-9]*" + export ABI_JOB_HEADER_PREFIX=${ABI_JOB_SOFTWARE_NAME/[ignz]*-/}[0-9]* +fi # check if NEED_C17_COMPILER should be set if [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-gazebo" ]] || \ From ba96688ff58e796593751101486d1352a8ecaca9 Mon Sep 17 00:00:00 2001 From: "Addisu Z. Taddese" Date: Wed, 5 Apr 2023 11:40:14 -0500 Subject: [PATCH 311/540] Add pybind11 dependency to gz-math Signed-off-by: Addisu Z. Taddese --- jenkins-scripts/ign_math-default-devel-windows-amd64.bat | 1 + 1 file changed, 1 insertion(+) diff --git a/jenkins-scripts/ign_math-default-devel-windows-amd64.bat b/jenkins-scripts/ign_math-default-devel-windows-amd64.bat index 0c383bf0a..44a8d2086 100644 --- a/jenkins-scripts/ign_math-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_math-default-devel-windows-amd64.bat @@ -3,6 +3,7 @@ set SCRIPT_DIR=%~dp0 set VCS_DIRECTORY=gz-math set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true +set DEPEN_PKGS=pybind11 set COLCON_PACKAGE=gz-math set COLCON_AUTO_MAJOR_VERSION=true From b28ea1925d9ad5997c13df13f7923d2bf8770e93 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 11 Apr 2023 13:14:32 +0200 Subject: [PATCH 312/540] DSL: ignore nodes labeled as test-instance Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/_configs_/Globals.groovy | 4 ++++ jenkins-scripts/dsl/brew_release.dsl | 4 ++-- jenkins-scripts/dsl/core.dsl | 4 ++-- jenkins-scripts/dsl/debian.dsl | 2 +- jenkins-scripts/dsl/extra.dsl | 4 ++-- jenkins-scripts/dsl/gazebo.dsl | 22 ++++++++++---------- jenkins-scripts/dsl/gazebo_ros_pkgs.dsl | 2 +- jenkins-scripts/dsl/gzdev.dsl | 2 +- jenkins-scripts/dsl/ignition.dsl | 8 +++---- jenkins-scripts/dsl/ignition_collection.dsl | 6 +++--- jenkins-scripts/dsl/ros_gz_bridge.dsl | 2 +- 11 files changed, 32 insertions(+), 28 deletions(-) diff --git a/jenkins-scripts/dsl/_configs_/Globals.groovy b/jenkins-scripts/dsl/_configs_/Globals.groovy index 58cbfcd19..616055f19 100644 --- a/jenkins-scripts/dsl/_configs_/Globals.groovy +++ b/jenkins-scripts/dsl/_configs_/Globals.groovy @@ -119,4 +119,8 @@ class Globals static String get_ros2_development_distro() { return 'rolling' } + + static String nontest_label(String original_label) { + return "(${original_label}) && !test-instance" + } } diff --git a/jenkins-scripts/dsl/brew_release.dsl b/jenkins-scripts/dsl/brew_release.dsl index 90bd02f8b..5adb36bd0 100644 --- a/jenkins-scripts/dsl/brew_release.dsl +++ b/jenkins-scripts/dsl/brew_release.dsl @@ -59,7 +59,7 @@ release_job.with String PR_URL_export_file_name = 'pull_request_created.properties' String PR_URL_export_file = '${WORKSPACE}/' + PR_URL_export_file_name - label "master" + label Globals.nontest_label("master") wrappers { preBuildCleanup() @@ -246,7 +246,7 @@ GenericRemoteToken.create(bottle_job_hash_updater) include_common_params(bottle_job_hash_updater) bottle_job_hash_updater.with { - label "master" + label Globals.nontest_label("master") wrappers { diff --git a/jenkins-scripts/dsl/core.dsl b/jenkins-scripts/dsl/core.dsl index f63f32238..75f82a1d8 100644 --- a/jenkins-scripts/dsl/core.dsl +++ b/jenkins-scripts/dsl/core.dsl @@ -44,7 +44,7 @@ testing_vcpkg_job.with steps { - label "win_testing" + label Globals.nontest_label("win_testing") batchFile("""\ call "%WORKSPACE%/scripts/jenkins-scripts/vcpkg-bootstrap.bat" || exit /B %errorlevel% @@ -58,7 +58,7 @@ def reprepro = job("reprepro_importer") OSRFUNIXBase.create(reprepro) reprepro.with { - label "packages" + label Globals.nontest_label("packages") parameters { diff --git a/jenkins-scripts/dsl/debian.dsl b/jenkins-scripts/dsl/debian.dsl index 394ef7f83..bd87834ab 100644 --- a/jenkins-scripts/dsl/debian.dsl +++ b/jenkins-scripts/dsl/debian.dsl @@ -89,7 +89,7 @@ OSRFLinuxBase.create(ratt_pkg_job) ratt_pkg_job.with { // use only the most powerful nodes - label "large-memory" + label Globals.nontest_label("large-memory") parameters { diff --git a/jenkins-scripts/dsl/extra.dsl b/jenkins-scripts/dsl/extra.dsl index 0b95f6a33..59d6feeeb 100644 --- a/jenkins-scripts/dsl/extra.dsl +++ b/jenkins-scripts/dsl/extra.dsl @@ -23,7 +23,7 @@ release_repo_debbuilds.each { software -> build_pkg_job.with { // use only the most powerful nodes - label "large-memory" + label Globals.nontest_label("large-memory") steps { shell("""\ @@ -43,7 +43,7 @@ gbp_repo_debbuilds.each { software -> build_pkg_job.with { // use only the most powerful nodes - label "large-memory" + label Globals.nontest_label("large-memory") parameters { diff --git a/jenkins-scripts/dsl/gazebo.dsl b/jenkins-scripts/dsl/gazebo.dsl index 30ec47a3f..e71f80c07 100644 --- a/jenkins-scripts/dsl/gazebo.dsl +++ b/jenkins-scripts/dsl/gazebo.dsl @@ -57,7 +57,7 @@ void generate_install_job(Job job, gz_branch, distro, arch, use_osrf_repos = fal gzdev_str = "export GZDEV_PROJECT_NAME=${gz_branch}" // Need gpu for running the runtime test - label "gpu-reliable" + label Globals.nontest_label("gpu-reliable") steps { shell("""\ @@ -91,7 +91,7 @@ abi_distro.each { distro -> GenericAnyJobGitHub.create(abi_job, 'gazebosim/gazebo-classic', gazebo_supported_branches) abi_job.with { - label "large-memory" + label Globals.nontest_label("large-memory") steps { shell("""\ @@ -122,7 +122,7 @@ ci_distro.each { distro -> OSRFLinuxCompilationAnyGitHub.create(gazebo_ci_any_job, "gazebosim/gazebo-classic") gazebo_ci_any_job.with { - label "gpu-reliable && large-memory" + label Globals.nontest_label("gpu-reliable && large-memory") steps { @@ -164,7 +164,7 @@ other_supported_distros.each { distro -> // gazebo builds require a powerful node not to take too long and // block backup for hours - label "gpu-reliable && large-memory" + label Globals.nontest_label("gpu-reliable && large-memory") triggers { scm('@daily') @@ -205,7 +205,7 @@ ci_distro.each { distro -> stringParam('IGN_TRANSPORT_BRANCH', 'main', 'ignition transport branch to use') } - label "gpu-reliable" + label Globals.nontest_label("gpu-reliable") steps { shell("""\ @@ -242,7 +242,7 @@ gazebo_supported_branches.each { branch -> gazebo_ci_job.with { - label "gpu-reliable && large-memory" + label Globals.nontest_label("gpu-reliable && large-memory") triggers { scm('@daily') @@ -276,7 +276,7 @@ ci_distro.each { distro -> gazebo_ci_job.with { - label "gpu-reliable && large-memory" + label Globals.nontest_label("gpu-reliable && large-memory") triggers { scm('@weekly') @@ -309,7 +309,7 @@ ci_distro.each { distro -> scm('@daily') } - label "gpu-reliable && large-memory" + label Globals.nontest_label("gpu-reliable && large-memory") // Problem with the compilation of Gazebo under bullseyes // See: https://github.com/gazebo-tooling/release-tools/issues/129 @@ -341,7 +341,7 @@ ci_distro.each { distro -> gazebo_ci_job.with { - label "gpu-reliable && large-memory" + label Globals.nontest_label("gpu-reliable && large-memory") triggers { scm('@daily') @@ -376,7 +376,7 @@ all_supported_distros.each { distro -> cron(Globals.CRON_EVERY_THREE_DAYS) } - label "gpu-reliable" + label Globals.nontest_label("gpu-reliable") steps { shell("""\ @@ -447,7 +447,7 @@ def gazebo_brew_ci_any_job = job(ci_build_any_job_name_brew) OSRFBrewCompilationAnyGitHub.create(gazebo_brew_ci_any_job, "gazebosim/gazebo-classic") gazebo_brew_ci_any_job.with { - label "osx" + label Globals.nontest_label("osx") steps { shell("""\ diff --git a/jenkins-scripts/dsl/gazebo_ros_pkgs.dsl b/jenkins-scripts/dsl/gazebo_ros_pkgs.dsl index f30a3cb07..ac94c6d03 100644 --- a/jenkins-scripts/dsl/gazebo_ros_pkgs.dsl +++ b/jenkins-scripts/dsl/gazebo_ros_pkgs.dsl @@ -115,7 +115,7 @@ void include_common_params(Job gazebo_ros_pkgs_job, ros2_str = "export ROS2=false" } - label "gpu-reliable" + label Globals.nontest_label("gpu-reliable") steps { shell("""\ diff --git a/jenkins-scripts/dsl/gzdev.dsl b/jenkins-scripts/dsl/gzdev.dsl index e76b2fef3..1137eac9b 100644 --- a/jenkins-scripts/dsl/gzdev.dsl +++ b/jenkins-scripts/dsl/gzdev.dsl @@ -55,7 +55,7 @@ supported_distros.each { distro -> gzdev_any_job.with { // use only the most powerful nodes - label "large-memory" + label Globals.nontest_label("large-memory") steps { shell("""\ diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index f5d7cca3c..affb43c57 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -308,7 +308,7 @@ void include_gpu_label_if_needed(Job job, String gz_software_name) gz_gpu.each { gz_each -> if (gz_software_name == gz_each) { - label "gpu-reliable" + label Globals.nontest_label("gpu-reliable") // unstable build if missing valid gpu display publishers { @@ -373,7 +373,7 @@ gz_software.each { gz_sw -> extra_str="" if (gz_sw == 'physics') { - label "huge-memory" + label Globals.nontest_label("huge-memory") // on ARM native nodes in buildfarm we need to restrict to 1 the // compilation threads to avoid OOM killer extra_str += '\nif [ $(uname -m) = "aarch64" ]; then export MAKE_JOBS=1; fi' @@ -421,7 +421,7 @@ gz_software.each { gz_sw -> gz_ci_any_job.with { if (gz_sw == 'physics') - label "huge-memory" + label Globals.nontest_label("huge-memory") steps { @@ -529,7 +529,7 @@ void generate_ci_job(gz_ci_job, gz_sw, branch, distro, arch, gz_ci_job.with { if (gz_sw == 'physics') - label "huge-memory" + label Globals.nontest_label("huge-memory") steps { shell("""\ diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index c2722c01e..abfea3211 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -361,7 +361,7 @@ void generate_install_job(prefix, gz_collection_name, distro, arch) def dev_package = "${prefix}-${gz_collection_name}" def job_name = 'ign_launch-install-test-job.bash' - label "gpu-reliable" + label Globals.nontest_label("gpu-reliable") steps { shell("""\ @@ -425,7 +425,7 @@ gz_collections.each { gz_collection -> } // Designed to be run manually. No triggers. - label "gpu-reliable" + label Globals.nontest_label("gpu-reliable") steps { systemGroovyCommand("""\ @@ -599,7 +599,7 @@ OSRFUNIXBase.create(nightly_scheduler_job) nightly_scheduler_job.with { - label "master" + label Globals.nontest_label("master") parameters { diff --git a/jenkins-scripts/dsl/ros_gz_bridge.dsl b/jenkins-scripts/dsl/ros_gz_bridge.dsl index c90ea2cd9..c20e53ad8 100644 --- a/jenkins-scripts/dsl/ros_gz_bridge.dsl +++ b/jenkins-scripts/dsl/ros_gz_bridge.dsl @@ -156,7 +156,7 @@ install_test_job.with } // Designed to be run manually. No triggers. - label "gpu-reliable" + label Globals.nontest_label("gpu-reliable") steps { systemGroovyCommand("""\ From c17bc1546b6930437d9a01e1ec65f494d1d6ee5c Mon Sep 17 00:00:00 2001 From: claraberendsen Date: Wed, 12 Apr 2023 19:52:20 -0400 Subject: [PATCH 313/540] (feat): refactor docker cleanup to use prune if available --- .../docker/lib/boilerplate_prepare.sh | 31 ++------ jenkins-scripts/docker/lib/docker_cleanup.sh | 73 +++++++++++++++++++ 2 files changed, 78 insertions(+), 26 deletions(-) mode change 100644 => 100755 jenkins-scripts/docker/lib/boilerplate_prepare.sh create mode 100755 jenkins-scripts/docker/lib/docker_cleanup.sh diff --git a/jenkins-scripts/docker/lib/boilerplate_prepare.sh b/jenkins-scripts/docker/lib/boilerplate_prepare.sh old mode 100644 new mode 100755 index 81510e945..dccf34355 --- a/jenkins-scripts/docker/lib/boilerplate_prepare.sh +++ b/jenkins-scripts/docker/lib/boilerplate_prepare.sh @@ -18,13 +18,13 @@ fi if [[ -z ${DO_NOT_CHECK_DOCKER_DISK_USAGE} ]]; then # get the mount point of the docker directory, not always / docker_device=$(df '/var/lib/docker' | awk '{ print $1 }' | tail -n 1) + # in seconds: 5 days = 432000s PERCENT_DISK_USED=$(df -h ${docker_device} | grep ${docker_device} | sed 's:.* \([0-9]*\)%.*:\1:') if [[ $PERCENT_DISK_USED -gt 70 ]]; then echo "Space left is low: ${PERCENT_DISK_USED}% used" echo "Run docker cleaner !!" - wget https://raw.githubusercontent.com/spotify/docker-gc/master/docker-gc - sudo bash -c "GRACE_PERIOD_SECONDS=432000 bash docker-gc" + source ${SCRIPT_DIR}/lib/docker_cleanup.sh low fi # if not enough, run again with 1 day = 86400s @@ -32,8 +32,7 @@ if [[ -z ${DO_NOT_CHECK_DOCKER_DISK_USAGE} ]]; then if [[ $PERCENT_DISK_USED -gt 80 ]]; then echo "Space left is still low: ${PERCENT_DISK_USED}% used" echo "Run docker cleaner !!" - wget https://raw.githubusercontent.com/spotify/docker-gc/master/docker-gc - sudo bash -c "GRACE_PERIOD_SECONDS=86400 bash docker-gc" + source ${SCRIPT_DIR}/lib/docker_cleanup.sh medium fi # if not enough, kill the whole cache @@ -41,35 +40,15 @@ if [[ -z ${DO_NOT_CHECK_DOCKER_DISK_USAGE} ]]; then if [[ $PERCENT_DISK_USED -gt 85 ]]; then echo "Space left is low again: ${PERCENT_DISK_USED}% used" echo "Kill the whole docker cache !!" - # use system prune if available - docker_version="$(sudo docker version --format '{{.Server.APIVersion}}') > 1.25" - if [[ $(echo $docker_version | bc -l) ]]; then - sudo docker system prune --all -f - else - [[ -n $(sudo docker ps -q) ]] && sudo docker kill $(sudo docker ps -q) || true - [[ -n $(sudo docker images -a -q) ]] && sudo docker rmi $(sudo docker images -a -q) || true - fi + source ${SCRIPT_DIR}/lib/docker_cleanup.sh high fi # if not enough, try to clean up build/ directories PERCENT_DISK_USED=$(df -h ${docker_device} | grep ${docker_device} | sed 's:.* \([0-9]*\)%.*:\1:') if [[ $PERCENT_DISK_USED -gt 85 ]]; then - MAX_SIZE_FOR_BUILD_DIRS="5G" - CMD_FIND_BIG_DIRS=(find ${HOME}/workspace -name build -exec du -h -d 0 -t ${MAX_SIZE_FOR_BUILD_DIRS} {} \;) echo "Space left is low again: ${PERCENT_DISK_USED}% used" echo "Clean up the whole cache was not enough. Look for build/ directories bigger than ${MAX_SIZE_FOR_BUILD_DIRS}:" - # run command twice to avoid parsing. First for information proposes - ${CMD_FIND_BIG_DIRS[@]} - for d in $("${CMD_FIND_BIG_DIRS[@]}" | awk '{ print $2 }'); do - # safe checks on paths - if [[ $d == ${d/$HOME} ]] || [[ $d == ${d/build} ]]; then - echo "System is trying to delete a path $d outside Jenkins home: $HOME with subdir build/" - exit -1 - fi - # avoid to rm -fr without a path. The ugly trick should leave d as it - # is, in case of a bug in code it will not remove anything at random - sudo rm -fr ${d/build}build - done + source ${SCRIPT_DIR}/lib/docker_cleanup.sh extreme fi fi diff --git a/jenkins-scripts/docker/lib/docker_cleanup.sh b/jenkins-scripts/docker/lib/docker_cleanup.sh new file mode 100755 index 000000000..32c39da22 --- /dev/null +++ b/jenkins-scripts/docker/lib/docker_cleanup.sh @@ -0,0 +1,73 @@ +#function that cleans docker +# Receives parameters levels + +docker_version="$(sudo docker version --format '{{.Server.APIVersion}}')" + if [[ $(echo "$docker_version > 1.25" | bc -l) ]]; then + echo "Running version of docker $(sudo docker version --format '{{.Server.APIVersion}}') with prune" + case $1 in + # preferred utilizing each object prune to not affect the build cache in low and medium + low) + echo "Cleaning dangling images created more than 5 days ago ..." + sudo bash -c "docker image prune -f --filter "until=35h"" + echo "Cleaning stopped containers created more than 5 days ago ..." + sudo bash -c "docker container prune -f --filter "until=35h" " + ;; + medium) + echo "Cleaning dangling images created more than 1 day ago ..." + sudo bash -c "docker image prune -f --filter "until=24h"" + echo "Cleaning stopped containers created more than 1 day ago ..." + sudo bash -c "docker container prune -f --filter "until=24h"" + ;; + high) + echo "Cleaning all unusued images, containers, networks and cache ..." + sudo docker system prune --all -f + ;; + extreme) + MAX_SIZE_FOR_BUILD_DIRS="5G" + echo "Removing all cache " + sudo docker builder prune -f --keep-storage $MAX_SIZE_FOR_BUILD_DIRS + ;; + *) + echo "Did not run docker cleaner, level $1 not recognized (accepted values low | medium | high | extreme!! " + ;; + esac + else + + echo "WARNING - Running version of docker $docker_version without prune, utilizing deprecated docker_gc as cleanup method!" + case $1 in + low) + wget https://raw.githubusercontent.com/spotify/docker-gc/master/docker-gc + sudo bash -c "GRACE_PERIOD_SECONDS=432000 bash docker-gc" + ;; + medium) + wget https://raw.githubusercontent.com/spotify/docker-gc/master/docker-gc + sudo bash -c "GRACE_PERIOD_SECONDS=86400 bash docker-gc" + ;; + + high) + [[ -n $(sudo docker ps -q) ]] && sudo docker kill $(sudo docker ps -q) || true + [[ -n $(sudo docker images -a -q) ]] && sudo docker rmi $(sudo docker images -a -q) || true + ;; + extreme) + MAX_SIZE_FOR_BUILD_DIRS="5G" + CMD_FIND_BIG_DIRS=(find ${HOME}/workspace -name build -exec du -h -d 0 -t ${MAX_SIZE_FOR_BUILD_DIRS} {} \;) + echo "Clean up the whole cache was not enough. Look for build/ directories bigger than ${MAX_SIZE_FOR_BUILD_DIRS}:" + # run command twice to avoid parsing. First for information proposes + echo ${CMD_FIND_BIG_DIRS[@]} + for d in $("${CMD_FIND_BIG_DIRS[@]}" | awk '{ print $2 }'); do + # safe checks on paths + if [[ $d == ${d/$HOME} ]] || [[ $d == ${d/build} ]]; then + echo "System is trying to delete a path $d outside Jenkins home: $HOME with subdir build/" + exit -1 + fi + # avoid to rm -fr without a path. The ugly trick should leave d as it + # is, in case of a bug in code it will not remove anything at random + sudo rm -fr ${d/build}build + done + ;; + *) + echo "Did not run docker cleaner, level $1 not recognized (accepted values low | medium | high | extreme!! " + ;; + esac + fi + From b645ddc5476d12be539969da9769563b4bda78b9 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 12 Apr 2023 12:55:51 +0200 Subject: [PATCH 314/540] Refactor install of vcpkg packages Signed-off-by: Jose Luis Rivero --- jenkins-scripts/lib/windows_env_vars.bat | 3 +++ jenkins-scripts/lib/windows_library.bat | 22 ++++++++++++++++++++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/jenkins-scripts/lib/windows_env_vars.bat b/jenkins-scripts/lib/windows_env_vars.bat index 5d8b1f1a2..4242ac1bc 100644 --- a/jenkins-scripts/lib/windows_env_vars.bat +++ b/jenkins-scripts/lib/windows_env_vars.bat @@ -17,4 +17,7 @@ if NOT DEFINED VCPKG_SNAPSHOT ( :: see https://github.com/microsoft/vcpkg/releases set VCPKG_SNAPSHOT=2022.02.23 ) +:: Set of common gz dependencies expected up to Garden +set VCPKG_DEPENDENCIES_LEGACY=assimp boost-core bullet3 ccd console-bridge cppzmq cuda curl dlfcn-win32 eigen3 fcl ffmpeg freeimage gdal gflags glib gts jsoncpp libyaml libzip ogre ogre2 ogre22 openssl protobuf pybind11 qt5 qt5-winextras qwt sqlite3 tinyxml2 zeromq + goto :EOF diff --git a/jenkins-scripts/lib/windows_library.bat b/jenkins-scripts/lib/windows_library.bat index f66fed21e..45023d5b1 100644 --- a/jenkins-scripts/lib/windows_library.bat +++ b/jenkins-scripts/lib/windows_library.bat @@ -268,8 +268,7 @@ if NOT %VCPKG_HEAD% == %VCPKG_TAG% ( goto :EOF :: ################################## -:install_vcpkg_package -:: arg1: package to install +:_prepare_vcpkg_to_install set LIB_DIR=%~dp0 call %LIB_DIR%\windows_env_vars.bat || goto :error call %win_lib% :check_vcpkg_snapshot || goto :error @@ -278,7 +277,15 @@ pushd . cd %VCPKG_OSRF_DIR% git pull origin master || goto :error popd +goto :EOF +:: ################################## +:_install_and_upgrade_vcpkg_package +:: arg1: package to install +if [ %1 ] == [] ( + echo "_install_and_upgrade_vcpkg_package called with no argument" + goto :error +) %VCPKG_CMD% install --recurse "%1" --overlay-ports="%VCPKG_OSRF_DIR%" :: vcpkg does not upgrade installed packages using the install command :: since most of the packages are coming from a frozen snapshot, it is @@ -286,6 +293,17 @@ popd %VCPKG_CMD% upgrade "%1" --no-dry-run --overlay-ports="%VCPKG_OSRF_DIR%" goto :EOF +:: ################################## +:install_vcpkg_package +:: arg1: package to install +if [ %1 ] == [] ( + echo "install_vcpkg_package called with no argument" + goto :error +) +call %win_lib% :_prepare_vcpkg_to_install|| goto :error +call %win_lib% :_install_and_upgrade_vcpkg_package "%1" || goto :error +goto :EOF + :: ################################## :remove_vcpkg_package :: arg1: package to install From ac59b1d0030410c175c44b4a8249c3cb2d450622 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 12 Apr 2023 12:56:05 +0200 Subject: [PATCH 315/540] Add setup_vcpkg_all_dependencies Signed-off-by: Jose Luis Rivero --- jenkins-scripts/lib/windows_library.bat | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/jenkins-scripts/lib/windows_library.bat b/jenkins-scripts/lib/windows_library.bat index 45023d5b1..ee1e82f1a 100644 --- a/jenkins-scripts/lib/windows_library.bat +++ b/jenkins-scripts/lib/windows_library.bat @@ -282,7 +282,7 @@ goto :EOF :: ################################## :_install_and_upgrade_vcpkg_package :: arg1: package to install -if [ %1 ] == [] ( +if [%1] == [] ( echo "_install_and_upgrade_vcpkg_package called with no argument" goto :error ) @@ -296,7 +296,7 @@ goto :EOF :: ################################## :install_vcpkg_package :: arg1: package to install -if [ %1 ] == [] ( +if [%1] == [] ( echo "install_vcpkg_package called with no argument" goto :error ) @@ -323,11 +323,17 @@ goto :EOF %VCPKG_CMD% integrate remove || goto :error goto :EOF - -:: copy port to the official tree -xcopy %VCPKG_OSRF_DIR%\%PKG% %PORT_DIR% /s /i /e || goto :error - -call %win_lib% :install_vcpkg_package %1 || goto :error +:: ################################## +:setup_vcpkg_all_dependencies +set LIB_DIR=%~dp0 +call %LIB_DIR%\windows_env_vars.bat || goto :error +call %win_lib% :enable_vcpkg_integration || goto :error +call %win_lib% :_prepare_vcpkg_to_install|| goto :error +for %%p in (%VCPKG_DEPENDENCIES_LEGACY%) do ( + echo # BEGIN SECTION: install external dependency %%p + call %win_lib% :_install_and_upgrade_vcpkg_package %%p || goto :error + echo # END SECTION +) goto :EOF :: ################################## From 82ba2f0e35d5d02400ff62117f9476dc585cbe06 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Mon, 10 Apr 2023 15:37:56 +0200 Subject: [PATCH 316/540] Remove previous installed packages in vcpkg Signed-off-by: Jose Luis Rivero --- .../lib/colcon-default-devel-windows.bat | 3 +++ jenkins-scripts/lib/windows_env_vars.bat | 1 + jenkins-scripts/lib/windows_library.bat | 13 +++++++++++++ 3 files changed, 17 insertions(+) diff --git a/jenkins-scripts/lib/colcon-default-devel-windows.bat b/jenkins-scripts/lib/colcon-default-devel-windows.bat index 9a47b3ab3..57292ee98 100644 --- a/jenkins-scripts/lib/colcon-default-devel-windows.bat +++ b/jenkins-scripts/lib/colcon-default-devel-windows.bat @@ -36,6 +36,9 @@ if "%COLCON_AUTO_MAJOR_VERSION%" == "true" ( echo "MAJOR_VERSION detected: !PKG_MAJOR_VERSION!" ) +:: remove all previous packages installed by vcpkg +call %win_lib% :remove_vcpkg_installation || goto :error + :: Check if package is in colcon workspace echo # BEGIN SECTION: Update package !COLCON_PACKAGE! from gz to ignition echo Packages in workspace: diff --git a/jenkins-scripts/lib/windows_env_vars.bat b/jenkins-scripts/lib/windows_env_vars.bat index 4242ac1bc..c3d54a152 100644 --- a/jenkins-scripts/lib/windows_env_vars.bat +++ b/jenkins-scripts/lib/windows_env_vars.bat @@ -11,6 +11,7 @@ if DEFINED MAKE_JOBS ( set VCPKG_MAX_CONCURRENCY=%MAKE_JOBS% ) set VCPKG_OSRF_DIR=%VCPKG_DIR%\osrf_vcpkg_ports +set VCPKG_INSTALLED_FILES_DIR=%VCPKG_DIR%\installed set VCPKG_CMD=%VCPKG_DIR%\vcpkg.exe set VCPKG_CMAKE_TOOLCHAIN_FILE=%VCPKG_DIR%/scripts/buildsystems/vcpkg.cmake if NOT DEFINED VCPKG_SNAPSHOT ( diff --git a/jenkins-scripts/lib/windows_library.bat b/jenkins-scripts/lib/windows_library.bat index ee1e82f1a..73e929ea0 100644 --- a/jenkins-scripts/lib/windows_library.bat +++ b/jenkins-scripts/lib/windows_library.bat @@ -267,6 +267,19 @@ if NOT %VCPKG_HEAD% == %VCPKG_TAG% ( ) goto :EOF +:: ################################## +:remove_vcpkg_installation +:: remove the installed directory to simulate all packages removal +:: vcpkg cli does not support the operation +set LIB_DIR="%~dp0" +call %LIB_DIR%\windows_env_vars.bat || goto :error +if [%VCPKG_INSTALLED_FILES_DIR%]==[] ( + echo VCPKG_INSTALLED_FILES_DIR seems empty, refuse to delete + goto :error +) +del /s /f /q %VCPKG_INSTALLED_FILES_DIR% +goto :EOF + :: ################################## :_prepare_vcpkg_to_install set LIB_DIR=%~dp0 From 66d5b453680ae8f9097815d3f904823133e1c6f4 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 12 Apr 2023 13:00:03 +0200 Subject: [PATCH 317/540] Change pkgs installation to use common dependencies Signed-off-by: Jose Luis Rivero --- jenkins-scripts/lib/colcon-default-devel-windows.bat | 10 ++++------ jenkins-scripts/lib/windows_library.bat | 2 -- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/jenkins-scripts/lib/colcon-default-devel-windows.bat b/jenkins-scripts/lib/colcon-default-devel-windows.bat index 57292ee98..634e88912 100644 --- a/jenkins-scripts/lib/colcon-default-devel-windows.bat +++ b/jenkins-scripts/lib/colcon-default-devel-windows.bat @@ -110,12 +110,10 @@ if exist %LOCAL_WS_SOFTWARE_DIR% ( rmdir /q /s %LOCAL_WS_SOFTWARE_DIR% ) xcopy %WORKSPACE%\%VCS_DIRECTORY% %LOCAL_WS_SOFTWARE_DIR% /s /e /i > xcopy_vcs_directory.log || goto :error echo # END SECTION -for %%p in (%DEPEN_PKGS%) do ( - call %win_lib% :enable_vcpkg_integration || goto :error - echo # BEGIN SECTION: install external dependency %%p - call %win_lib% :install_vcpkg_package %%p || goto :error - echo # END SECTION -) +:: Install all +echo # BEGIN SECTION: install all vcpkg dependencies +call %win_lib% :setup_vcpkg_all_dependencies || goto :error +echo # END SECTION echo # BEGIN SECTION: packages in workspace call %win_lib% :list_workspace_pkgs || goto :error diff --git a/jenkins-scripts/lib/windows_library.bat b/jenkins-scripts/lib/windows_library.bat index 73e929ea0..52901b0f4 100644 --- a/jenkins-scripts/lib/windows_library.bat +++ b/jenkins-scripts/lib/windows_library.bat @@ -343,9 +343,7 @@ call %LIB_DIR%\windows_env_vars.bat || goto :error call %win_lib% :enable_vcpkg_integration || goto :error call %win_lib% :_prepare_vcpkg_to_install|| goto :error for %%p in (%VCPKG_DEPENDENCIES_LEGACY%) do ( - echo # BEGIN SECTION: install external dependency %%p call %win_lib% :_install_and_upgrade_vcpkg_package %%p || goto :error - echo # END SECTION ) goto :EOF From 2dfba38f0dc7838aed350cb5f9bb05cb1b312c3c Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Mon, 10 Apr 2023 20:52:26 +0200 Subject: [PATCH 318/540] Use Temp for permissions problems Signed-off-by: Jose Luis Rivero --- jenkins-scripts/lib/windows_library.bat | 2 ++ 1 file changed, 2 insertions(+) diff --git a/jenkins-scripts/lib/windows_library.bat b/jenkins-scripts/lib/windows_library.bat index 52901b0f4..51d1c6cf6 100644 --- a/jenkins-scripts/lib/windows_library.bat +++ b/jenkins-scripts/lib/windows_library.bat @@ -299,6 +299,8 @@ if [%1] == [] ( echo "_install_and_upgrade_vcpkg_package called with no argument" goto :error ) +:: workaround on permissions problems for default VCPKG_DEFAULT_BINARY_CACHE +set VCPKG_DEFAULT_BINARY_CACHE=C:\Windows\Temp\vcpkg %VCPKG_CMD% install --recurse "%1" --overlay-ports="%VCPKG_OSRF_DIR%" :: vcpkg does not upgrade installed packages using the install command :: since most of the packages are coming from a frozen snapshot, it is From f7f4fe80caff26d6df8e92a82541bb4fcd0b3dd3 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 12 Apr 2023 13:09:44 +0200 Subject: [PATCH 319/540] Remove DEPEN_PKGS references to vcpkg Signed-off-by: Jose Luis Rivero --- .../ign_common-default-devel-windows-amd64.bat | 2 -- .../ign_fuel-tools-default-devel-windows-amd64.bat | 1 - .../ign_gazebo-default-devel-windows-amd64.bat | 7 ------- .../ign_gui-default-devel-windows-amd64.bat | 8 -------- .../ign_launch-default-devel-windows-amd64.bat | 6 ------ .../ign_math-default-devel-windows-amd64.bat | 1 - .../ign_msgs-default-devel-windows-amd64.bat | 1 - .../ign_physics-default-devel-windows-amd64.bat | 2 -- .../ign_plugin-default-devel-windows-amd64.bat | 1 - .../ign_rendering-default-devel-windows-amd64.bat | 11 ----------- .../ign_sensors-default-devel-windows-amd64.bat | 7 ------- .../ign_transport-default-devel-windows-amd64.bat | 1 - .../ign_utils-default-devel-windows-amd64.bat | 7 ++----- jenkins-scripts/lib/colcon-default-devel-windows.bat | 9 +++------ 14 files changed, 5 insertions(+), 59 deletions(-) diff --git a/jenkins-scripts/ign_common-default-devel-windows-amd64.bat b/jenkins-scripts/ign_common-default-devel-windows-amd64.bat index 39f0ecae9..722f705c1 100644 --- a/jenkins-scripts/ign_common-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_common-default-devel-windows-amd64.bat @@ -4,10 +4,8 @@ set VCS_DIRECTORY=gz-common set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true -set DEPEN_PKGS=assimp ffmpeg freeimage gts tinyxml2 for /f %%i in ('python "%SCRIPT_DIR%\tools\detect_cmake_major_version.py" "%WORKSPACE%\%VCS_DIRECTORY%\CMakeLists.txt"') do set GZ_MAJOR_VERSION=%%i if %GZ_MAJOR_VERSION% GEQ 5 ( - set DEPEN_PKGS=%DEPEN_PKGS% gdal ) set COLCON_PACKAGE=gz-common diff --git a/jenkins-scripts/ign_fuel-tools-default-devel-windows-amd64.bat b/jenkins-scripts/ign_fuel-tools-default-devel-windows-amd64.bat index 36c81fee0..a0fe03ff2 100644 --- a/jenkins-scripts/ign_fuel-tools-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_fuel-tools-default-devel-windows-amd64.bat @@ -5,7 +5,6 @@ set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true :: tinyxml2 from msgs -set DEPEN_PKGS=libyaml libzip tinyxml2 openssl protobuf curl set COLCON_PACKAGE=gz-fuel_tools set COLCON_AUTO_MAJOR_VERSION=true diff --git a/jenkins-scripts/ign_gazebo-default-devel-windows-amd64.bat b/jenkins-scripts/ign_gazebo-default-devel-windows-amd64.bat index 2b057d01d..9a803131f 100644 --- a/jenkins-scripts/ign_gazebo-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_gazebo-default-devel-windows-amd64.bat @@ -4,13 +4,6 @@ set SCRIPT_DIR=%~dp0 set VCS_DIRECTORY=gz-sim set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true -:: dlfcn -set DEPEN_PKGS=assimp boost-core ccd cppzmq cuda curl dlfcn-win32 eigen3 fcl ffmpeg freeimage gflags glib gts jsoncpp libyaml libzip ogre ogre2 ogre22 openssl protobuf qt5 qwt tinyxml2 zeromq -for /f %%i in ('python "%SCRIPT_DIR%\tools\detect_cmake_major_version.py" "%WORKSPACE%\%VCS_DIRECTORY%\CMakeLists.txt"') do set GZ_MAJOR_VERSION=%%i -if %GZ_MAJOR_VERSION% GEQ 7 ( - set DEPEN_PKGS=%DEPEN_PKGS% gdal -) -:: This needs to be migrated to DSL to get multi-major versions correctly set COLCON_PACKAGE=gz-sim set COLCON_AUTO_MAJOR_VERSION=true diff --git a/jenkins-scripts/ign_gui-default-devel-windows-amd64.bat b/jenkins-scripts/ign_gui-default-devel-windows-amd64.bat index 31707055f..3590abbac 100644 --- a/jenkins-scripts/ign_gui-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_gui-default-devel-windows-amd64.bat @@ -4,14 +4,6 @@ set SCRIPT_DIR=%~dp0 set VCS_DIRECTORY=gz-gui set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true - -:: ogre2 from vcpkg-ports -set DEPEN_PKGS=qt5 qt5-winextras qwt protobuf tinyxml2 freeimage ogre ogre22 -for /f %%i in ('python "%SCRIPT_DIR%\tools\detect_cmake_major_version.py" "%WORKSPACE%\%VCS_DIRECTORY%\CMakeLists.txt"') do set GZ_MAJOR_VERSION=%%i -if %GZ_MAJOR_VERSION% GEQ 7 ( - set DEPEN_PKGS=%DEPEN_PKGS% gdal -) -:: This needs to be migrated to DSL to get multi-major versions correctly set COLCON_PACKAGE=gz-gui set COLCON_AUTO_MAJOR_VERSION=true diff --git a/jenkins-scripts/ign_launch-default-devel-windows-amd64.bat b/jenkins-scripts/ign_launch-default-devel-windows-amd64.bat index 19b8edbe2..0a7533df7 100644 --- a/jenkins-scripts/ign_launch-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_launch-default-devel-windows-amd64.bat @@ -3,12 +3,6 @@ set SCRIPT_DIR=%~dp0 set VCS_DIRECTORY=gz-launch set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true - -set DEPEN_PKGS=assimp boost-core ccd cppzmq cuda curl dlfcn-win32 eigen3 fcl ffmpeg freeimage gflags glib gts jsoncpp libyaml libzip ogre ogre2 ogre22 openssl protobuf qt5 qwt tinyxml2 zeromq -for /f %%i in ('python "%SCRIPT_DIR%\tools\detect_cmake_major_version.py" "%WORKSPACE%\%VCS_DIRECTORY%\CMakeLists.txt"') do set GZ_MAJOR_VERSION=%%i -if %GZ_MAJOR_VERSION% GEQ 6 ( - set DEPEN_PKGS=%DEPEN_PKGS% gdal -) set COLCON_PACKAGE=gz-launch set COLCON_AUTO_MAJOR_VERSION=true diff --git a/jenkins-scripts/ign_math-default-devel-windows-amd64.bat b/jenkins-scripts/ign_math-default-devel-windows-amd64.bat index 44a8d2086..0c383bf0a 100644 --- a/jenkins-scripts/ign_math-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_math-default-devel-windows-amd64.bat @@ -3,7 +3,6 @@ set SCRIPT_DIR=%~dp0 set VCS_DIRECTORY=gz-math set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true -set DEPEN_PKGS=pybind11 set COLCON_PACKAGE=gz-math set COLCON_AUTO_MAJOR_VERSION=true diff --git a/jenkins-scripts/ign_msgs-default-devel-windows-amd64.bat b/jenkins-scripts/ign_msgs-default-devel-windows-amd64.bat index 2c1387f75..0d93f69db 100644 --- a/jenkins-scripts/ign_msgs-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_msgs-default-devel-windows-amd64.bat @@ -4,7 +4,6 @@ set VCS_DIRECTORY=gz-msgs set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true -set DEPEN_PKGS=protobuf tinyxml2 set COLCON_PACKAGE=gz-msgs set COLCON_AUTO_MAJOR_VERSION=true diff --git a/jenkins-scripts/ign_physics-default-devel-windows-amd64.bat b/jenkins-scripts/ign_physics-default-devel-windows-amd64.bat index 529d41c83..9b3d5de21 100644 --- a/jenkins-scripts/ign_physics-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_physics-default-devel-windows-amd64.bat @@ -4,10 +4,8 @@ set VCS_DIRECTORY=gz-physics set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true -set DEPEN_PKGS=eigen3 tinyxml2 for /f %%i in ('python "%SCRIPT_DIR%\tools\detect_cmake_major_version.py" "%WORKSPACE%\%VCS_DIRECTORY%\CMakeLists.txt"') do set GZ_MAJOR_VERSION=%%i if %GZ_MAJOR_VERSION% GEQ 6 ( - set DEPEN_PKGS=%DEPEN_PKGS% gdal ) :: This needs to be migrated to DSL to get multi-major versions correctly set COLCON_PACKAGE=gz-physics diff --git a/jenkins-scripts/ign_plugin-default-devel-windows-amd64.bat b/jenkins-scripts/ign_plugin-default-devel-windows-amd64.bat index cb06bbcd9..63ad952f0 100644 --- a/jenkins-scripts/ign_plugin-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_plugin-default-devel-windows-amd64.bat @@ -4,7 +4,6 @@ set VCS_DIRECTORY=gz-plugin set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true -set DEPEN_PKGS="dlfcn-win32" set COLCON_PACKAGE=gz-plugin set COLCON_AUTO_MAJOR_VERSION=true diff --git a/jenkins-scripts/ign_rendering-default-devel-windows-amd64.bat b/jenkins-scripts/ign_rendering-default-devel-windows-amd64.bat index c180f5118..00cfbb20e 100644 --- a/jenkins-scripts/ign_rendering-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_rendering-default-devel-windows-amd64.bat @@ -4,17 +4,6 @@ set SCRIPT_DIR=%~dp0 set VCS_DIRECTORY=gz-rendering set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true - -set DEPEN_PKGS=boost-core cuda dlfcn-win32 eigen3 freeimage glib gts ogre ogre2 -for /f %%i in ('python "%SCRIPT_DIR%\tools\detect_cmake_major_version.py" "%WORKSPACE%\%VCS_DIRECTORY%\CMakeLists.txt"') do set GZ_MAJOR_VERSION=%%i -if %GZ_MAJOR_VERSION% GEQ 5 ( - set DEPEN_PKGS=%DEPEN_PKGS% ogre22 -) -if %GZ_MAJOR_VERSION% GEQ 7 ( - set DEPEN_PKGS=%DEPEN_PKGS% gdal -) - -:: This needs to be migrated to DSL to get multi-major versions correctly set COLCON_PACKAGE=gz-rendering set COLCON_AUTO_MAJOR_VERSION=true diff --git a/jenkins-scripts/ign_sensors-default-devel-windows-amd64.bat b/jenkins-scripts/ign_sensors-default-devel-windows-amd64.bat index 206b5ee4e..99528bf07 100644 --- a/jenkins-scripts/ign_sensors-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_sensors-default-devel-windows-amd64.bat @@ -4,13 +4,6 @@ set SCRIPT_DIR=%~dp0 set VCS_DIRECTORY=gz-sensors set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true - -set DEPEN_PKGS=protobuf tinyxml2 -for /f %%i in ('python "%SCRIPT_DIR%\tools\detect_cmake_major_version.py" "%WORKSPACE%\%VCS_DIRECTORY%\CMakeLists.txt"') do set GZ_MAJOR_VERSION=%%i -if %GZ_MAJOR_VERSION% GEQ 7 ( - set DEPEN_PKGS=%DEPEN_PKGS% gdal -) -:: This needs to be migrated to DSL to get multi-major versions correctly set COLCON_PACKAGE=gz-sensors set COLCON_AUTO_MAJOR_VERSION=true diff --git a/jenkins-scripts/ign_transport-default-devel-windows-amd64.bat b/jenkins-scripts/ign_transport-default-devel-windows-amd64.bat index 1f2142f30..fd6baa68d 100644 --- a/jenkins-scripts/ign_transport-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_transport-default-devel-windows-amd64.bat @@ -6,7 +6,6 @@ set VCS_DIRECTORY=gz-transport set PLATFORM_TO_BUILD=amd64 set IGN_CLEAN_WORKSPACE=true -set DEPEN_PKGS=cppzmq dlfcn-win32 protobuf sqlite3 tinyxml2 zeromq set COLCON_PACKAGE=gz-transport set COLCON_AUTO_MAJOR_VERSION=true diff --git a/jenkins-scripts/ign_utils-default-devel-windows-amd64.bat b/jenkins-scripts/ign_utils-default-devel-windows-amd64.bat index 8da64ffcb..a65bdb866 100644 --- a/jenkins-scripts/ign_utils-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_utils-default-devel-windows-amd64.bat @@ -1,14 +1,11 @@ @echo on set SCRIPT_DIR=%~dp0 +:: ign-utils shouldn't have external dependencies + set VCS_DIRECTORY=gz-utils set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true - -:: ign-utils shouldn't have external dependencies -:: set DEPEN_PKGS= - -:: This needs to be migrated to DSL to get multi-major versions correctly set COLCON_PACKAGE=gz-utils set COLCON_AUTO_MAJOR_VERSION=true diff --git a/jenkins-scripts/lib/colcon-default-devel-windows.bat b/jenkins-scripts/lib/colcon-default-devel-windows.bat index 634e88912..926eec0e4 100644 --- a/jenkins-scripts/lib/colcon-default-devel-windows.bat +++ b/jenkins-scripts/lib/colcon-default-devel-windows.bat @@ -7,14 +7,13 @@ :: - COLCON_AUTO_MAJOR_VERSION (default false): auto detect major version from CMakeLists :: - COLCON_PACKAGE_EXTRA_CMAKE_ARGS : (optional) CMake arg to inject into colcon :: - BUILD_TYPE : (default Release) [ Release | Debug ] Build type to use -:: - DEPEN_PKGS : (optional) list of dependencies (separted by spaces) :: - KEEP_WORKSPACE : (optional) true | false. Clean workspace at the end :: - ENABLE_TESTS : (optional) true | false. Do not compile and run tests :: :: Actions :: - Configure the compiler :: - Clean and create the WORKSPACE/ws -:: - Download and unzip the DEPEN_PKGS +:: - Install the binary external dependencies :: - configure, compile and install :: - run tests @@ -36,9 +35,6 @@ if "%COLCON_AUTO_MAJOR_VERSION%" == "true" ( echo "MAJOR_VERSION detected: !PKG_MAJOR_VERSION!" ) -:: remove all previous packages installed by vcpkg -call %win_lib% :remove_vcpkg_installation || goto :error - :: Check if package is in colcon workspace echo # BEGIN SECTION: Update package !COLCON_PACKAGE! from gz to ignition echo Packages in workspace: @@ -110,7 +106,8 @@ if exist %LOCAL_WS_SOFTWARE_DIR% ( rmdir /q /s %LOCAL_WS_SOFTWARE_DIR% ) xcopy %WORKSPACE%\%VCS_DIRECTORY% %LOCAL_WS_SOFTWARE_DIR% /s /e /i > xcopy_vcs_directory.log || goto :error echo # END SECTION -:: Install all +:: Prepare a clean vcpkg environment with external dependencies +call %win_lib% :remove_vcpkg_installation || goto :error echo # BEGIN SECTION: install all vcpkg dependencies call %win_lib% :setup_vcpkg_all_dependencies || goto :error echo # END SECTION From 73040c3d4a190f35d74489e8cbb86310ae8b9670 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 12 Apr 2023 13:24:41 +0200 Subject: [PATCH 320/540] Relocate vcpkg installation at the beginning of the build Signed-off-by: Jose Luis Rivero --- jenkins-scripts/lib/colcon-default-devel-windows.bat | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/jenkins-scripts/lib/colcon-default-devel-windows.bat b/jenkins-scripts/lib/colcon-default-devel-windows.bat index 926eec0e4..abffd61e0 100644 --- a/jenkins-scripts/lib/colcon-default-devel-windows.bat +++ b/jenkins-scripts/lib/colcon-default-devel-windows.bat @@ -83,6 +83,12 @@ echo # BEGIN SECTION: configure the MSVC compiler call %win_lib% :configure_msvc2019_compiler echo # END SECTION +:: Prepare a clean vcpkg environment with external dependencies +call %win_lib% :remove_vcpkg_installation || goto :error +echo # BEGIN SECTION: install all vcpkg dependencies +call %win_lib% :setup_vcpkg_all_dependencies || goto :error +echo # END SECTION + echo # BEGIN SECTION: setup workspace if not defined KEEP_WORKSPACE ( IF exist %LOCAL_WS_BUILD% ( @@ -106,12 +112,6 @@ if exist %LOCAL_WS_SOFTWARE_DIR% ( rmdir /q /s %LOCAL_WS_SOFTWARE_DIR% ) xcopy %WORKSPACE%\%VCS_DIRECTORY% %LOCAL_WS_SOFTWARE_DIR% /s /e /i > xcopy_vcs_directory.log || goto :error echo # END SECTION -:: Prepare a clean vcpkg environment with external dependencies -call %win_lib% :remove_vcpkg_installation || goto :error -echo # BEGIN SECTION: install all vcpkg dependencies -call %win_lib% :setup_vcpkg_all_dependencies || goto :error -echo # END SECTION - echo # BEGIN SECTION: packages in workspace call %win_lib% :list_workspace_pkgs || goto :error echo # END SECTION From bc66abd21628144da220fe4936ccc3acf82c43ff Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 12 Apr 2023 13:32:42 +0200 Subject: [PATCH 321/540] Create cache directory if it does not exists Signed-off-by: Jose Luis Rivero --- jenkins-scripts/lib/windows_library.bat | 1 + 1 file changed, 1 insertion(+) diff --git a/jenkins-scripts/lib/windows_library.bat b/jenkins-scripts/lib/windows_library.bat index 51d1c6cf6..a3505ca71 100644 --- a/jenkins-scripts/lib/windows_library.bat +++ b/jenkins-scripts/lib/windows_library.bat @@ -301,6 +301,7 @@ if [%1] == [] ( ) :: workaround on permissions problems for default VCPKG_DEFAULT_BINARY_CACHE set VCPKG_DEFAULT_BINARY_CACHE=C:\Windows\Temp\vcpkg +if not exist %VCPKG_DEFAULT_BINARY_CACHE% mkdir %VCPKG_DEFAULT_BINARY_CACHE% %VCPKG_CMD% install --recurse "%1" --overlay-ports="%VCPKG_OSRF_DIR%" :: vcpkg does not upgrade installed packages using the install command :: since most of the packages are coming from a frozen snapshot, it is From 7c8c8aa9df7edbd8930e8ab111b995aebdd97b4a Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 13 Apr 2023 00:04:27 +0200 Subject: [PATCH 322/540] Remove death code Signed-off-by: Jose Luis Rivero --- jenkins-scripts/ign_common-default-devel-windows-amd64.bat | 2 -- .../ign_fuel-tools-default-devel-windows-amd64.bat | 5 +---- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/jenkins-scripts/ign_common-default-devel-windows-amd64.bat b/jenkins-scripts/ign_common-default-devel-windows-amd64.bat index 722f705c1..7d0937264 100644 --- a/jenkins-scripts/ign_common-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_common-default-devel-windows-amd64.bat @@ -5,8 +5,6 @@ set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true for /f %%i in ('python "%SCRIPT_DIR%\tools\detect_cmake_major_version.py" "%WORKSPACE%\%VCS_DIRECTORY%\CMakeLists.txt"') do set GZ_MAJOR_VERSION=%%i -if %GZ_MAJOR_VERSION% GEQ 5 ( -) set COLCON_PACKAGE=gz-common set COLCON_AUTO_MAJOR_VERSION=true diff --git a/jenkins-scripts/ign_fuel-tools-default-devel-windows-amd64.bat b/jenkins-scripts/ign_fuel-tools-default-devel-windows-amd64.bat index a0fe03ff2..590f32e41 100644 --- a/jenkins-scripts/ign_fuel-tools-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_fuel-tools-default-devel-windows-amd64.bat @@ -3,10 +3,7 @@ set SCRIPT_DIR=%~dp0 set VCS_DIRECTORY=gz-fuel-tools set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true - -:: tinyxml2 from msgs - set COLCON_PACKAGE=gz-fuel_tools set COLCON_AUTO_MAJOR_VERSION=true -call "%SCRIPT_DIR%\lib\colcon-default-devel-windows.bat" \ No newline at end of file +call "%SCRIPT_DIR%\lib\colcon-default-devel-windows.bat" From e8b22727144c824ccbe407cb5dd21d7625f969c5 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 13 Apr 2023 00:05:44 +0200 Subject: [PATCH 323/540] Clarify what should be an assertion in checking variable Signed-off-by: Jose Luis Rivero --- jenkins-scripts/lib/windows_library.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/lib/windows_library.bat b/jenkins-scripts/lib/windows_library.bat index a3505ca71..28251b683 100644 --- a/jenkins-scripts/lib/windows_library.bat +++ b/jenkins-scripts/lib/windows_library.bat @@ -274,7 +274,7 @@ goto :EOF set LIB_DIR="%~dp0" call %LIB_DIR%\windows_env_vars.bat || goto :error if [%VCPKG_INSTALLED_FILES_DIR%]==[] ( - echo VCPKG_INSTALLED_FILES_DIR seems empty, refuse to delete + echo VCPKG_INSTALLED_FILES_DIR variable seems empty, this is a bug goto :error ) del /s /f /q %VCPKG_INSTALLED_FILES_DIR% From a8ddb76e237d0164e70b5fe53800d4f77a54b9ba Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 13 Apr 2023 00:08:49 +0200 Subject: [PATCH 324/540] Remove death code in physics Signed-off-by: Jose Luis Rivero --- jenkins-scripts/ign_physics-default-devel-windows-amd64.bat | 5 ----- 1 file changed, 5 deletions(-) diff --git a/jenkins-scripts/ign_physics-default-devel-windows-amd64.bat b/jenkins-scripts/ign_physics-default-devel-windows-amd64.bat index 9b3d5de21..e07c16131 100644 --- a/jenkins-scripts/ign_physics-default-devel-windows-amd64.bat +++ b/jenkins-scripts/ign_physics-default-devel-windows-amd64.bat @@ -3,11 +3,6 @@ set SCRIPT_DIR=%~dp0 set VCS_DIRECTORY=gz-physics set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true - -for /f %%i in ('python "%SCRIPT_DIR%\tools\detect_cmake_major_version.py" "%WORKSPACE%\%VCS_DIRECTORY%\CMakeLists.txt"') do set GZ_MAJOR_VERSION=%%i -if %GZ_MAJOR_VERSION% GEQ 6 ( -) -:: This needs to be migrated to DSL to get multi-major versions correctly set COLCON_PACKAGE=gz-physics set COLCON_AUTO_MAJOR_VERSION=true From b6c49d6c24173d17f1394ece4d05e8ca4b4c77b7 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 13 Apr 2023 16:57:06 +0200 Subject: [PATCH 325/540] List vcpkg packages installed in the build log Signed-off-by: Jose Luis Rivero --- jenkins-scripts/lib/colcon-default-devel-windows.bat | 5 ++++- jenkins-scripts/lib/windows_library.bat | 5 +++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/jenkins-scripts/lib/colcon-default-devel-windows.bat b/jenkins-scripts/lib/colcon-default-devel-windows.bat index abffd61e0..809128092 100644 --- a/jenkins-scripts/lib/colcon-default-devel-windows.bat +++ b/jenkins-scripts/lib/colcon-default-devel-windows.bat @@ -85,9 +85,12 @@ echo # END SECTION :: Prepare a clean vcpkg environment with external dependencies call %win_lib% :remove_vcpkg_installation || goto :error -echo # BEGIN SECTION: install all vcpkg dependencies +echo # BEGIN SECTION: vcpkg: install all dependencies call %win_lib% :setup_vcpkg_all_dependencies || goto :error echo # END SECTION +echo # BEGIN SECTION: vcpkg: list installed packages +call %win_lib% :list_vcpkg_packages || goto :error +echo # END SECTION echo # BEGIN SECTION: setup workspace if not defined KEEP_WORKSPACE ( diff --git a/jenkins-scripts/lib/windows_library.bat b/jenkins-scripts/lib/windows_library.bat index 28251b683..995bb29a6 100644 --- a/jenkins-scripts/lib/windows_library.bat +++ b/jenkins-scripts/lib/windows_library.bat @@ -267,6 +267,11 @@ if NOT %VCPKG_HEAD% == %VCPKG_TAG% ( ) goto :EOF +:: ################################## +:list_vcpkg_packages +%VCPKG_CMD% list || goto :error +goto :EOF + :: ################################## :remove_vcpkg_installation :: remove the installed directory to simulate all packages removal From c786fdda0a9240961a846ddd593922011a6170ac Mon Sep 17 00:00:00 2001 From: Clara Berendsen Date: Fri, 14 Apr 2023 14:48:15 -0400 Subject: [PATCH 326/540] remove verbose commentary Co-authored-by: Jose Luis Rivero --- jenkins-scripts/docker/lib/docker_cleanup.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/jenkins-scripts/docker/lib/docker_cleanup.sh b/jenkins-scripts/docker/lib/docker_cleanup.sh index 32c39da22..f71dcddde 100755 --- a/jenkins-scripts/docker/lib/docker_cleanup.sh +++ b/jenkins-scripts/docker/lib/docker_cleanup.sh @@ -3,7 +3,6 @@ docker_version="$(sudo docker version --format '{{.Server.APIVersion}}')" if [[ $(echo "$docker_version > 1.25" | bc -l) ]]; then - echo "Running version of docker $(sudo docker version --format '{{.Server.APIVersion}}') with prune" case $1 in # preferred utilizing each object prune to not affect the build cache in low and medium low) From 98d2038351f8094d5f86910d9640548bd9035728 Mon Sep 17 00:00:00 2001 From: claraberendsen Date: Fri, 14 Apr 2023 16:09:58 -0400 Subject: [PATCH 327/540] (fix) : improve readability and remove backwards compatibility --- jenkins-scripts/docker/lib/docker_cleanup.sh | 91 +++++++------------- 1 file changed, 32 insertions(+), 59 deletions(-) diff --git a/jenkins-scripts/docker/lib/docker_cleanup.sh b/jenkins-scripts/docker/lib/docker_cleanup.sh index f71dcddde..17663983a 100755 --- a/jenkins-scripts/docker/lib/docker_cleanup.sh +++ b/jenkins-scripts/docker/lib/docker_cleanup.sh @@ -1,72 +1,45 @@ -#function that cleans docker -# Receives parameters levels +# DOCKER CLEANUP FUNCTION +# Provides different increasing levels of cleanup (low, medium, high, extreme) for cleaning docker files. +# (PARAMS) +# cleanup_level : +# low = cleanups dangling images and stopped containers created > 5 days ago +# medium = cleanups dangling images and stopped containers created > 1 days ago +# high = docker prune -a +# extreme= cleans docker builder cache +cleanup_level=${1} docker_version="$(sudo docker version --format '{{.Server.APIVersion}}')" if [[ $(echo "$docker_version > 1.25" | bc -l) ]]; then case $1 in # preferred utilizing each object prune to not affect the build cache in low and medium low) - echo "Cleaning dangling images created more than 5 days ago ..." - sudo bash -c "docker image prune -f --filter "until=35h"" - echo "Cleaning stopped containers created more than 5 days ago ..." - sudo bash -c "docker container prune -f --filter "until=35h" " - ;; + echo "Cleaning dangling images created more than 5 days ago ..." + sudo docker image prune -f --filter "until=35h" + echo "Cleaning stopped containers created more than 5 days ago ..." + sudo docker container prune -f --filter "until=35h" + ;; medium) - echo "Cleaning dangling images created more than 1 day ago ..." - sudo bash -c "docker image prune -f --filter "until=24h"" - echo "Cleaning stopped containers created more than 1 day ago ..." - sudo bash -c "docker container prune -f --filter "until=24h"" - ;; + echo "Cleaning dangling images created more than 1 day ago ..." + sudo docker image prune -f --filter "until=24h" + echo "Cleaning stopped containers created more than 1 day ago ..." + sudo docker container prune -f --filter "until=24h" + ;; high) - echo "Cleaning all unusued images, containers, networks and cache ..." - sudo docker system prune --all -f - ;; - extreme) - MAX_SIZE_FOR_BUILD_DIRS="5G" - echo "Removing all cache " - sudo docker builder prune -f --keep-storage $MAX_SIZE_FOR_BUILD_DIRS - ;; - *) - echo "Did not run docker cleaner, level $1 not recognized (accepted values low | medium | high | extreme!! " - ;; - esac - else - - echo "WARNING - Running version of docker $docker_version without prune, utilizing deprecated docker_gc as cleanup method!" - case $1 in - low) - wget https://raw.githubusercontent.com/spotify/docker-gc/master/docker-gc - sudo bash -c "GRACE_PERIOD_SECONDS=432000 bash docker-gc" - ;; - medium) - wget https://raw.githubusercontent.com/spotify/docker-gc/master/docker-gc - sudo bash -c "GRACE_PERIOD_SECONDS=86400 bash docker-gc" - ;; - - high) - [[ -n $(sudo docker ps -q) ]] && sudo docker kill $(sudo docker ps -q) || true - [[ -n $(sudo docker images -a -q) ]] && sudo docker rmi $(sudo docker images -a -q) || true - ;; + echo "Cleaning all unusued images, containers, networks and cache ..." + sudo docker system prune --all -f + ;; extreme) MAX_SIZE_FOR_BUILD_DIRS="5G" - CMD_FIND_BIG_DIRS=(find ${HOME}/workspace -name build -exec du -h -d 0 -t ${MAX_SIZE_FOR_BUILD_DIRS} {} \;) - echo "Clean up the whole cache was not enough. Look for build/ directories bigger than ${MAX_SIZE_FOR_BUILD_DIRS}:" - # run command twice to avoid parsing. First for information proposes - echo ${CMD_FIND_BIG_DIRS[@]} - for d in $("${CMD_FIND_BIG_DIRS[@]}" | awk '{ print $2 }'); do - # safe checks on paths - if [[ $d == ${d/$HOME} ]] || [[ $d == ${d/build} ]]; then - echo "System is trying to delete a path $d outside Jenkins home: $HOME with subdir build/" - exit -1 - fi - # avoid to rm -fr without a path. The ugly trick should leave d as it - # is, in case of a bug in code it will not remove anything at random - sudo rm -fr ${d/build}build - done - ;; + echo "Removing all cache " + sudo docker builder prune -f --keep-storage $MAX_SIZE_FOR_BUILD_DIRS + ;; *) - echo "Did not run docker cleaner, level $1 not recognized (accepted values low | medium | high | extreme!! " - ;; + echo "Did not run docker cleaner, level $cleanup_level not recognized (accepted values low | medium | high | extreme!! " + exit 1 + ;; esac - fi + else + echo ERROR: Please update docker to a newer version with prune! + exit 1 + fi From 5cfa380c6ae8ec81da446b0cef4b136d380f932d Mon Sep 17 00:00:00 2001 From: claraberendsen Date: Mon, 17 Apr 2023 12:00:41 -0400 Subject: [PATCH 328/540] (fix): remove last level from docker_cleanup script --- jenkins-scripts/docker/lib/docker_cleanup.sh | 6 ------ 1 file changed, 6 deletions(-) diff --git a/jenkins-scripts/docker/lib/docker_cleanup.sh b/jenkins-scripts/docker/lib/docker_cleanup.sh index 17663983a..5a556b914 100755 --- a/jenkins-scripts/docker/lib/docker_cleanup.sh +++ b/jenkins-scripts/docker/lib/docker_cleanup.sh @@ -5,7 +5,6 @@ # low = cleanups dangling images and stopped containers created > 5 days ago # medium = cleanups dangling images and stopped containers created > 1 days ago # high = docker prune -a -# extreme= cleans docker builder cache cleanup_level=${1} docker_version="$(sudo docker version --format '{{.Server.APIVersion}}')" @@ -28,11 +27,6 @@ docker_version="$(sudo docker version --format '{{.Server.APIVersion}}')" echo "Cleaning all unusued images, containers, networks and cache ..." sudo docker system prune --all -f ;; - extreme) - MAX_SIZE_FOR_BUILD_DIRS="5G" - echo "Removing all cache " - sudo docker builder prune -f --keep-storage $MAX_SIZE_FOR_BUILD_DIRS - ;; *) echo "Did not run docker cleaner, level $cleanup_level not recognized (accepted values low | medium | high | extreme!! " exit 1 From abe5d531679e741254826f309da2194ac4fadcdc Mon Sep 17 00:00:00 2001 From: claraberendsen Date: Thu, 20 Apr 2023 15:34:16 -0400 Subject: [PATCH 329/540] (feat): remove legacy win_rendering tag from windows_ci job --- jenkins-scripts/dsl/ignition.dsl | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index affb43c57..74c62ac80 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -777,13 +777,6 @@ gz_software.each { gz_sw -> enable_cmake_warnings(gz_sw)) gz_win_ci_any_job.with { - if (gz_sw == 'gui' || - gz_sw == 'rendering' || - gz_sw == 'sensors' || - gz_sw == 'gazebo') - label('win_rendering') - - steps { batchFile("""\ call "./scripts/jenkins-scripts/ign_${gz_sw}-default-devel-windows-amd64.bat" @@ -837,12 +830,6 @@ gz_software.each { gz_sw -> call "./scripts/jenkins-scripts/ign_${gz_sw}-default-devel-windows-amd64.bat" """.stripIndent()) } - - if (gz_sw == 'gui' || - gz_sw == 'rendering' || - gz_sw == 'sensors' || - gz_sw == 'gazebo') - label('win_rendering') } } } From 767622531710b6c7678fc0c4d323896f179e69fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Steven!=20Ragnar=C3=B6k?= Date: Fri, 5 May 2023 20:07:48 -0400 Subject: [PATCH 330/540] Create a Groovy script to apply singleton labels. (#917) * Create a Groovy script to apply singleton labels. I think this script will need to be run as a system groovy script rather than with the sandbox. I have successfully tested it using the Jenkins script console. * Add a comment describing the form. I'd like to make this a struct/class but I'm not quite sure yet. * Replace shuffle with an in-place pick of a random number. I was bending around the immutable set in order to use shuffle. But this is much shorter and makes me happier. * Add a global coment in the file Signed-off-by: Jose Luis Rivero * Add the _nightly_node_labeler DSL job Signed-off-by: Jose Luis Rivero * Filter nodes to only online nodes Signed-off-by: Jose Luis Rivero * Mark build as unstable if labels were not set correctly Signed-off-by: Jose Luis Rivero * Support for cron operations on nightly preparation Signed-off-by: Jose Luis Rivero * Mark build unstable for non expected scenarios Signed-off-by: Jose Luis Rivero * Fix path expected by Jenkins Signed-off-by: Jose Luis Rivero --------- Signed-off-by: Jose Luis Rivero Co-authored-by: Jose Luis Rivero --- jenkins-scripts/dsl/_configs_/Globals.groovy | 2 + jenkins-scripts/dsl/core.dsl | 18 ++++++ jenkins-scripts/dsl/ignition_collection.dsl | 2 +- jenkins-scripts/dsl/scripts | 1 + .../tools/label-assignment-backstop.groovy | 59 +++++++++++++++++++ 5 files changed, 81 insertions(+), 1 deletion(-) create mode 120000 jenkins-scripts/dsl/scripts create mode 100644 jenkins-scripts/tools/label-assignment-backstop.groovy diff --git a/jenkins-scripts/dsl/_configs_/Globals.groovy b/jenkins-scripts/dsl/_configs_/Globals.groovy index 616055f19..0ea44c616 100644 --- a/jenkins-scripts/dsl/_configs_/Globals.groovy +++ b/jenkins-scripts/dsl/_configs_/Globals.groovy @@ -12,6 +12,8 @@ class Globals static CRON_EVERY_THREE_DAYS = 'H H * * H/3' static CRON_ON_WEEKEND = 'H H * * 6-7' + static CRON_PREPARE_NIGHTLY = '45 8 * * *' + static CRON_START_NIGHTLY = '0 9 * * *' // Only one -E regex can be passed, so make a regex that matches both // _ign_TEST and _gz_TEST diff --git a/jenkins-scripts/dsl/core.dsl b/jenkins-scripts/dsl/core.dsl index 75f82a1d8..7dbb18967 100644 --- a/jenkins-scripts/dsl/core.dsl +++ b/jenkins-scripts/dsl/core.dsl @@ -96,3 +96,21 @@ reprepro.with """.stripIndent()) } } + +// ------------------------------------------------------------------- +def nightly_labeler = job("_nightly_node_labeler") +OSRFBase.create(nightly_labeler) +nightly_labeler.with +{ + label Globals.nontest_label("built-in") + + triggers { + cron(Globals.CRON_PREPARE_NIGHTLY) + } + + steps + { + // path root changes from standalone to Jenkins. Be careful + systemGroovyCommand(readFileFromWorkspace('scripts/jenkins-scripts/tools/label-assignment-backstop.groovy')) + } +} diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index abfea3211..c298b826f 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -611,7 +611,7 @@ nightly_scheduler_job.with } triggers { - cron('0 9 * * *') + cron(Globals.CRON_START_NIGHTLY) } cmake_branch = get_nightly_branch(collection_data, 'cmake') diff --git a/jenkins-scripts/dsl/scripts b/jenkins-scripts/dsl/scripts new file mode 120000 index 000000000..6581736d6 --- /dev/null +++ b/jenkins-scripts/dsl/scripts @@ -0,0 +1 @@ +../../ \ No newline at end of file diff --git a/jenkins-scripts/tools/label-assignment-backstop.groovy b/jenkins-scripts/tools/label-assignment-backstop.groovy new file mode 100644 index 000000000..fe2ee3c35 --- /dev/null +++ b/jenkins-scripts/tools/label-assignment-backstop.groovy @@ -0,0 +1,59 @@ +/* + * The script is designed to assign the nightly labels + * for nodes dynamically, Only one node should build + * nightlies for each distribution in order to ensure + * the correct orchestration of the Gazebo libraries + * interdependencies. + */ + +import jenkins.*; +import jenkins.model.*; +import hudson.model.Label; + +/* + Each label is in a 2-tuple with a label expression that indicates its "pool" + of potential assignees. + The first field must be a single label (no spaces) but the second may be an + arbitrarily complex label expression like "(docker && linux) && !armhf" +*/ +def exactly_one_labels = [ + ["linux-nightly-focal", "docker"], + ["linux-nightly-jammy", "docker"], +] + +for (tup in exactly_one_labels) { + label_nodes = Label.get(tup[0]).getNodes().findAll { it.toComputer().isOnline() } + + if (label_nodes.size() == 1) { + println(tup[0] + " is currently applied to " + label_nodes[0].name) + continue + } + + // return unstable if the labels are not set correctly before running this + println("MARK_AS_UNSTABLE") + + if (label_nodes.size() > 1) { + println("WARNING: Too many online nodes with the label " + tup[0]) + for (node in label_nodes) { + println(" " + node.name) + } + continue + } + + if (label_nodes.size() < 1) { + println("No online host currently has the label " + tup[0]) + println("Appointing a node from the configured pool matching '" + tup[1] + "'") + node_pool = Label.get(tup[1]).getNodes().findAll { it.toComputer().isOnline() } + if (node_pool.size() <= 0) { + println("WARNING: Pool of '" + tup[1] + "' machines for " + tup[0] + " is empty!") + } + // Pick a random node from the pool to receive the label. + appointed_node = node_pool[(new Random()).nextInt(node_pool.size())] + new_label_string = appointed_node.getAssignedLabels().join(" ") + new_label_string = tup[0] + " " + new_label_string + appointed_node.setLabelString(new_label_string) + appointed_node.save() + println("Added label to " + appointed_node.name) + continue + } +} From 15def1c1254e4b62ca8385610e8039575fb464e1 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Mon, 8 May 2023 12:23:01 +0200 Subject: [PATCH 331/540] Add the unstable parser to the nightly_labelers Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/core.dsl | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/core.dsl b/jenkins-scripts/dsl/core.dsl index 7dbb18967..6800dd94f 100644 --- a/jenkins-scripts/dsl/core.dsl +++ b/jenkins-scripts/dsl/core.dsl @@ -102,7 +102,7 @@ def nightly_labeler = job("_nightly_node_labeler") OSRFBase.create(nightly_labeler) nightly_labeler.with { - label Globals.nontest_label("built-in") + label Globals.nontest_label("master") triggers { cron(Globals.CRON_PREPARE_NIGHTLY) @@ -113,4 +113,16 @@ nightly_labeler.with // path root changes from standalone to Jenkins. Be careful systemGroovyCommand(readFileFromWorkspace('scripts/jenkins-scripts/tools/label-assignment-backstop.groovy')) } + + publishers + { + // Added the checker result parser (UNSTABLE if not success) + configure { project -> + project / publishers << 'hudson.plugins.logparser.LogParserPublisher' { + unstableOnWarning true + failBuildOnError false + parsingRulesPath('/var/lib/jenkins/logparser_warn_on_mark_unstable') + } + } + } } From 27fb5e8421a7a184c66c09aef1b29c35925a02cf Mon Sep 17 00:00:00 2001 From: claraberendsen Date: Wed, 10 May 2023 11:33:00 -0400 Subject: [PATCH 332/540] add nontest_label to _config_ classes --- jenkins-scripts/dsl/_configs_/OSRFLinuxBase.groovy | 3 ++- jenkins-scripts/dsl/_configs_/OSRFLinuxBuildPkg.groovy | 3 ++- jenkins-scripts/dsl/_configs_/OSRFOsXBase.groovy | 3 ++- jenkins-scripts/dsl/_configs_/OSRFWinBase.groovy | 3 ++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/jenkins-scripts/dsl/_configs_/OSRFLinuxBase.groovy b/jenkins-scripts/dsl/_configs_/OSRFLinuxBase.groovy index bdce34f57..acfa7dec7 100644 --- a/jenkins-scripts/dsl/_configs_/OSRFLinuxBase.groovy +++ b/jenkins-scripts/dsl/_configs_/OSRFLinuxBase.groovy @@ -1,6 +1,7 @@ package _configs_ import javaposse.jobdsl.dsl.Job +import _configs_.Globals /* -> OSRFUNIXBase @@ -17,7 +18,7 @@ class OSRFLinuxBase job.with { - label "docker" + label Globals.nontest_label("docker") publishers { archiveArtifacts { diff --git a/jenkins-scripts/dsl/_configs_/OSRFLinuxBuildPkg.groovy b/jenkins-scripts/dsl/_configs_/OSRFLinuxBuildPkg.groovy index 3d3caf4db..9f61a68fe 100644 --- a/jenkins-scripts/dsl/_configs_/OSRFLinuxBuildPkg.groovy +++ b/jenkins-scripts/dsl/_configs_/OSRFLinuxBuildPkg.groovy @@ -1,6 +1,7 @@ package _configs_ import javaposse.jobdsl.dsl.Job +import _configs_.Globals /* -> OSRFLinuxBuildPkgBase @@ -71,7 +72,7 @@ class OSRFLinuxBuildPkg "OSRF repos name to use when building the package") labelParam('JENKINS_NODE_TAG') { description('Jenkins node or group to run build') - defaultValue('docker') + defaultValue(Globals.nontest_label('docker')) } } diff --git a/jenkins-scripts/dsl/_configs_/OSRFOsXBase.groovy b/jenkins-scripts/dsl/_configs_/OSRFOsXBase.groovy index 1e92e3ef5..dd5dbed54 100644 --- a/jenkins-scripts/dsl/_configs_/OSRFOsXBase.groovy +++ b/jenkins-scripts/dsl/_configs_/OSRFOsXBase.groovy @@ -1,6 +1,7 @@ package _configs_ import javaposse.jobdsl.dsl.Job +import _configs_.Globals /* -> OSRFOsXBase @@ -17,7 +18,7 @@ class OSRFOsXBase job.with { - label "osx" + label Globals.nontest_label("osx") } } } diff --git a/jenkins-scripts/dsl/_configs_/OSRFWinBase.groovy b/jenkins-scripts/dsl/_configs_/OSRFWinBase.groovy index cd6d8f0fe..4c9bbffcd 100644 --- a/jenkins-scripts/dsl/_configs_/OSRFWinBase.groovy +++ b/jenkins-scripts/dsl/_configs_/OSRFWinBase.groovy @@ -1,6 +1,7 @@ package _configs_ import javaposse.jobdsl.dsl.Job +import _configs_.Globals /* Implements: @@ -14,7 +15,7 @@ class OSRFWinBase extends OSRFBase OSRFBase.create(job) job.with { - label "win" + label Globals.nontest_label("win") steps { From 9fbfe60133d2b7b8b280b92f7c563dc64c8367a5 Mon Sep 17 00:00:00 2001 From: Clara Berendsen Date: Thu, 11 May 2023 10:55:53 -0400 Subject: [PATCH 333/540] NVidia mismatch recovery (#915) If the build failed due to the Nvidia error, we do the following steps: * Remove all labels from current agent and add a recovery-process label to indicate that that agent is performing recovery actions, preventing it from taking on any other build. * Requeue the job with same parameters with a delay of 70s to give time for the next step. * Schedule a system restart on 1 minute (the delay is needed here so that the postbuild action finishes correctly --- .../dsl/_configs_/OSRFUNIXBase.groovy | 55 ++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/_configs_/OSRFUNIXBase.groovy b/jenkins-scripts/dsl/_configs_/OSRFUNIXBase.groovy index 037a59c06..2c9989464 100644 --- a/jenkins-scripts/dsl/_configs_/OSRFUNIXBase.groovy +++ b/jenkins-scripts/dsl/_configs_/OSRFUNIXBase.groovy @@ -38,6 +38,59 @@ class OSRFUNIXBase extends OSRFBase git clone https://github.com/gazebo-tooling/release-tools scripts -b \$RTOOLS_BRANCH """.stripIndent()) } + publishers { + postBuildScripts { + steps{ + conditionalSteps { + condition { + expression("(.)* gpu-nvidia (.)*",'${NODE_LABELS}') + } + steps { + systemGroovyCommand('''\ + import hudson.model.Cause.UpstreamCause; + import hudson.model.*; + + def node = build.getBuiltOn() + def old_labels = node.getLabelString() + + println("# BEGIN SECTION: NVIDIA MISMATCH RECOVERY") + if (!(build.getLog(1000) =~ "nvml error: driver/library version mismatch")) { + println(" NVIDIA driver/library version mismatch not detected in the log - Not performing any recovery automatic recovery step") + return 1; + } else { + try { + println(" PROBLEM: NVIDIA driver/library version mismatch was detected in the log. Try to automatically resolve it:") + println("Removing labels and adding 'recovery-process' label to node") + node.setLabelString("recovery-process") + } catch (Exception ex) { + println("ERROR - CANNOT PERFORM RECOVERY ACTIONS FOR NVIDIA ERROR") + println("Restoring to previous state") + node.setLabelString(old_labels) + throw ex + } + } + println("# END SECTION: NVIDIA MISMATCH RECOVERY") + '''.stripIndent() + ) + shell("""sudo shutdown -r +1""") + } + } + } + onlyIfBuildSucceeds(false) + onlyIfBuildFails(true) + } + // Manual insertion of xml for Naginator plugin because of this issue https://issues.jenkins.io/browse/JENKINS-66458 + configure { project -> + project / publishers / 'com.chikli.hudson.plugin.naginator.NaginatorPublisher' { + regexpForRerun("nvml error: driver/library version mismatch") + checkRegexp(true) + maxSchedule(1) + delay(class: 'com.chikli.hudson.plugin.naginator.FixedDelay') { + delay(70) + } + } + } + } } } -} +} \ No newline at end of file From bbb84ae233fc14ec5ee6c5e19faa44b87ccb76c4 Mon Sep 17 00:00:00 2001 From: claraberendsen Date: Mon, 15 May 2023 18:11:18 -0400 Subject: [PATCH 334/540] add README for jenkins scripts --- jenkins-scripts/README.md | 43 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 jenkins-scripts/README.md diff --git a/jenkins-scripts/README.md b/jenkins-scripts/README.md new file mode 100644 index 000000000..acf9b32f9 --- /dev/null +++ b/jenkins-scripts/README.md @@ -0,0 +1,43 @@ +# Jenkins scripts + +We use [DSL Jenkins plugin](https://plugins.jenkins.io/job-dsl/) to allow us to manage and define the jobs as human readable code. You can find the different job configs under the [`dsl`](./dsl/) folder. + +## Useful links +- [List of installed plugins in Jenkins](https://github.com/osrf/chef-osrf/blob/latest/cookbooks/osrfbuild/attributes/plugins.rb) +- [Jenkins DSL API docs](https://jenkinsci.github.io/job-dsl-plugin/) +- [Jenkins DSL Wiki](https://github.com/jenkinsci/job-dsl-plugin/wiki) + + +## Local testing + +To test locally the build of the different `dsl` jobs you need the following: + +1. Download `job_dsl-core.standalone.jar` to be able to build the dsl jobs locally. [Download here](https://repo.jenkins-ci.org/artifactory/releases/org/jenkins-ci/plugins/job-dsl-core/1.77/) +2. Add the downloaded file to a folder not tracked in the repo. +3. In the terminal execute: +``` bash +java -jar `PATH/job_dsl-core.jar` `PATH/job_name.dsl` +``` +For more information go [here](https://github.com/jenkinsci/job-dsl-plugin/wiki/User-Power-Moves#run-a-dsl-script-locally). + +## Development workflow + +1. Make changes locally and test that it builds correctly. +2. Push changes to a specific branch in `release-tools` +3. Go to seed job for the job you wanna test (usually you would use `_dsl_test` to not affect the jobs in production) and build with the parameter pointing to your new custom branch. +4. After it builds correctly, you will have generated jobs with the changes you implemented. You can use and modify the generated job. + +> WARNING! : Running the _dsl job for a specific job that it's not `test` will modify the configuration for production. You should always aim to utilize `test` jobs. + +## :arrow_forward: Playbook + +### XML injection into DSL +Sometimes we find a situation where we have a plugin installed that it's not mapped to DSL and you may think: How can I use this plugin?. + +There is a feature called [configure blocks](https://github.com/jenkinsci/job-dsl-plugin/wiki/The-Configure-Block) that allows us to represent xml job configs as DSL. Below is an example of it's usage with the retryBuild plugin: + +https://github.com/gazebo-tooling/release-tools/blob/9fbfe60133d2b7b8b280b92f7c563dc64c8367a5/jenkins-scripts/dsl/_configs_/OSRFUNIXBase.groovy#LL83C1-L92C10 + +Here `checkRegexp(true)` gets converted into `true` and the hierarchy of the definition is respected, so `checkRegexp` exists as a child of `com.chikli.hudson.plugin.naginator.NaginatorPublisher` in the XML definition. + +To check what are the corresponding names for the XML tags you can refer to the plugin documentation or as an alternative you can manually modify the job to add the information you want and then go to `https://build.osrfoundation.com/job/myjob/config.xml` and match the XML there in the DSL config. From 5e2578c49b4ddd7cad9a117b1a8195d1ce4d0f39 Mon Sep 17 00:00:00 2001 From: claraberendsen Date: Mon, 15 May 2023 18:32:23 -0400 Subject: [PATCH 335/540] change permalink since code snippets is not working --- jenkins-scripts/README.md | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/jenkins-scripts/README.md b/jenkins-scripts/README.md index acf9b32f9..053ecb253 100644 --- a/jenkins-scripts/README.md +++ b/jenkins-scripts/README.md @@ -34,10 +34,6 @@ For more information go [here](https://github.com/jenkinsci/job-dsl-plugin/wiki/ ### XML injection into DSL Sometimes we find a situation where we have a plugin installed that it's not mapped to DSL and you may think: How can I use this plugin?. -There is a feature called [configure blocks](https://github.com/jenkinsci/job-dsl-plugin/wiki/The-Configure-Block) that allows us to represent xml job configs as DSL. Below is an example of it's usage with the retryBuild plugin: - -https://github.com/gazebo-tooling/release-tools/blob/9fbfe60133d2b7b8b280b92f7c563dc64c8367a5/jenkins-scripts/dsl/_configs_/OSRFUNIXBase.groovy#LL83C1-L92C10 - -Here `checkRegexp(true)` gets converted into `true` and the hierarchy of the definition is respected, so `checkRegexp` exists as a child of `com.chikli.hudson.plugin.naginator.NaginatorPublisher` in the XML definition. +There is a feature called [configure blocks](https://github.com/jenkinsci/job-dsl-plugin/wiki/The-Configure-Block) that allows us to represent xml job configs as DSL. [Here](https://github.com/gazebo-tooling/release-tools/blob/9fbfe60133d2b7b8b280b92f7c563dc64c8367a5/jenkins-scripts/dsl/_configs_/OSRFUNIXBase.groovy#LL83C1-L92C10) is an example of it's usage with the retryBuild plugin, where `checkRegexp(true)` gets converted into `true` and the hierarchy of the definition is respected, so `checkRegexp` exists as a child of `com.chikli.hudson.plugin.naginator.NaginatorPublisher` in the XML definition. To check what are the corresponding names for the XML tags you can refer to the plugin documentation or as an alternative you can manually modify the job to add the information you want and then go to `https://build.osrfoundation.com/job/myjob/config.xml` and match the XML there in the DSL config. From 3ea77b39355ee209efcdf05da98eca05b4e7ef87 Mon Sep 17 00:00:00 2001 From: Clara Berendsen Date: Tue, 16 May 2023 13:00:42 -0400 Subject: [PATCH 336/540] more information on seed job Co-authored-by: Jose Luis Rivero --- jenkins-scripts/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/README.md b/jenkins-scripts/README.md index 053ecb253..8a3b64c45 100644 --- a/jenkins-scripts/README.md +++ b/jenkins-scripts/README.md @@ -24,7 +24,7 @@ For more information go [here](https://github.com/jenkinsci/job-dsl-plugin/wiki/ 1. Make changes locally and test that it builds correctly. 2. Push changes to a specific branch in `release-tools` -3. Go to seed job for the job you wanna test (usually you would use `_dsl_test` to not affect the jobs in production) and build with the parameter pointing to your new custom branch. +3. Go to seed job for the job you wanna test (usually you would use [`_dsl_test`](https://build.osrfoundation.org/job/_dsl_test/) to not affect the jobs in production) and build with the parameter pointing to your new custom branch in the `RTOOLS_BRANCH` parameter. 4. After it builds correctly, you will have generated jobs with the changes you implemented. You can use and modify the generated job. > WARNING! : Running the _dsl job for a specific job that it's not `test` will modify the configuration for production. You should always aim to utilize `test` jobs. From aebe4419a7ced6ff53cbc3fe261cfca9cadc3049 Mon Sep 17 00:00:00 2001 From: claraberendsen Date: Tue, 16 May 2023 15:26:04 -0400 Subject: [PATCH 337/540] fix review comments --- jenkins-scripts/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jenkins-scripts/README.md b/jenkins-scripts/README.md index 8a3b64c45..132c79621 100644 --- a/jenkins-scripts/README.md +++ b/jenkins-scripts/README.md @@ -1,6 +1,6 @@ # Jenkins scripts -We use [DSL Jenkins plugin](https://plugins.jenkins.io/job-dsl/) to allow us to manage and define the jobs as human readable code. You can find the different job configs under the [`dsl`](./dsl/) folder. +The release-tools repository uses the [DSL Jenkins plugin](https://plugins.jenkins.io/job-dsl/) to allow us to programmatically generate the job configuration (configuration as code). You can find the different job configs under the [`dsl`](./dsl/) folder. ## Useful links - [List of installed plugins in Jenkins](https://github.com/osrf/chef-osrf/blob/latest/cookbooks/osrfbuild/attributes/plugins.rb) @@ -12,7 +12,7 @@ We use [DSL Jenkins plugin](https://plugins.jenkins.io/job-dsl/) to allow us to To test locally the build of the different `dsl` jobs you need the following: -1. Download `job_dsl-core.standalone.jar` to be able to build the dsl jobs locally. [Download here](https://repo.jenkins-ci.org/artifactory/releases/org/jenkins-ci/plugins/job-dsl-core/1.77/) +1. Download `job_dsl-core.standalone.jar` to be able to build the dsl jobs locally. It's necessary to use the [same version](https://github.com/osrf/chef-osrf/blob/latest/cookbooks/osrfbuild/attributes/plugins.rb#L70) for the jarfrom the dsl plugin. [Download here](https://repo.jenkins-ci.org/artifactory/releases/org/jenkins-ci/plugins/job-dsl-core) 2. Add the downloaded file to a folder not tracked in the repo. 3. In the terminal execute: ``` bash From c7f97a46a945c4d29796ba00cc9df2c96852096f Mon Sep 17 00:00:00 2001 From: Clara Berendsen Date: Tue, 16 May 2023 18:06:37 -0400 Subject: [PATCH 338/540] Update jenkins-scripts/README.md Co-authored-by: Jose Luis Rivero --- jenkins-scripts/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/README.md b/jenkins-scripts/README.md index 132c79621..1b34d7c1e 100644 --- a/jenkins-scripts/README.md +++ b/jenkins-scripts/README.md @@ -32,7 +32,7 @@ For more information go [here](https://github.com/jenkinsci/job-dsl-plugin/wiki/ ## :arrow_forward: Playbook ### XML injection into DSL -Sometimes we find a situation where we have a plugin installed that it's not mapped to DSL and you may think: How can I use this plugin?. +How to deal with plugins that do not implement the DSL layer and don't provide a DSL API? There is a feature called [configure blocks](https://github.com/jenkinsci/job-dsl-plugin/wiki/The-Configure-Block) that allows us to represent xml job configs as DSL. [Here](https://github.com/gazebo-tooling/release-tools/blob/9fbfe60133d2b7b8b280b92f7c563dc64c8367a5/jenkins-scripts/dsl/_configs_/OSRFUNIXBase.groovy#LL83C1-L92C10) is an example of it's usage with the retryBuild plugin, where `checkRegexp(true)` gets converted into `true` and the hierarchy of the definition is respected, so `checkRegexp` exists as a child of `com.chikli.hudson.plugin.naginator.NaginatorPublisher` in the XML definition. From f0c9422f6209eb44a0922ce3709e5f509f4659ff Mon Sep 17 00:00:00 2001 From: Crola1702 Date: Wed, 17 May 2023 10:46:21 -0500 Subject: [PATCH 339/540] Retry the build when an agent disconnection ocurrs Signed-off-by: Crola1702 --- jenkins-scripts/dsl/_configs_/OSRFBase.groovy | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/jenkins-scripts/dsl/_configs_/OSRFBase.groovy b/jenkins-scripts/dsl/_configs_/OSRFBase.groovy index 2ebec76ec..cfaa8489e 100644 --- a/jenkins-scripts/dsl/_configs_/OSRFBase.groovy +++ b/jenkins-scripts/dsl/_configs_/OSRFBase.groovy @@ -38,5 +38,15 @@ class OSRFBase } } } + publishers { + configure { project -> + // When a build fails because an agent disconnects, retry it once + project / publishers / 'com.chikli.hudson.plugin.naginator.NaginatorPublisher' { + regexpForRerun("java.nio.channels.ClosedChannelException") + checkRegexp(true) + maxSchedule(1) + } + } + } } } From 542553d2b0dc87fb9855bb6cfe6358617b078fbc Mon Sep 17 00:00:00 2001 From: Crola1702 Date: Wed, 17 May 2023 11:11:30 -0500 Subject: [PATCH 340/540] Move publishers inside job configuration Signed-off-by: Crola1702 --- jenkins-scripts/dsl/_configs_/OSRFBase.groovy | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/jenkins-scripts/dsl/_configs_/OSRFBase.groovy b/jenkins-scripts/dsl/_configs_/OSRFBase.groovy index cfaa8489e..7a0cf713c 100644 --- a/jenkins-scripts/dsl/_configs_/OSRFBase.groovy +++ b/jenkins-scripts/dsl/_configs_/OSRFBase.groovy @@ -37,14 +37,14 @@ class OSRFBase systemGroovyCommand("build.setDescription('RTOOLS_BRANCH: ' + build.buildVariableResolver.resolve('RTOOLS_BRANCH'));") } } - } - publishers { - configure { project -> - // When a build fails because an agent disconnects, retry it once - project / publishers / 'com.chikli.hudson.plugin.naginator.NaginatorPublisher' { - regexpForRerun("java.nio.channels.ClosedChannelException") - checkRegexp(true) - maxSchedule(1) + publishers { + configure { project -> + // When a build fails because an agent disconnects, retry it once + project / publishers / 'com.chikli.hudson.plugin.naginator.NaginatorPublisher' { + regexpForRerun("java.nio.channels.ClosedChannelException") + checkRegexp(true) + maxSchedule(1) + } } } } From d756656bb9ada3835b55dc662b405e148cc53ed3 Mon Sep 17 00:00:00 2001 From: Crola1702 Date: Wed, 17 May 2023 15:15:03 -0500 Subject: [PATCH 341/540] Add RetryFailures for updating naginator regex parameter in child classes Signed-off-by: Crola1702 --- .../dsl/_configs_/RetryFailures.groovy | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 jenkins-scripts/dsl/_configs_/RetryFailures.groovy diff --git a/jenkins-scripts/dsl/_configs_/RetryFailures.groovy b/jenkins-scripts/dsl/_configs_/RetryFailures.groovy new file mode 100644 index 000000000..02511796f --- /dev/null +++ b/jenkins-scripts/dsl/_configs_/RetryFailures.groovy @@ -0,0 +1,17 @@ +package _configs_ + +import javaposse.jobdsl.dsl.Job + +class RetryFailures { + static void update_retry_parameters(Job job, String new_retry_regex) { + + job.with { + publishers { + configure { project -> + def current_retry_regex = (project / publishers / 'com.chikli.hudson.plugin.naginator.NaginatorPublisher' / regexpForRerun).text() + (project / publishers / 'com.chikli.hudson.plugin.naginator.NaginatorPublisher' / regexpForRerun).value = current_retry_regex + "|" + new_retry_regex + } + } + } + } +} From 0071cd252b34fd5d711d3a3e48bfa340a131cfd0 Mon Sep 17 00:00:00 2001 From: Crola1702 Date: Wed, 17 May 2023 15:16:22 -0500 Subject: [PATCH 342/540] Change how regex is added to naginator for nvidia mismatch problem OSRFBase.groovy creates the naginator publisher retry tag. However, having the previous functionality replaces the ClosedChannel exception error. I'm using the `update_retry_parameters` to add the nvml error at the same regex as CloseChannelException Signed-off-by: Crola1702 --- .../dsl/_configs_/OSRFUNIXBase.groovy | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/jenkins-scripts/dsl/_configs_/OSRFUNIXBase.groovy b/jenkins-scripts/dsl/_configs_/OSRFUNIXBase.groovy index 2c9989464..eaeda9515 100644 --- a/jenkins-scripts/dsl/_configs_/OSRFUNIXBase.groovy +++ b/jenkins-scripts/dsl/_configs_/OSRFUNIXBase.groovy @@ -80,17 +80,19 @@ class OSRFUNIXBase extends OSRFBase onlyIfBuildFails(true) } // Manual insertion of xml for Naginator plugin because of this issue https://issues.jenkins.io/browse/JENKINS-66458 - configure { project -> - project / publishers / 'com.chikli.hudson.plugin.naginator.NaginatorPublisher' { - regexpForRerun("nvml error: driver/library version mismatch") - checkRegexp(true) - maxSchedule(1) - delay(class: 'com.chikli.hudson.plugin.naginator.FixedDelay') { - delay(70) - } - } - } + // configure { project -> + // project / publishers / 'com.chikli.hudson.plugin.naginator.NaginatorPublisher' { + // regexpForRerun("nvml error: driver/library version mismatch") + // checkRegexp(true) + // maxSchedule(1) + // delay(class: 'com.chikli.hudson.plugin.naginator.FixedDelay') { + // delay(70) + // } + // } + // } } } + + RetryFailures.update_retry_parameters(job, 'nvml error driver/library version mismatch') } } \ No newline at end of file From b35a70d696824f743ae1aa6783293ef7ef70a0ac Mon Sep 17 00:00:00 2001 From: Crola1702 Date: Wed, 17 May 2023 15:22:24 -0500 Subject: [PATCH 343/540] Remove code comments Signed-off-by: Crola1702 --- jenkins-scripts/dsl/_configs_/OSRFUNIXBase.groovy | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/jenkins-scripts/dsl/_configs_/OSRFUNIXBase.groovy b/jenkins-scripts/dsl/_configs_/OSRFUNIXBase.groovy index eaeda9515..5e562d1c7 100644 --- a/jenkins-scripts/dsl/_configs_/OSRFUNIXBase.groovy +++ b/jenkins-scripts/dsl/_configs_/OSRFUNIXBase.groovy @@ -79,20 +79,10 @@ class OSRFUNIXBase extends OSRFBase onlyIfBuildSucceeds(false) onlyIfBuildFails(true) } - // Manual insertion of xml for Naginator plugin because of this issue https://issues.jenkins.io/browse/JENKINS-66458 - // configure { project -> - // project / publishers / 'com.chikli.hudson.plugin.naginator.NaginatorPublisher' { - // regexpForRerun("nvml error: driver/library version mismatch") - // checkRegexp(true) - // maxSchedule(1) - // delay(class: 'com.chikli.hudson.plugin.naginator.FixedDelay') { - // delay(70) - // } - // } - // } } } + // Add the new regex to naginator parameter RetryFailures.update_retry_parameters(job, 'nvml error driver/library version mismatch') } } \ No newline at end of file From d7634d83deebb56b3f08d401dea7fb81134adc2a Mon Sep 17 00:00:00 2001 From: Crola1702 Date: Wed, 17 May 2023 16:10:53 -0500 Subject: [PATCH 344/540] Add option to update naginator parameters Signed-off-by: Crola1702 --- jenkins-scripts/dsl/_configs_/OSRFUNIXBase.groovy | 2 +- jenkins-scripts/dsl/_configs_/RetryFailures.groovy | 13 ++++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/jenkins-scripts/dsl/_configs_/OSRFUNIXBase.groovy b/jenkins-scripts/dsl/_configs_/OSRFUNIXBase.groovy index 5e562d1c7..37326b204 100644 --- a/jenkins-scripts/dsl/_configs_/OSRFUNIXBase.groovy +++ b/jenkins-scripts/dsl/_configs_/OSRFUNIXBase.groovy @@ -83,6 +83,6 @@ class OSRFUNIXBase extends OSRFBase } // Add the new regex to naginator parameter - RetryFailures.update_retry_parameters(job, 'nvml error driver/library version mismatch') + RetryFailures.update_retry_parameters(job, 'nvml error driver/library version mismatch', 70) } } \ No newline at end of file diff --git a/jenkins-scripts/dsl/_configs_/RetryFailures.groovy b/jenkins-scripts/dsl/_configs_/RetryFailures.groovy index 02511796f..eb3ecaf28 100644 --- a/jenkins-scripts/dsl/_configs_/RetryFailures.groovy +++ b/jenkins-scripts/dsl/_configs_/RetryFailures.groovy @@ -3,13 +3,20 @@ package _configs_ import javaposse.jobdsl.dsl.Job class RetryFailures { - static void update_retry_parameters(Job job, String new_retry_regex) { + static void update_retry_parameters(Job job, String new_regexpForRerun, int new_delay = 0, boolean new_checkRegexp = true, int new_maxSchedule = 1) { job.with { publishers { configure { project -> - def current_retry_regex = (project / publishers / 'com.chikli.hudson.plugin.naginator.NaginatorPublisher' / regexpForRerun).text() - (project / publishers / 'com.chikli.hudson.plugin.naginator.NaginatorPublisher' / regexpForRerun).value = current_retry_regex + "|" + new_retry_regex + def new_retry_regex = (project / publishers / 'com.chikli.hudson.plugin.naginator.NaginatorPublisher' / regexpForRerun).text() + "|" + new_regexpForRerun + + (project / publishers / 'com.chikli.hudson.plugin.naginator.NaginatorPublisher' / regexpForRerun).value = new_retry_regex + (project / publishers / 'com.chikli.hudson.plugin.naginator.NaginatorPublisher' / checkRegexp).value = new_checkRegexp + (project / publishers / 'com.chikli.hudson.plugin.naginator.NaginatorPublisher' / maxSchedule).value = new_maxSchedule + + if (new_delay) { + (project / publishers / 'com.chikli.hudson.plugin.naginator.NaginatorPublisher' / delay(class: "com.chikli.hudson.plugin.naginator.FixedDelay") / delay).value = new_delay + } } } } From 27f683df946d313b44685138b821d935bcfc3845 Mon Sep 17 00:00:00 2001 From: Crola1702 Date: Thu, 18 May 2023 11:22:18 -0500 Subject: [PATCH 345/540] Test: shutdown step should retrigger the build Signed-off-by: Crola1702 --- jenkins-scripts/dsl/_configs_/OSRFBase.groovy | 3 +++ 1 file changed, 3 insertions(+) diff --git a/jenkins-scripts/dsl/_configs_/OSRFBase.groovy b/jenkins-scripts/dsl/_configs_/OSRFBase.groovy index 7a0cf713c..da3142221 100644 --- a/jenkins-scripts/dsl/_configs_/OSRFBase.groovy +++ b/jenkins-scripts/dsl/_configs_/OSRFBase.groovy @@ -37,6 +37,9 @@ class OSRFBase systemGroovyCommand("build.setDescription('RTOOLS_BRANCH: ' + build.buildVariableResolver.resolve('RTOOLS_BRANCH'));") } } + steps { + shell("""sudo shutdown -r""") + } publishers { configure { project -> // When a build fails because an agent disconnects, retry it once From 22ba0e9979aec6c5dc46f86fbbbf98b60c07182d Mon Sep 17 00:00:00 2001 From: Crola1702 Date: Thu, 18 May 2023 11:34:49 -0500 Subject: [PATCH 346/540] Test: shutdown step should retrigger the build version 2 Signed-off-by: Crola1702 --- jenkins-scripts/dsl/_configs_/OSRFBase.groovy | 3 --- jenkins-scripts/dsl/_configs_/OSRFUNIXBase.groovy | 1 + 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/jenkins-scripts/dsl/_configs_/OSRFBase.groovy b/jenkins-scripts/dsl/_configs_/OSRFBase.groovy index da3142221..7a0cf713c 100644 --- a/jenkins-scripts/dsl/_configs_/OSRFBase.groovy +++ b/jenkins-scripts/dsl/_configs_/OSRFBase.groovy @@ -37,9 +37,6 @@ class OSRFBase systemGroovyCommand("build.setDescription('RTOOLS_BRANCH: ' + build.buildVariableResolver.resolve('RTOOLS_BRANCH'));") } } - steps { - shell("""sudo shutdown -r""") - } publishers { configure { project -> // When a build fails because an agent disconnects, retry it once diff --git a/jenkins-scripts/dsl/_configs_/OSRFUNIXBase.groovy b/jenkins-scripts/dsl/_configs_/OSRFUNIXBase.groovy index 37326b204..8e9526d1d 100644 --- a/jenkins-scripts/dsl/_configs_/OSRFUNIXBase.groovy +++ b/jenkins-scripts/dsl/_configs_/OSRFUNIXBase.groovy @@ -37,6 +37,7 @@ class OSRFUNIXBase extends OSRFBase [[ -d ./scripts ]] && rm -fr ./scripts git clone https://github.com/gazebo-tooling/release-tools scripts -b \$RTOOLS_BRANCH """.stripIndent()) + shell("sudo shutdown -r") } publishers { postBuildScripts { From 3b3caaf353b9a2ceec7f46c9298259704b82d2e4 Mon Sep 17 00:00:00 2001 From: Crola1702 Date: Thu, 18 May 2023 11:45:10 -0500 Subject: [PATCH 347/540] Test: shutdown step should retrigger the build version 3 Signed-off-by: Crola1702 --- jenkins-scripts/dsl/_configs_/OSRFLinuxBase.groovy | 3 +++ jenkins-scripts/dsl/_configs_/OSRFUNIXBase.groovy | 1 - 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/_configs_/OSRFLinuxBase.groovy b/jenkins-scripts/dsl/_configs_/OSRFLinuxBase.groovy index acfa7dec7..ef80197ee 100644 --- a/jenkins-scripts/dsl/_configs_/OSRFLinuxBase.groovy +++ b/jenkins-scripts/dsl/_configs_/OSRFLinuxBase.groovy @@ -29,6 +29,9 @@ class OSRFLinuxBase // present allowEmpty(true) } + steps { + shell('sudo shutdown -r now') + } } } } diff --git a/jenkins-scripts/dsl/_configs_/OSRFUNIXBase.groovy b/jenkins-scripts/dsl/_configs_/OSRFUNIXBase.groovy index 8e9526d1d..37326b204 100644 --- a/jenkins-scripts/dsl/_configs_/OSRFUNIXBase.groovy +++ b/jenkins-scripts/dsl/_configs_/OSRFUNIXBase.groovy @@ -37,7 +37,6 @@ class OSRFUNIXBase extends OSRFBase [[ -d ./scripts ]] && rm -fr ./scripts git clone https://github.com/gazebo-tooling/release-tools scripts -b \$RTOOLS_BRANCH """.stripIndent()) - shell("sudo shutdown -r") } publishers { postBuildScripts { From e9547e99c29171eb4a8e292ab30ca209335023b1 Mon Sep 17 00:00:00 2001 From: Crola1702 Date: Thu, 18 May 2023 11:49:24 -0500 Subject: [PATCH 348/540] Test: shutdown step should retrigger the build version 4 Signed-off-by: Crola1702 --- jenkins-scripts/dsl/_configs_/OSRFBase.groovy | 5 +++++ jenkins-scripts/dsl/_configs_/OSRFLinuxBase.groovy | 6 +++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/jenkins-scripts/dsl/_configs_/OSRFBase.groovy b/jenkins-scripts/dsl/_configs_/OSRFBase.groovy index 7a0cf713c..67e82c59f 100644 --- a/jenkins-scripts/dsl/_configs_/OSRFBase.groovy +++ b/jenkins-scripts/dsl/_configs_/OSRFBase.groovy @@ -38,6 +38,11 @@ class OSRFBase } } publishers { + postBuildScripts { + steps { + shell('sudo shutdown -r now') + } + } configure { project -> // When a build fails because an agent disconnects, retry it once project / publishers / 'com.chikli.hudson.plugin.naginator.NaginatorPublisher' { diff --git a/jenkins-scripts/dsl/_configs_/OSRFLinuxBase.groovy b/jenkins-scripts/dsl/_configs_/OSRFLinuxBase.groovy index ef80197ee..be8811d47 100644 --- a/jenkins-scripts/dsl/_configs_/OSRFLinuxBase.groovy +++ b/jenkins-scripts/dsl/_configs_/OSRFLinuxBase.groovy @@ -29,9 +29,9 @@ class OSRFLinuxBase // present allowEmpty(true) } - steps { - shell('sudo shutdown -r now') - } + } + steps { + shell('sudo shutdown -r now') } } } From 1bfc93fa06da2f167835e8b1f99edf7204558365 Mon Sep 17 00:00:00 2001 From: Crola1702 Date: Thu, 18 May 2023 11:54:37 -0500 Subject: [PATCH 349/540] Test: shutdown step should retrigger the build version 5 Signed-off-by: Crola1702 --- jenkins-scripts/dsl/_configs_/OSRFBase.groovy | 5 ----- jenkins-scripts/dsl/_configs_/OSRFLinuxBase.groovy | 3 --- jenkins-scripts/dsl/_configs_/OSRFUNIXBase.groovy | 1 + 3 files changed, 1 insertion(+), 8 deletions(-) diff --git a/jenkins-scripts/dsl/_configs_/OSRFBase.groovy b/jenkins-scripts/dsl/_configs_/OSRFBase.groovy index 67e82c59f..7a0cf713c 100644 --- a/jenkins-scripts/dsl/_configs_/OSRFBase.groovy +++ b/jenkins-scripts/dsl/_configs_/OSRFBase.groovy @@ -38,11 +38,6 @@ class OSRFBase } } publishers { - postBuildScripts { - steps { - shell('sudo shutdown -r now') - } - } configure { project -> // When a build fails because an agent disconnects, retry it once project / publishers / 'com.chikli.hudson.plugin.naginator.NaginatorPublisher' { diff --git a/jenkins-scripts/dsl/_configs_/OSRFLinuxBase.groovy b/jenkins-scripts/dsl/_configs_/OSRFLinuxBase.groovy index be8811d47..acfa7dec7 100644 --- a/jenkins-scripts/dsl/_configs_/OSRFLinuxBase.groovy +++ b/jenkins-scripts/dsl/_configs_/OSRFLinuxBase.groovy @@ -30,9 +30,6 @@ class OSRFLinuxBase allowEmpty(true) } } - steps { - shell('sudo shutdown -r now') - } } } } diff --git a/jenkins-scripts/dsl/_configs_/OSRFUNIXBase.groovy b/jenkins-scripts/dsl/_configs_/OSRFUNIXBase.groovy index 37326b204..33ae6bb75 100644 --- a/jenkins-scripts/dsl/_configs_/OSRFUNIXBase.groovy +++ b/jenkins-scripts/dsl/_configs_/OSRFUNIXBase.groovy @@ -41,6 +41,7 @@ class OSRFUNIXBase extends OSRFBase publishers { postBuildScripts { steps{ + shell("""sudo shutdown -r""") conditionalSteps { condition { expression("(.)* gpu-nvidia (.)*",'${NODE_LABELS}') From fa7dac4addb9518ac26bf80bae441a4b7a08cc07 Mon Sep 17 00:00:00 2001 From: Crola1702 Date: Thu, 18 May 2023 11:57:53 -0500 Subject: [PATCH 350/540] Test: shutdown step should retrigger the build version 6 Signed-off-by: Crola1702 --- jenkins-scripts/dsl/_configs_/OSRFUNIXBase.groovy | 1 - 1 file changed, 1 deletion(-) diff --git a/jenkins-scripts/dsl/_configs_/OSRFUNIXBase.groovy b/jenkins-scripts/dsl/_configs_/OSRFUNIXBase.groovy index 33ae6bb75..37326b204 100644 --- a/jenkins-scripts/dsl/_configs_/OSRFUNIXBase.groovy +++ b/jenkins-scripts/dsl/_configs_/OSRFUNIXBase.groovy @@ -41,7 +41,6 @@ class OSRFUNIXBase extends OSRFBase publishers { postBuildScripts { steps{ - shell("""sudo shutdown -r""") conditionalSteps { condition { expression("(.)* gpu-nvidia (.)*",'${NODE_LABELS}') From b1bd5a2347bf362b330322fe2b525bb0a22a0c5c Mon Sep 17 00:00:00 2001 From: Crola1702 Date: Thu, 18 May 2023 18:51:22 -0500 Subject: [PATCH 351/540] Change Naginator Publisher creation in HelperRetryFailures script Signed-off-by: Crola1702 --- .../dsl/_configs_/HelperRetryFailures.groovy | 58 +++++++++++++++++++ jenkins-scripts/dsl/_configs_/OSRFBase.groovy | 17 +++--- .../dsl/_configs_/OSRFUNIXBase.groovy | 8 ++- .../dsl/_configs_/RetryFailures.groovy | 24 -------- 4 files changed, 71 insertions(+), 36 deletions(-) create mode 100644 jenkins-scripts/dsl/_configs_/HelperRetryFailures.groovy delete mode 100644 jenkins-scripts/dsl/_configs_/RetryFailures.groovy diff --git a/jenkins-scripts/dsl/_configs_/HelperRetryFailures.groovy b/jenkins-scripts/dsl/_configs_/HelperRetryFailures.groovy new file mode 100644 index 000000000..d3207d54d --- /dev/null +++ b/jenkins-scripts/dsl/_configs_/HelperRetryFailures.groovy @@ -0,0 +1,58 @@ +package _configs_ + +import javaposse.jobdsl.dsl.Job + +class HelperRetryFailures { + + /**@ + * Create method for Nagiator Publisher retry plugin + * @param job - Job to add the plugin to + * @param args - Map of arguments to pass to the plugin + * @param args.regexpForRerun - Regular expression to match against the console output + * @param args.checkRegexp - Whether to check the regular expression + * @param args.maxSchedule - Maximum number of times to retry the build + * @param args.delay - Delay in seconds to wait before retrying the build + */ + static void create(Job job, Map args) { + job.with { + publishers { + configure { project -> + def old_regexpForRerun = (project / publishers / 'com.chikli.hudson.plugin.naginator.NaginatorPublisher' / regexpForRerun).text() + def old_checkRegexp = (project / publishers / 'com.chikli.hudson.plugin.naginator.NaginatorPublisher' / checkRegexp).text() + def old_maxSchedule = (project / publishers / 'com.chikli.hudson.plugin.naginator.NaginatorPublisher' / maxSchedule).text() + def old_delay = (project / publishers / 'com.chikli.hudson.plugin.naginator.NaginatorPublisher' / delay(class: 'com.chikli.hudson.plugin.naginator.FixedDelay') / delay).text() + + // Convert the values to the correct type + old_checkRegexp = old_checkRegexp ? old_checkRegexp.toBoolean() : null + old_maxSchedule = old_maxSchedule ? old_maxSchedule.toInteger() : 0 + old_delay = old_delay ? old_delay.toInteger() : 0 + + + if (args.regexpForRerun) { + // If the old value exists, append the new value to it + def updated_regexpForRerun = old_regexpForRerun ? old_regexpForRerun + '|' + args.regexpForRerun : args.regexpForRerun + (project / publishers / 'com.chikli.hudson.plugin.naginator.NaginatorPublisher' / regexpForRerun).value = updated_regexpForRerun + } + + if (args.checkRegexp) { + // Get the previous value if it exists (that means it only has to be set once) + def updated_checkRegexp = old_checkRegexp ? old_checkRegexp : args.checkRegexp + (project / publishers / 'com.chikli.hudson.plugin.naginator.NaginatorPublisher' / checkRegexp).value = updated_checkRegexp + } + + if (args.maxSchedule) { + // Use the highest maxSchedule + def updated_maxSchedule = old_maxSchedule > args.maxSchedule ? old_maxSchedule : args.maxSchedule + (project / publishers / 'com.chikli.hudson.plugin.naginator.NaginatorPublisher' / maxSchedule).value = updated_maxSchedule + } + + if (args.delay) { + // Use the highest delay + def updated_delay = old_delay > args.delay ? old_delay : args.delay + (project / publishers / 'com.chikli.hudson.plugin.naginator.NaginatorPublisher' / delay(class: 'com.chikli.hudson.plugin.naginator.FixedDelay') / delay).value = updated_delay + } + } + } + } + } +} diff --git a/jenkins-scripts/dsl/_configs_/OSRFBase.groovy b/jenkins-scripts/dsl/_configs_/OSRFBase.groovy index 7a0cf713c..c79e92d59 100644 --- a/jenkins-scripts/dsl/_configs_/OSRFBase.groovy +++ b/jenkins-scripts/dsl/_configs_/OSRFBase.groovy @@ -37,16 +37,13 @@ class OSRFBase systemGroovyCommand("build.setDescription('RTOOLS_BRANCH: ' + build.buildVariableResolver.resolve('RTOOLS_BRANCH'));") } } - publishers { - configure { project -> - // When a build fails because an agent disconnects, retry it once - project / publishers / 'com.chikli.hudson.plugin.naginator.NaginatorPublisher' { - regexpForRerun("java.nio.channels.ClosedChannelException") - checkRegexp(true) - maxSchedule(1) - } - } - } + + // Create the naginator retry tag + HelperRetryFailures.create(job, [ + regexpForRerun: "java.nio.channels.ClosedChannelException", + checkRegexp: true, + maxSchedule: 1 + ]) } } } diff --git a/jenkins-scripts/dsl/_configs_/OSRFUNIXBase.groovy b/jenkins-scripts/dsl/_configs_/OSRFUNIXBase.groovy index 37326b204..f7462aa49 100644 --- a/jenkins-scripts/dsl/_configs_/OSRFUNIXBase.groovy +++ b/jenkins-scripts/dsl/_configs_/OSRFUNIXBase.groovy @@ -82,7 +82,11 @@ class OSRFUNIXBase extends OSRFBase } } - // Add the new regex to naginator parameter - RetryFailures.update_retry_parameters(job, 'nvml error driver/library version mismatch', 70) + // Add the new regex to naginator tag + // There is no need to specify checkRegexp and maxSchedule because they are the default values + HelperRetryFailures.create(job, [ + regexpForRerun: "nvml error driver/library version mismatch", + delay: 70 + ]) } } \ No newline at end of file diff --git a/jenkins-scripts/dsl/_configs_/RetryFailures.groovy b/jenkins-scripts/dsl/_configs_/RetryFailures.groovy deleted file mode 100644 index eb3ecaf28..000000000 --- a/jenkins-scripts/dsl/_configs_/RetryFailures.groovy +++ /dev/null @@ -1,24 +0,0 @@ -package _configs_ - -import javaposse.jobdsl.dsl.Job - -class RetryFailures { - static void update_retry_parameters(Job job, String new_regexpForRerun, int new_delay = 0, boolean new_checkRegexp = true, int new_maxSchedule = 1) { - - job.with { - publishers { - configure { project -> - def new_retry_regex = (project / publishers / 'com.chikli.hudson.plugin.naginator.NaginatorPublisher' / regexpForRerun).text() + "|" + new_regexpForRerun - - (project / publishers / 'com.chikli.hudson.plugin.naginator.NaginatorPublisher' / regexpForRerun).value = new_retry_regex - (project / publishers / 'com.chikli.hudson.plugin.naginator.NaginatorPublisher' / checkRegexp).value = new_checkRegexp - (project / publishers / 'com.chikli.hudson.plugin.naginator.NaginatorPublisher' / maxSchedule).value = new_maxSchedule - - if (new_delay) { - (project / publishers / 'com.chikli.hudson.plugin.naginator.NaginatorPublisher' / delay(class: "com.chikli.hudson.plugin.naginator.FixedDelay") / delay).value = new_delay - } - } - } - } - } -} From c2336847adcafc82d80febfb8661fb30713821cc Mon Sep 17 00:00:00 2001 From: Crola1702 Date: Thu, 18 May 2023 19:04:47 -0500 Subject: [PATCH 352/540] Fail if maxSchedule and checkRegexp values are set Signed-off-by: Crola1702 --- .../dsl/_configs_/HelperRetryFailures.groovy | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/jenkins-scripts/dsl/_configs_/HelperRetryFailures.groovy b/jenkins-scripts/dsl/_configs_/HelperRetryFailures.groovy index d3207d54d..f1e44197f 100644 --- a/jenkins-scripts/dsl/_configs_/HelperRetryFailures.groovy +++ b/jenkins-scripts/dsl/_configs_/HelperRetryFailures.groovy @@ -35,15 +35,19 @@ class HelperRetryFailures { } if (args.checkRegexp) { - // Get the previous value if it exists (that means it only has to be set once) - def updated_checkRegexp = old_checkRegexp ? old_checkRegexp : args.checkRegexp - (project / publishers / 'com.chikli.hudson.plugin.naginator.NaginatorPublisher' / checkRegexp).value = updated_checkRegexp + // Fail if old and new checkRegexp are both set and different + if (old_checkRegexp && old_checkRegexp != args.checkRegexp) { + throw new Exception("checkRegexp is already set to ${old_checkRegexp} and cannot be changed to ${args.checkRegexp}") + } + (project / publishers / 'com.chikli.hudson.plugin.naginator.NaginatorPublisher' / checkRegexp).value = args.checkRegexp } if (args.maxSchedule) { - // Use the highest maxSchedule - def updated_maxSchedule = old_maxSchedule > args.maxSchedule ? old_maxSchedule : args.maxSchedule - (project / publishers / 'com.chikli.hudson.plugin.naginator.NaginatorPublisher' / maxSchedule).value = updated_maxSchedule + // Fail if old and new maxSchedule are both set and different + if (old_maxSchedule && old_maxSchedule != args.maxSchedule) { + throw new Exception("maxSchedule is already set to ${old_maxSchedule} and cannot be changed to ${args.maxSchedule}") + } + (project / publishers / 'com.chikli.hudson.plugin.naginator.NaginatorPublisher' / maxSchedule).value = args.maxSchedule } if (args.delay) { From d3bb9d30dd52381ccc858b8ac76f64f11a008cce Mon Sep 17 00:00:00 2001 From: Crola1702 Date: Fri, 19 May 2023 10:01:20 -0500 Subject: [PATCH 353/540] Refactor naginator_root node instance Signed-off-by: Crola1702 --- .../dsl/_configs_/HelperRetryFailures.groovy | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/jenkins-scripts/dsl/_configs_/HelperRetryFailures.groovy b/jenkins-scripts/dsl/_configs_/HelperRetryFailures.groovy index f1e44197f..74fa268aa 100644 --- a/jenkins-scripts/dsl/_configs_/HelperRetryFailures.groovy +++ b/jenkins-scripts/dsl/_configs_/HelperRetryFailures.groovy @@ -5,7 +5,7 @@ import javaposse.jobdsl.dsl.Job class HelperRetryFailures { /**@ - * Create method for Nagiator Publisher retry plugin + * Create method for Naginator Publisher retry plugin * @param job - Job to add the plugin to * @param args - Map of arguments to pass to the plugin * @param args.regexpForRerun - Regular expression to match against the console output @@ -17,10 +17,12 @@ class HelperRetryFailures { job.with { publishers { configure { project -> - def old_regexpForRerun = (project / publishers / 'com.chikli.hudson.plugin.naginator.NaginatorPublisher' / regexpForRerun).text() - def old_checkRegexp = (project / publishers / 'com.chikli.hudson.plugin.naginator.NaginatorPublisher' / checkRegexp).text() - def old_maxSchedule = (project / publishers / 'com.chikli.hudson.plugin.naginator.NaginatorPublisher' / maxSchedule).text() - def old_delay = (project / publishers / 'com.chikli.hudson.plugin.naginator.NaginatorPublisher' / delay(class: 'com.chikli.hudson.plugin.naginator.FixedDelay') / delay).text() + def naginator_root = project / publishers / 'com.chikli.hudson.plugin.naginator.NaginatorPublisher' + + def old_regexpForRerun = (naginator_root / regexpForRerun).text() + def old_checkRegexp = (naginator_root / checkRegexp).text() + def old_maxSchedule = (naginator_root / maxSchedule).text() + def old_delay = (naginator_root / delay(class: 'com.chikli.hudson.plugin.naginator.FixedDelay') / delay).text() // Convert the values to the correct type old_checkRegexp = old_checkRegexp ? old_checkRegexp.toBoolean() : null @@ -31,7 +33,7 @@ class HelperRetryFailures { if (args.regexpForRerun) { // If the old value exists, append the new value to it def updated_regexpForRerun = old_regexpForRerun ? old_regexpForRerun + '|' + args.regexpForRerun : args.regexpForRerun - (project / publishers / 'com.chikli.hudson.plugin.naginator.NaginatorPublisher' / regexpForRerun).value = updated_regexpForRerun + (naginator_root / regexpForRerun).value = updated_regexpForRerun } if (args.checkRegexp) { @@ -39,7 +41,7 @@ class HelperRetryFailures { if (old_checkRegexp && old_checkRegexp != args.checkRegexp) { throw new Exception("checkRegexp is already set to ${old_checkRegexp} and cannot be changed to ${args.checkRegexp}") } - (project / publishers / 'com.chikli.hudson.plugin.naginator.NaginatorPublisher' / checkRegexp).value = args.checkRegexp + (naginator_root / checkRegexp).value = args.checkRegexp } if (args.maxSchedule) { @@ -47,13 +49,13 @@ class HelperRetryFailures { if (old_maxSchedule && old_maxSchedule != args.maxSchedule) { throw new Exception("maxSchedule is already set to ${old_maxSchedule} and cannot be changed to ${args.maxSchedule}") } - (project / publishers / 'com.chikli.hudson.plugin.naginator.NaginatorPublisher' / maxSchedule).value = args.maxSchedule + (naginator_root / maxSchedule).value = args.maxSchedule } if (args.delay) { // Use the highest delay def updated_delay = old_delay > args.delay ? old_delay : args.delay - (project / publishers / 'com.chikli.hudson.plugin.naginator.NaginatorPublisher' / delay(class: 'com.chikli.hudson.plugin.naginator.FixedDelay') / delay).value = updated_delay + (naginator_root / delay(class: 'com.chikli.hudson.plugin.naginator.FixedDelay') / delay).value = updated_delay } } } From 44577a95a8b3ebc27dd49d9c4033db04855ca934 Mon Sep 17 00:00:00 2001 From: Crola1702 Date: Fri, 19 May 2023 10:05:55 -0500 Subject: [PATCH 354/540] Improve method parameters documentation Signed-off-by: Crola1702 --- .../dsl/_configs_/HelperRetryFailures.groovy | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/jenkins-scripts/dsl/_configs_/HelperRetryFailures.groovy b/jenkins-scripts/dsl/_configs_/HelperRetryFailures.groovy index 74fa268aa..0a51ad3c5 100644 --- a/jenkins-scripts/dsl/_configs_/HelperRetryFailures.groovy +++ b/jenkins-scripts/dsl/_configs_/HelperRetryFailures.groovy @@ -5,13 +5,15 @@ import javaposse.jobdsl.dsl.Job class HelperRetryFailures { /**@ - * Create method for Naginator Publisher retry plugin - * @param job - Job to add the plugin to - * @param args - Map of arguments to pass to the plugin - * @param args.regexpForRerun - Regular expression to match against the console output - * @param args.checkRegexp - Whether to check the regular expression - * @param args.maxSchedule - Maximum number of times to retry the build - * @param args.delay - Delay in seconds to wait before retrying the build + * Creates a Naginator Publisher retry plugin for a given job. + * + * @param job - The job to add the plugin to. + * @param args - A map of arguments to pass to the plugin: + * - args.regexpForRerun: Regular expression to match against the console output (new value is appended to the old value if it exists) + * - args.checkRegexp: Whether to check the regular expression (true/false) + * - args.maxSchedule: Maximum number of times to retry the build + * - args.delay: Delay in seconds to wait before retrying the build (highest value between old and new is used) + * @throws Exception if there checkRegexp or maxSchedule are already set and different */ static void create(Job job, Map args) { job.with { From e60fa9bb4d558dbbb09ef6477414c1f19bf4909c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Crist=C3=B3bal=20Arroyo?= Date: Fri, 19 May 2023 12:44:18 -0500 Subject: [PATCH 355/540] Apply suggestions from code review Co-authored-by: Jose Luis Rivero --- jenkins-scripts/dsl/_configs_/HelperRetryFailures.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jenkins-scripts/dsl/_configs_/HelperRetryFailures.groovy b/jenkins-scripts/dsl/_configs_/HelperRetryFailures.groovy index 0a51ad3c5..4cc5e4835 100644 --- a/jenkins-scripts/dsl/_configs_/HelperRetryFailures.groovy +++ b/jenkins-scripts/dsl/_configs_/HelperRetryFailures.groovy @@ -41,7 +41,7 @@ class HelperRetryFailures { if (args.checkRegexp) { // Fail if old and new checkRegexp are both set and different if (old_checkRegexp && old_checkRegexp != args.checkRegexp) { - throw new Exception("checkRegexp is already set to ${old_checkRegexp} and cannot be changed to ${args.checkRegexp}") + throw new Exception("checkRegexp is already set to ${old_checkRegexp} and cannot be changed to ${args.checkRegexp}. Both values needs to be the same.") } (naginator_root / checkRegexp).value = args.checkRegexp } @@ -49,7 +49,7 @@ class HelperRetryFailures { if (args.maxSchedule) { // Fail if old and new maxSchedule are both set and different if (old_maxSchedule && old_maxSchedule != args.maxSchedule) { - throw new Exception("maxSchedule is already set to ${old_maxSchedule} and cannot be changed to ${args.maxSchedule}") + throw new Exception("maxSchedule is already set to ${old_maxSchedule} and cannot be changed to ${args.maxSchedule}. Both values needs to be the same.") } (naginator_root / maxSchedule).value = args.maxSchedule } From f7423021aced46d9c6f93b54cf1d7a890e612102 Mon Sep 17 00:00:00 2001 From: claraberendsen Date: Mon, 22 May 2023 11:37:28 -0400 Subject: [PATCH 356/540] remove legacy reference to extreme in docker cleanup process --- jenkins-scripts/docker/lib/boilerplate_prepare.sh | 7 ------- 1 file changed, 7 deletions(-) diff --git a/jenkins-scripts/docker/lib/boilerplate_prepare.sh b/jenkins-scripts/docker/lib/boilerplate_prepare.sh index dccf34355..006d4a1a7 100755 --- a/jenkins-scripts/docker/lib/boilerplate_prepare.sh +++ b/jenkins-scripts/docker/lib/boilerplate_prepare.sh @@ -43,13 +43,6 @@ if [[ -z ${DO_NOT_CHECK_DOCKER_DISK_USAGE} ]]; then source ${SCRIPT_DIR}/lib/docker_cleanup.sh high fi - # if not enough, try to clean up build/ directories - PERCENT_DISK_USED=$(df -h ${docker_device} | grep ${docker_device} | sed 's:.* \([0-9]*\)%.*:\1:') - if [[ $PERCENT_DISK_USED -gt 85 ]]; then - echo "Space left is low again: ${PERCENT_DISK_USED}% used" - echo "Clean up the whole cache was not enough. Look for build/ directories bigger than ${MAX_SIZE_FOR_BUILD_DIRS}:" - source ${SCRIPT_DIR}/lib/docker_cleanup.sh extreme - fi fi # Timing From 31369dc4c6c5d99cff6f72ba7b45f49b8ab1873d Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Mon, 22 May 2023 19:04:23 +0200 Subject: [PATCH 357/540] Improve nightly node selection (#924) * Use meaningful names for nightly values defined * Filter nodes to select for nightly assignments * No gpu nodes * No test nodes * No existing nodes with one nightly label Signed-off-by: Jose Luis Rivero --- .../tools/label-assignment-backstop.groovy | 33 +++++++++++++------ 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/jenkins-scripts/tools/label-assignment-backstop.groovy b/jenkins-scripts/tools/label-assignment-backstop.groovy index fe2ee3c35..cd80915e3 100644 --- a/jenkins-scripts/tools/label-assignment-backstop.groovy +++ b/jenkins-scripts/tools/label-assignment-backstop.groovy @@ -16,24 +16,28 @@ import hudson.model.Label; The first field must be a single label (no spaces) but the second may be an arbitrarily complex label expression like "(docker && linux) && !armhf" */ +def nightly_label_prefix = "linux-nightly" def exactly_one_labels = [ - ["linux-nightly-focal", "docker"], - ["linux-nightly-jammy", "docker"], + ["${nightly_label_prefix}-focal", "docker"], + ["${nightly_label_prefix}-jammy", "docker"], ] for (tup in exactly_one_labels) { label_nodes = Label.get(tup[0]).getNodes().findAll { it.toComputer().isOnline() } + nightly_label = tup[0] + pool_label = tup[1] if (label_nodes.size() == 1) { - println(tup[0] + " is currently applied to " + label_nodes[0].name) + println("${nightly_label} is currently applied to " + label_nodes[0].name) continue } - // return unstable if the labels are not set correctly before running this + // return unstable if the labels are not set correctly before running the + // process to assign the new nightly labels. println("MARK_AS_UNSTABLE") if (label_nodes.size() > 1) { - println("WARNING: Too many online nodes with the label " + tup[0]) + println("WARNING: Too many online nodes with the label ${nightly_label}") for (node in label_nodes) { println(" " + node.name) } @@ -41,16 +45,25 @@ for (tup in exactly_one_labels) { } if (label_nodes.size() < 1) { - println("No online host currently has the label " + tup[0]) - println("Appointing a node from the configured pool matching '" + tup[1] + "'") - node_pool = Label.get(tup[1]).getNodes().findAll { it.toComputer().isOnline() } + println("No online host currently has the label ${nightly_label}") + println("Appointing a node from the configured pool matching '${pool_label}'") + + def node_pool = Jenkins.instance.nodes.findAll { node -> + node.computer.online && + node.getLabelString().contains(pool_label) && + !node.getLabelString().contains(nightly_label_prefix) && + !node.getLabelString().contains("gpu-nvidia") && + !node.getLabelString().contains("test-instance") + } + if (node_pool.size() <= 0) { - println("WARNING: Pool of '" + tup[1] + "' machines for " + tup[0] + " is empty!") + println("WARNING: Pool of '${pool_label}' machines for ${nightly_label} is empty!") } + // Pick a random node from the pool to receive the label. appointed_node = node_pool[(new Random()).nextInt(node_pool.size())] new_label_string = appointed_node.getAssignedLabels().join(" ") - new_label_string = tup[0] + " " + new_label_string + new_label_string = nightly_label + " " + new_label_string appointed_node.setLabelString(new_label_string) appointed_node.save() println("Added label to " + appointed_node.name) From 8d02e99fff0c9e67193e91c5da8b05960d70cf9c Mon Sep 17 00:00:00 2001 From: claraberendsen Date: Mon, 22 May 2023 14:09:53 -0400 Subject: [PATCH 358/540] remove comments reference to extreme option --- jenkins-scripts/docker/lib/docker_cleanup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/docker/lib/docker_cleanup.sh b/jenkins-scripts/docker/lib/docker_cleanup.sh index 5a556b914..9f1f3ad7e 100755 --- a/jenkins-scripts/docker/lib/docker_cleanup.sh +++ b/jenkins-scripts/docker/lib/docker_cleanup.sh @@ -28,7 +28,7 @@ docker_version="$(sudo docker version --format '{{.Server.APIVersion}}')" sudo docker system prune --all -f ;; *) - echo "Did not run docker cleaner, level $cleanup_level not recognized (accepted values low | medium | high | extreme!! " + echo "Did not run docker cleaner, level $cleanup_level not recognized (accepted values low | medium | high !! " exit 1 ;; esac From 2fc7692b2b978272dac446f21a1c537aa8675589 Mon Sep 17 00:00:00 2001 From: Clara Berendsen Date: Mon, 22 May 2023 14:53:41 -0400 Subject: [PATCH 359/540] fi typo Co-authored-by: Jose Luis Rivero --- jenkins-scripts/docker/lib/docker_cleanup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/docker/lib/docker_cleanup.sh b/jenkins-scripts/docker/lib/docker_cleanup.sh index 9f1f3ad7e..b39810912 100755 --- a/jenkins-scripts/docker/lib/docker_cleanup.sh +++ b/jenkins-scripts/docker/lib/docker_cleanup.sh @@ -28,7 +28,7 @@ docker_version="$(sudo docker version --format '{{.Server.APIVersion}}')" sudo docker system prune --all -f ;; *) - echo "Did not run docker cleaner, level $cleanup_level not recognized (accepted values low | medium | high !! " + echo "Did not run docker cleaner, level $cleanup_level not recognized (accepted values low | medium | high )" exit 1 ;; esac From a76c184846e51537504024723dce550820e98f4b Mon Sep 17 00:00:00 2001 From: Clara Berendsen Date: Mon, 22 May 2023 14:54:13 -0400 Subject: [PATCH 360/540] fix typos on boilerplate Co-authored-by: Jose Luis Rivero --- jenkins-scripts/docker/lib/boilerplate_prepare.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/jenkins-scripts/docker/lib/boilerplate_prepare.sh b/jenkins-scripts/docker/lib/boilerplate_prepare.sh index 006d4a1a7..03b1d7cf0 100755 --- a/jenkins-scripts/docker/lib/boilerplate_prepare.sh +++ b/jenkins-scripts/docker/lib/boilerplate_prepare.sh @@ -42,7 +42,6 @@ if [[ -z ${DO_NOT_CHECK_DOCKER_DISK_USAGE} ]]; then echo "Kill the whole docker cache !!" source ${SCRIPT_DIR}/lib/docker_cleanup.sh high fi - fi # Timing From 1194b1498fc087bc38903d0e0dc4b4eda491bb36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Crist=C3=B3bal=20Arroyo?= Date: Tue, 23 May 2023 08:31:32 -0500 Subject: [PATCH 361/540] Change huge-memory to large-memory (#932) Signed-off-by: Crola1702 --- docs/BUILDFARM_NODES.md | 2 +- jenkins-scripts/dsl/ignition.dsl | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/BUILDFARM_NODES.md b/docs/BUILDFARM_NODES.md index 7f1bb9914..e13b4d96e 100644 --- a/docs/BUILDFARM_NODES.md +++ b/docs/BUILDFARM_NODES.md @@ -11,7 +11,7 @@ used (can check this in the Jenkins UI) | -------- | ----------- | ------------ | | docker | Node has capabilities to run Docker CI (standard Linux CI) | Linux system with docker installed | | gpu-reliable | Node has a real GPU able to run simulation for Gazebo | Nvidia card and nvidia-docker installed on Linux | -| huge-memory | Node has enough RAM to run really demanding RAM compilations | Hardware has no less than 16Gb of RAM and can run abichecker on ign-physics | +| large-memory | Node has enough RAM to run really demanding RAM compilations | Hardware has no less than 16Gb of RAM and can run abichecker on ign-physics | | large-memory | Node has enough RAM to run non trivial compilations | Hardware has no less than 16GB of RAM on Linux | | linux-arm64 | Node has capabilities to run native arm64 code (mostly used in packaging) | Bare-metal ARM machine | | linux-armhf | Node has capabilities to run native armhf code (mostly used in packaging) | Bare-metal ARM machine | diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index 74c62ac80..c36397cdc 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -373,7 +373,7 @@ gz_software.each { gz_sw -> extra_str="" if (gz_sw == 'physics') { - label Globals.nontest_label("huge-memory") + label Globals.nontest_label("large-memory") // on ARM native nodes in buildfarm we need to restrict to 1 the // compilation threads to avoid OOM killer extra_str += '\nif [ $(uname -m) = "aarch64" ]; then export MAKE_JOBS=1; fi' @@ -421,7 +421,7 @@ gz_software.each { gz_sw -> gz_ci_any_job.with { if (gz_sw == 'physics') - label Globals.nontest_label("huge-memory") + label Globals.nontest_label("large-memory") steps { @@ -529,7 +529,7 @@ void generate_ci_job(gz_ci_job, gz_sw, branch, distro, arch, gz_ci_job.with { if (gz_sw == 'physics') - label Globals.nontest_label("huge-memory") + label Globals.nontest_label("large-memory") steps { shell("""\ From ead55392975de900f7f1ae9dcc5e772fe011f4a8 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 23 May 2023 16:07:40 +0200 Subject: [PATCH 362/540] Remove Windows CI from Gazebo Classic (#933) Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/gazebo.dsl | 38 ----- .../gazebo-default-devel-windows7-amd64.bat | 11 -- jenkins-scripts/lib/gazebo-base-windows.bat | 148 ------------------ .../lib/gazebo9_10-base-windows.bat | 128 --------------- 4 files changed, 325 deletions(-) delete mode 100644 jenkins-scripts/gazebo-default-devel-windows7-amd64.bat delete mode 100644 jenkins-scripts/lib/gazebo-base-windows.bat delete mode 100644 jenkins-scripts/lib/gazebo9_10-base-windows.bat diff --git a/jenkins-scripts/dsl/gazebo.dsl b/jenkins-scripts/dsl/gazebo.dsl index e71f80c07..357c860c0 100644 --- a/jenkins-scripts/dsl/gazebo.dsl +++ b/jenkins-scripts/dsl/gazebo.dsl @@ -509,47 +509,9 @@ all_branches.each { branch -> Globals.extra_emails = "" } -// -------------------------------------------------------------- -// WINDOWS: CI job - -// 1. any - String ci_build_any_job_name_win7 = "gazebo-ci-pr_any-windows7-amd64" - def gazebo_win_ci_any_job = job(ci_build_any_job_name_win7) - OSRFWinCompilationAnyGitHub.create(gazebo_win_ci_any_job, "gazebosim/gazebo-classic") - gazebo_win_ci_any_job.with - { - steps { - batchFile("""\ - call "./scripts/jenkins-scripts/gazebo-default-devel-windows7-amd64.bat" - """.stripIndent()) - } - } - -// 2. default / @ SCM/Daily -all_branches = gazebo_supported_branches -all_branches.each { branch -> - def gazebo_win_ci_job = job("gazebo-ci-${branch}-windows7-amd64") - OSRFWinCompilation.create(gazebo_win_ci_job) - OSRFGitHub.create(gazebo_win_ci_job, "gazebosim/gazebo-classic", branch) - - gazebo_win_ci_job.with - { - triggers { - scm('@daily') - } - - steps { - batchFile("""\ - call "./scripts/jenkins-scripts/gazebo-default-devel-windows7-amd64.bat" - """.stripIndent()) - } - } -} - // -------------------------------------------------------------- // Create the main CI work flow job def gazebo_ci_main = pipelineJob("gazebo-ci-manual_any") OSRFCIWorkFlowMultiAnyGitHub.create(gazebo_ci_main, [ci_build_any_job_name_linux, - ci_build_any_job_name_win7, ci_build_any_job_name_brew]) diff --git a/jenkins-scripts/gazebo-default-devel-windows7-amd64.bat b/jenkins-scripts/gazebo-default-devel-windows7-amd64.bat deleted file mode 100644 index d44d52a33..000000000 --- a/jenkins-scripts/gazebo-default-devel-windows7-amd64.bat +++ /dev/null @@ -1,11 +0,0 @@ -set SCRIPT_DIR=%~dp0 -set PLATFORM_TO_BUILD=amd64 -set VCS_DIRECTORY=gazebo-classic - -for /f %%i in ('python "%SCRIPT_DIR%\tools\detect_cmake_major_version.py" "%WORKSPACE%\%VCS_DIRECTORY%\CMakeLists.txt"') do set GAZEBO_MAJOR_VERSION=%%i - -if %GAZEBO_MAJOR_VERSION% GEQ 11 ( - call "%SCRIPT_DIR%/lib/gazebo-base-windows.bat" -) else ( - call "%SCRIPT_DIR%/lib/gazebo9_10-base-windows.bat" -) diff --git a/jenkins-scripts/lib/gazebo-base-windows.bat b/jenkins-scripts/lib/gazebo-base-windows.bat deleted file mode 100644 index b221ceee4..000000000 --- a/jenkins-scripts/lib/gazebo-base-windows.bat +++ /dev/null @@ -1,148 +0,0 @@ -set win_lib=%SCRIPT_DIR%\lib\windows_library.bat -set LOCAL_WS=%WORKSPACE%\ws - -:: Let's keep the workspace by default -set KEEP_WORKSPACE=0 - -:: Call vcvarsall and all the friends -echo # BEGIN SECTION: configure the MSVC compiler -call %win_lib% :configure_msvc2019_compiler -echo # END SECTION - -:: avoid conflicts with vcpkg packages -call %win_lib% :disable_vcpkg_integration -call %win_lib% :remove_vcpkg_package boost-uninstall -call %win_lib% :remove_vcpkg_package gdal -call %win_lib% :remove_vcpkg_package ogre -call %win_lib% :remove_vcpkg_package protobuf -call %win_lib% :remove_vcpkg_package qt5 -call %win_lib% :remove_vcpkg_package qwt - -:: msgs5 needs tinyxml2 which was never in the win32 deps but it can be used -:: from vcpkg. -call %win_lib% :install_vcpkg_package tinyxml2 - -:: IF exist %LOCAL_WS% ( rmdir /s /q %LOCAL_WS% ) || goto %win_lib% :error -:: reusing the workspace - -IF NOT exist %LOCAL_WS% ( - -mkdir %LOCAL_WS% -cd %LOCAL_WS% -set WORKSPACE_INSTALL_DIR=%LOCAL_WS%\install -mkdir %WORKSPACE_INSTALL_DIR% - -echo # BEGIN SECTION: downloading gazebo dependencies and unip -call %win_lib% :download_7za -call %win_lib% :download_unzip_install boost_1_67_0.zip -call %win_lib% :download_unzip_install bzip2-1.0.6-vc12-x64-release-debug.zip -call %win_lib% :download_unzip_install curl-7.57.0-vc15-x64-dll-MD.zip -call %win_lib% :download_unzip_install dlfcn-win32-vc15-x64-dll-MD.zip -call %win_lib% :download_unzip_install FreeImage3180Win32Win64.zip -call %win_lib% :download_unzip_install freetype-2.4.0-vc12-x64-release-debug.zip -call %win_lib% :download_unzip_install jsoncpp-1.8.4-vc15-x64-dll-MD.zip -call %win_lib% :download_unzip_install libyaml-0.1.7-vc15-x64-md.zip -call %win_lib% :download_unzip_install libzip-1.4.0_zlip-1.2.11_vc15-x64-dll-MD.zip -call %win_lib% :download_unzip_install protobuf-3.4.1-vc15-x64-dll-MD.zip -call %win_lib% :download_unzip_install ogre-sdk-1.10.12-vc15-x64.zip -call %win_lib% :download_unzip_install qt-opensource-windows-x86-msvc2015_64-5.7.0.zip -call %win_lib% :download_unzip_install qwt_6.1.2~osrf_qt5.zip -call %win_lib% :download_unzip_install tbb43_20141023oss_win.zip -call %win_lib% :download_unzip_install zziplib-0.13.62-vc12-x64-release-debug.zip - -call %win_lib% :wget http://mirrors.ukfast.co.uk/sites/qt.io/official_releases/jom/jom_1_1_3.zip jom.zip -call %win_lib% :unzip_7za jom.zip - -echo # END SECTION -) ELSE ( - echo # BEGIN SECTION: reusing workspace - :: Remove gazebo copy - IF EXIST %LOCAL_WS%\gazebo ( rmdir /s /q %LOCAL_WS%\gazebo ) || goto :error - IF EXIST %LOCAL_WS%\gazebo-classic ( rmdir /s /q %LOCAL_WS%\gazebo-classic ) || goto :error - echo # END SECTION -) - -echo # BEGIN SECTION: compile and install ign-fuel-tools -set IGN_TRANSPORT_DIR=%WORKSPACE%\ign-fuel-tools -if EXIST %IGN_TRANSPORT_DIR% ( rmdir /s /q %IGN_TRANSPORT_DIR% ) -git clone https://github.com/gazebosim/gz-fuel-tools %IGN_TRANSPORT_DIR% -b ign-fuel-tools4 -set VCS_DIRECTORY=ign-fuel-tools -set KEEP_WORKSPACE=TRUE -set ENABLE_TESTS=FALSE -set BUILD_TYPE=Release -call "%SCRIPT_DIR%/lib/generic-default-devel-windows.bat" -echo # END SECTION - -echo # BEGIN SECTION: compile and install ign-transport -set IGN_TRANSPORT_DIR=%WORKSPACE%\ign-transport -if EXIST %IGN_TRANSPORT_DIR% ( rmdir /s /q %IGN_TRANSPORT_DIR% ) -git clone https://github.com/gazebosim/gz-transport %IGN_TRANSPORT_DIR% -b ign-transport8 -set VCS_DIRECTORY=ign-transport -set KEEP_WORKSPACE=TRUE -set ENABLE_TESTS=FALSE -set BUILD_TYPE=Release -call "%SCRIPT_DIR%/lib/generic-default-devel-windows.bat" -echo # END SECTION - -echo # BEGIN SECTION: compile and install sdformat -set SDFORMAT_DIR=%WORKSPACE%\sdformat -if EXIST %SDFORMAT_DIR% ( rmdir /s /q %SDFORMAT_DIR% ) -git clone https://github.com/gazebosim/sdformat %SDFORMAT_DIR% -b sdf9 -set VCS_DIRECTORY=sdformat -set KEEP_WORKSPACE=TRUE -set ENABLE_TESTS=FALSE -set BUILD_TYPE=Release -call "%SCRIPT_DIR%/lib/generic-default-devel-windows.bat" -echo # END SECTION - -echo # BEGIN SECTION: overwrite, since sdformat downloads boost_1.67.0_any_variant_version.zip -xcopy %WORKSPACE_INSTALL_DIR%\boost_1_67_0\boost %WORKSPACE_INSTALL_DIR%\include\boost /s /i /e /y > xcopy.log -xcopy %WORKSPACE_INSTALL_DIR%\boost_1_67_0\lib64-msvc-14.1 %WORKSPACE_INSTALL_DIR%\lib /s /i /e /y > xcopy.log -echo # END SECTION - -echo # BEGIN SECTION: copy gazebo sources to workspace -:: Note that your jenkins job should put source in %WORKSPACE%/gazebo-classic -xcopy %WORKSPACE%\gazebo-classic %LOCAL_WS%\gazebo-classic /s /i /e > xcopy.log -cd %LOCAL_WS%\gazebo-classic -echo # END SECTION - -echo # BEGIN SECTION: configure gazebo -mkdir build -cd build -call "..\configure.bat" Release %BITNESS% || goto %win_lib% :error -echo # END SECTION - -echo # BEGIN SECTION: compiling gazebo -copy %LOCAL_WS%\jom.exe . -jom -j%MAKE_JOBS% || goto :error -echo # END SECTION - -echo # BEGIN SECTION: compiling test suite -jom -j%MAKE_JOBS% tests || goto :error -echo # END SECTION - -echo # BEGIN SECTION: run tests -:: nmake test is not working test/ directory exists and nmake is not able to handle it. -ctest -C "%BUILD_TYPE%" --force-new-ctest-process -VV -echo # END SECTION - -echo # BEGIN SECTION: export testing results -rmdir /q /s %TEST_RESULT_PATH% -if exist %TEST_RESULT_PATH_LEGACY% ( rmdir /q /s %TEST_RESULT_PATH_LEGACY% ) -mkdir %WORKSPACE%\build\ -xcopy test_results %TEST_RESULT_PATH% /s /i /e || goto :error -xcopy %TEST_RESULT_PATH% %TEST_RESULT_PATH_LEGACY% /s /e /i -echo # END SECTION - -if NOT DEFINED KEEP_WORKSPACE ( - echo # BEGIN SECTION: clean up workspace - cd %WORKSPACE% - rmdir /s /q %LOCAL_WS% || goto :error - echo # END SECTION -) - -goto :EOF - -:error -echo "The program is stopping with errors! Check the log" -exit /b %errorlevel% diff --git a/jenkins-scripts/lib/gazebo9_10-base-windows.bat b/jenkins-scripts/lib/gazebo9_10-base-windows.bat deleted file mode 100644 index 63c9ba678..000000000 --- a/jenkins-scripts/lib/gazebo9_10-base-windows.bat +++ /dev/null @@ -1,128 +0,0 @@ -set win_lib=%SCRIPT_DIR%\lib\windows_library.bat -set LOCAL_WS=%WORKSPACE%\ws - -:: Let's keep the workspace by default -set KEEP_WORKSPACE=0 - -:: Call vcvarsall and all the friends -echo # BEGIN SECTION: configure the MSVC compiler -call %win_lib% :configure_msvc2019_compiler -echo # END SECTION - -:: avoid conflicts with vcpkg packages -call %win_lib% :disable_vcpkg_integration -call %win_lib% :remove_vcpkg_package protobuf -call %win_lib% :remove_vcpkg_package qt5 -call %win_lib% :remove_vcpkg_package qwt - -:: IF exist %LOCAL_WS% ( rmdir /s /q %LOCAL_WS% ) || goto %win_lib% :error -:: reusing the workspace - -IF NOT exist %LOCAL_WS% ( - -mkdir %LOCAL_WS% -cd %LOCAL_WS% -set WORKSPACE_INSTALL_DIR=%LOCAL_WS%\install -mkdir %WORKSPACE_INSTALL_DIR% - -echo # BEGIN SECTION: downloading gazebo dependencies and unip -call %win_lib% :download_7za -call %win_lib% :download_unzip_install dlfcn-win32-vc15-x64-dll-MD.zip -call %win_lib% :download_unzip_install curl-7.57.0-vc15-x64-dll-MD.zip -call %win_lib% :download_unzip_install protobuf-3.4.1-vc15-x64-dll-MD.zip -call %win_lib% :download_unzip_install libzip-1.4.0_zlip-1.2.11_vc15-x64-dll-MD.zip -call %win_lib% :download_unzip_install FreeImage3180Win32Win64.zip -call %win_lib% :download_unzip_install boost_1_67_0.zip -call %win_lib% :download_unzip_install ogre-sdk-1.10.12-vc15-x64.zip -call %win_lib% :download_unzip_install qt-opensource-windows-x86-msvc2015_64-5.7.0.zip -call %win_lib% :download_unzip_install bzip2-1.0.6-vc12-x64-release-debug.zip -call %win_lib% :download_unzip_install freetype-2.4.0-vc12-x64-release-debug.zip -call %win_lib% :download_unzip_install qwt_6.1.2~osrf_qt5.zip -call %win_lib% :download_unzip_install tbb43_20141023oss_win.zip -call %win_lib% :download_unzip_install zziplib-0.13.62-vc12-x64-release-debug.zip - -call %win_lib% :wget http://mirrors.ukfast.co.uk/sites/qt.io/official_releases/jom/jom_1_1_3.zip jom.zip -call %win_lib% :unzip_7za jom.zip - -echo # END SECTION -) ELSE ( - echo # BEGIN SECTION: reusing workspace - :: Remove gazebo copy - IF EXIST %LOCAL_WS%\gazebo ( rmdir /s /q %LOCAL_WS%\gazebo ) || goto :error - IF EXIST %LOCAL_WS%\gazebo-classic ( rmdir /s /q %LOCAL_WS%\gazebo-classic ) || goto :error - echo # END SECTION -) - -echo # BEGIN SECTION: compile and install ign-transport -set IGN_TRANSPORT_DIR=%WORKSPACE%\ign-transport -if EXIST %IGN_TRANSPORT_DIR% ( rmdir /s /q %IGN_TRANSPORT_DIR% ) -git clone https://github.com/gazebosim/gz-transport %IGN_TRANSPORT_DIR% -b ign-transport4 -set VCS_DIRECTORY=ign-transport -set KEEP_WORKSPACE=TRUE -set ENABLE_TESTS=FALSE -set BUILD_TYPE=Release -call "%SCRIPT_DIR%/lib/generic-default-devel-windows.bat" -echo # END SECTION - -echo # BEGIN SECTION: compile and install sdformat -set SDFORMAT_DIR=%WORKSPACE%\sdformat -if EXIST %SDFORMAT_DIR% ( rmdir /s /q %SDFORMAT_DIR% ) -git clone https://github.com/gazebosim/sdformat %SDFORMAT_DIR% -b sdf6 -set VCS_DIRECTORY=sdformat -set KEEP_WORKSPACE=TRUE -set ENABLE_TESTS=FALSE -set BUILD_TYPE=Release -call "%SCRIPT_DIR%/lib/generic-default-devel-windows.bat" -echo # END SECTION - -echo # BEGIN SECTION: overwrite, since sdformat downloads boost_1.67.0_any_variant_version.zip -xcopy %WORKSPACE_INSTALL_DIR%\boost_1_67_0\boost %WORKSPACE_INSTALL_DIR%\include\boost /s /i /e /y > xcopy.log -xcopy %WORKSPACE_INSTALL_DIR%\boost_1_67_0\lib64-msvc-14.1 %WORKSPACE_INSTALL_DIR%\lib /s /i /e /y > xcopy.log -echo # END SECTION - -echo # BEGIN SECTION: copy gazebo sources to workspace -:: Note that your jenkins job should put source in %WORKSPACE%/gazebo-classic -xcopy %WORKSPACE%\gazebo-classic %LOCAL_WS%\gazebo-classic /s /i /e > xcopy.log -cd %LOCAL_WS%\gazebo-classic -echo # END SECTION - -echo # BEGIN SECTION: configure gazebo -mkdir build -cd build -call "..\configure.bat" Release %BITNESS% || goto %win_lib% :error -echo # END SECTION - -echo # BEGIN SECTION: compiling gazebo -copy %LOCAL_WS%\jom.exe . -jom -j%MAKE_JOBS% || goto :error -echo # END SECTION - -echo # BEGIN SECTION: compiling test suite -jom -j%MAKE_JOBS% tests || goto :error -echo # END SECTION - -echo # BEGIN SECTION: run tests -:: nmake test is not working test/ directory exists and nmake is not able to handle it. -ctest -C "%BUILD_TYPE%" --force-new-ctest-process -VV -echo # END SECTION - -echo # BEGIN SECTION: export testing results -rmdir /q /s %TEST_RESULT_PATH% -if exist %TEST_RESULT_PATH_LEGACY% ( rmdir /q /s %TEST_RESULT_PATH_LEGACY% ) -mkdir %WORKSPACE%\build\ -xcopy test_results %TEST_RESULT_PATH% /s /i /e || goto :error -xcopy %TEST_RESULT_PATH% %TEST_RESULT_PATH_LEGACY% /s /e /i -echo # END SECTION - -if NOT DEFINED KEEP_WORKSPACE ( - echo # BEGIN SECTION: clean up workspace - cd %WORKSPACE% - rmdir /s /q %LOCAL_WS% || goto :error - echo # END SECTION -) - -goto :EOF - -:error -echo "The program is stopping with errors! Check the log" -exit /b %errorlevel% From 74db31d4ad37d4263f88c24d80b8ad86eb7c9268 Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Tue, 23 May 2023 07:08:43 -0700 Subject: [PATCH 363/540] Add brew sbin to PATH to fix doctor complaint (#931) Signed-off-by: Steve Peters --- jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash | 2 +- jenkins-scripts/lib/project-install-homebrew.bash | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash b/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash index d8fc2a912..df94a79b1 100644 --- a/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash +++ b/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash @@ -32,7 +32,7 @@ PROJECT_FORMULA=${PROJECT//[0-9]}$(\ export HOMEBREW_PREFIX=/usr/local export HOMEBREW_CELLAR=${HOMEBREW_PREFIX}/Cellar -export PATH=${HOMEBREW_PREFIX}/bin:$PATH +export PATH=${HOMEBREW_PREFIX}/bin:${HOMEBREW_PREFIX}/sbin:$PATH export PYTHONPATH=$PYTHONPATH:/usr/local/lib/python diff --git a/jenkins-scripts/lib/project-install-homebrew.bash b/jenkins-scripts/lib/project-install-homebrew.bash index cfdedf0d2..b9f35eaf5 100644 --- a/jenkins-scripts/lib/project-install-homebrew.bash +++ b/jenkins-scripts/lib/project-install-homebrew.bash @@ -12,7 +12,7 @@ BOTTLE_NAME=$1 # project will have the major version included (ex gazebo2) export HOMEBREW_PREFIX=/usr/local export HOMEBREW_CELLAR=${HOMEBREW_PREFIX}/Cellar -export PATH=${HOMEBREW_PREFIX}/bin:$PATH +export PATH=${HOMEBREW_PREFIX}/bin:${HOMEBREW_PREFIX}/sbin:$PATH # make verbose mode? MAKE_VERBOSE_STR="" From 5f0ce592d2febeea6ec5d5834fca5085fbcdfa3e Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 24 May 2023 12:14:31 +0200 Subject: [PATCH 364/540] Remove gazebo9 support on gazebo.dsl (#934) Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/gazebo.dsl | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/jenkins-scripts/dsl/gazebo.dsl b/jenkins-scripts/dsl/gazebo.dsl index 357c860c0..1aede5a6c 100644 --- a/jenkins-scripts/dsl/gazebo.dsl +++ b/jenkins-scripts/dsl/gazebo.dsl @@ -1,12 +1,10 @@ import _configs_.* import javaposse.jobdsl.dsl.Job -def gazebo_supported_branches = [ 'gazebo9', 'gazebo11' ] +def gazebo_supported_branches = [ 'gazebo11' ] def gazebo_supported_build_types = [ 'Release', 'Debug', 'Coverage' ] // testing official packages without osrf repo -def ubuntu_official_packages_distros = [ 'bionic' : 'gazebo9', - 'focal' : 'gazebo9', - 'jammy' : 'gazebo11'] +def ubuntu_official_packages_distros = [ 'jammy' : 'gazebo11'] // Main platform using for quick CI def ci_distro = [ 'focal' ] def ci_gpu = Globals.get_ci_gpu() @@ -32,7 +30,7 @@ String abi_job_name = '' boolean is_watched_by_buildcop(branch, distro = 'bionic', gpu = 'nvidia') { - if (branch == 'master' || branch == 'gazebo9' || branch == 'gazebo11') + if (branch == 'master' || branch == 'gazebo11') return true return false From 329d5a723d89a6204d69d72af614188d14bcace4 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 24 May 2023 16:52:37 +0200 Subject: [PATCH 365/540] Reorder main jobs in ignition.dsl to help refactor (#937) Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/ignition.dsl | 155 ++++++++++++++++--------------- 1 file changed, 79 insertions(+), 76 deletions(-) diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index c36397cdc..1a5097814 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -347,11 +347,53 @@ boolean is_a_colcon_package(String gz_software_name) return false } -// ABI Checker job +void generate_install_job(prefix, gz_sw, major_version, distro, arch) +{ + def install_default_job = job("${prefix}_${gz_sw}${major_version}-install-pkg-${distro}-${arch}") + OSRFLinuxInstall.create(install_default_job) + include_gpu_label_if_needed(install_default_job, gz_sw) + + install_default_job.with + { + triggers { + cron(Globals.CRON_EVERY_THREE_DAYS) + } + + def dev_package = "lib${prefix}-${gz_sw}${major_version}-dev" + def gzdev_project = "${prefix}-${gz_sw}${major_version}" + + steps { + shell("""\ + #!/bin/bash -xe + + ${GLOBAL_SHELL_CMD} + + export DISTRO=${distro} + export ARCH=${arch} + export INSTALL_JOB_PKG=${dev_package} + export GZDEV_PROJECT_NAME="${gzdev_project}" + /bin/bash -x ./scripts/jenkins-scripts/docker/generic-install-test-job.bash + """.stripIndent()) + } + } +} + // Need to be before the ci-pr_any so the abi job name is defined gz_software.each { gz_sw -> - abi_distro.each { distro -> - supported_arches.each { arch -> + supported_arches.each { arch -> + // 1 Per library and per linux arch + // 1.1 Per abi_distro + // 1.1.1 [job] ABI checker for main branches + // 1.2 Per ci_str_distro + // 1.2.1 [job] Main PR jobs (-ci-pr_any-) + // 1.3 Per all supported_distros + // 1.3.1 Per all supported branches on each library + // 1.3.1.1 [job] Branch jobs -ci-$branch- + // 1.3.1.2 [job] Branch ASAN jobs -ci_asan-$branch- + + // 1.1.1 ABI checker for main branches + // -------------------------------------------------------------- + abi_distro.each { distro -> // Packages without ABI if (gz_sw == 'tools' || gz_sw == 'cmake') return @@ -397,15 +439,10 @@ gz_software.each { gz_sw -> """.stripIndent()) } // end of steps } // end of with - } // end of arch - } // end of distro -} // end of ignition + } // end of abi_distro -// MAIN CI JOBS (check every 5 minutes) -gz_software.each { gz_sw -> - supported_arches.each { arch -> + // 1.2.1 Main PR jobs (-ci-pr_any-) (pulling check every 5 minutes) // -------------------------------------------------------------- - // 1. Create the any job software_name = gz_sw // Necessary substitution. gz_sw won't overwrite if (gz_sw == 'sim') @@ -442,39 +479,40 @@ gz_software.each { gz_sw -> // add ci-pr_any to the list for CIWorkflow if (gz_sw != 'sim') ci_pr_any_list[software_name] << gz_ci_job_name - } -} -void generate_install_job(prefix, gz_sw, major_version, distro, arch) -{ - def install_default_job = job("${prefix}_${gz_sw}${major_version}-install-pkg-${distro}-${arch}") - OSRFLinuxInstall.create(install_default_job) - include_gpu_label_if_needed(install_default_job, gz_sw) - - install_default_job.with - { - triggers { - cron(Globals.CRON_EVERY_THREE_DAYS) - } - - def dev_package = "lib${prefix}-${gz_sw}${major_version}-dev" - def gzdev_project = "${prefix}-${gz_sw}${major_version}" - - steps { - shell("""\ - #!/bin/bash -xe + all_supported_distros.each { distro -> + all_branches("${gz_sw}").each { branch -> + // 1. Standard CI + software_name = gz_sw // Necessary substitution. gz_sw won't overwrite + + if (gz_sw == 'sim') + software_name = "gazebo" + + // 1.3.1.1 Branch jobs -ci-$branch- + // -------------------------------------------------------------- + def gz_ci_job = job("ignition_${software_name}-ci-${branch}-${distro}-${arch}") + generate_ci_job(gz_ci_job, software_name, branch, distro, arch) + gz_ci_job.with + { + triggers { + scm('@daily') + } + } + // 1.3.1.2 Branch ASAN jobs -ci_asan-$branch- + // -------------------------------------------------------------- + def gz_ci_asan_job = job("ignition_${software_name}-ci_asan-${branch}-${distro}-${arch}") + generate_asan_ci_job(gz_ci_asan_job, software_name, branch, distro, arch) + gz_ci_asan_job.with + { + triggers { + scm(Globals.CRON_ON_WEEKEND) + } + } + } + } // end of all_supported_distros + } // end of supported_arches +} // end of gz_software - ${GLOBAL_SHELL_CMD} - - export DISTRO=${distro} - export ARCH=${arch} - export INSTALL_JOB_PKG=${dev_package} - export GZDEV_PROJECT_NAME="${gzdev_project}" - /bin/bash -x ./scripts/jenkins-scripts/docker/generic-install-test-job.bash - """.stripIndent()) - } - } -} // INSTALL PACKAGE ALL PLATFORMS / DAILY gz_software.each { gz_sw -> @@ -547,41 +585,6 @@ void generate_ci_job(gz_ci_job, gz_sw, branch, distro, arch, } } -// OTHER CI SUPPORTED JOBS -gz_software.each { gz_sw -> - all_supported_distros.each { distro -> - supported_arches.each { arch -> - // -------------------------------------------------------------- - // branches CI job scm@daily - all_branches("${gz_sw}").each { branch -> - // 1. Standard CI - software_name = gz_sw // Necessary substitution. gz_sw won't overwrite - - if (gz_sw == 'sim') - software_name = "gazebo" - - def gz_ci_job = job("ignition_${software_name}-ci-${branch}-${distro}-${arch}") - generate_ci_job(gz_ci_job, software_name, branch, distro, arch) - gz_ci_job.with - { - triggers { - scm('@daily') - } - } - // 2. ASAN CI - def gz_ci_asan_job = job("ignition_${software_name}-ci_asan-${branch}-${distro}-${arch}") - generate_asan_ci_job(gz_ci_asan_job, software_name, branch, distro, arch) - gz_ci_asan_job.with - { - triggers { - scm(Globals.CRON_ON_WEEKEND) - } - } - } - } - } -} - // -------------------------------------------------------------- // DEBBUILD: linux package builder all_debbuilders().each { debbuilder_name -> From 71df95927e9dcd2e754f6a23b7a1750cd47ca2fb Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Mon, 29 May 2023 11:42:51 +0200 Subject: [PATCH 366/540] Initial work to move gz-collections metadata to a yaml file (#938) Move some nightly configuration to be defined in a yaml file Add a helper tool to create the code dsl + yaml - Use the helper script in github actions - Update instructions to run code locally Signed-off-by: Jose Luis Rivero --- .github/workflows/ci.yaml | 9 +-- .gitignore | 1 + jenkins-scripts/README.md | 5 +- jenkins-scripts/dsl/gz-collections.yaml | 57 +++++++++++++++++++ jenkins-scripts/dsl/ignition_collection.dsl | 26 ++++----- .../dsl/tools/setup_local_generation.bash | 22 +++++++ 6 files changed, 99 insertions(+), 21 deletions(-) create mode 100644 jenkins-scripts/dsl/gz-collections.yaml create mode 100755 jenkins-scripts/dsl/tools/setup_local_generation.bash diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index d40944b46..2afcf0b03 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -39,9 +39,10 @@ jobs: with: distribution: 'temurin' java-version: '11' - - name: Download job dsl jar + - name: Download and setup job dsl jar if: steps.dsl_check.outputs.run_job == 'true' - run: curl -sSL https://repo.jenkins-ci.org/public/org/jenkins-ci/plugins/job-dsl-core/1.77/job-dsl-core-1.77-standalone.jar -o jobdsl.jar + run: ./jenkins-scripts/dsl/tools/setup_local_generation.bash + - name: Generate all DSL files if: steps.dsl_check.outputs.run_job == 'true' run: | @@ -49,7 +50,7 @@ jobs: sudo mkdir -p /var/lib/jenkins/ && sudo touch /var/lib/jenkins/remote_token sudo chown -R ${USER} /var/lib/jenkins cd jenkins-scripts/dsl - java -jar ../../jobdsl.jar *.dsl + java -jar tools/jobdsl.jar *.dsl # export files for later diff mkdir /tmp/pr_xml_configuration mv *.xml /tmp/pr_xml_configuration/ @@ -59,7 +60,7 @@ jobs: git clean -f -e jobdsl.jar git checkout master cd jenkins-scripts/dsl - java -jar ../../jobdsl.jar *.dsl + java -jar tools/jobdsl.jar *.dsl mkdir /tmp/current_xml_configuration mv *.xml /tmp/current_xml_configuration/ - name: Generating diffs diff --git a/.gitignore b/.gitignore index 15584cace..24c982ec5 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ .pyc +jenkins-scripts/dsl/tools/*.jar diff --git a/jenkins-scripts/README.md b/jenkins-scripts/README.md index 1b34d7c1e..dd02dd950 100644 --- a/jenkins-scripts/README.md +++ b/jenkins-scripts/README.md @@ -12,11 +12,10 @@ The release-tools repository uses the [DSL Jenkins plugin](https://plugins.jenki To test locally the build of the different `dsl` jobs you need the following: -1. Download `job_dsl-core.standalone.jar` to be able to build the dsl jobs locally. It's necessary to use the [same version](https://github.com/osrf/chef-osrf/blob/latest/cookbooks/osrfbuild/attributes/plugins.rb#L70) for the jarfrom the dsl plugin. [Download here](https://repo.jenkins-ci.org/artifactory/releases/org/jenkins-ci/plugins/job-dsl-core) -2. Add the downloaded file to a folder not tracked in the repo. +1. Run the `dsl/tools/setup_local_generation.bash` script to produce the necessary jar files 3. In the terminal execute: ``` bash -java -jar `PATH/job_dsl-core.jar` `PATH/job_name.dsl` +java -jar /jobdsl.jar ``` For more information go [here](https://github.com/jenkinsci/job-dsl-plugin/wiki/User-Power-Moves#run-a-dsl-script-locally). diff --git a/jenkins-scripts/dsl/gz-collections.yaml b/jenkins-scripts/dsl/gz-collections.yaml new file mode 100644 index 000000000..ee3af72f2 --- /dev/null +++ b/jenkins-scripts/dsl/gz-collections.yaml @@ -0,0 +1,57 @@ +# +# SPEC: not yet approved, used as draft +# +#--- +# spec_version: 0 +# collections: +# - name: gz-collection1 +# libs: +# - gz-lib: +# target_branch: gz-lib1 +# current_branch: main +# ci: +# - linux: +# pr_distro: +# abi_distro: +# install_distros: +# - distro1 +# - distro2 +# - systems: +# - Jenkins: +# view: +# auto: false +# jobs: +# - job1 +# - job2 +# nightly: +# - linux: +# nightly_distros: +# - distro1 + +spec_version: 0 +collections: + - name: 'citadel' + ci: + linux: + reference_distro: + - bionic + - name: 'fortress' + ci: + linux: + reference_distro: + - focal + - name: 'garden' + ci: + linux: + reference_distro: + - focal + - name: 'harmonic' + ci: + linux: + reference_distro: + - jammy + nightly: + linux: + nightly_distros: + - jammy + diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index c298b826f..77231077c 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -1,21 +1,18 @@ import _configs_.* import javaposse.jobdsl.dsl.Job +// If failed to import locally be sure of using tools/ scripts +import org.yaml.snakeyaml.Yaml // GZ COLLECTIONS arch = 'amd64' -gz_nightly = 'harmonic' +// Jenkins needs the relative path to work and locally the simulation is done +// using a symlink +file = readFileFromWorkspace("scripts/jenkins-scripts/dsl/gz-collections.yaml") +gz_collections_yaml = new Yaml().load(file) +gz_nightly = 'harmonic' gz_collections = [ - [ name : 'citadel', - distros : [ 'bionic' ], - ], - [ name : 'fortress', - distros : [ 'focal' ], - ], - [ name : 'garden', - distros : [ 'focal' ], - ], [ name : 'harmonic', distros : [ 'focal' ], // These are the branches currently targeted at the upcoming collection @@ -381,10 +378,11 @@ void generate_install_job(prefix, gz_collection_name, distro, arch) } // Testing compilation from source -gz_collections.each { gz_collection -> - // COLCON - Windows - gz_collection_name = gz_collection.get('name') +gz_collections_yaml.collections.each { collection -> + gz_collection_name = collection.name + distros = collection.ci.linux.reference_distro + // COLCON - Windows def gz_win_ci_job = job("ign_${gz_collection_name}-ci-win") Globals.gazebodistro_branch = true OSRFWinCompilation.create(gz_win_ci_job, false) @@ -399,7 +397,7 @@ gz_collections.each { gz_collection -> } Globals.gazebodistro_branch = false - gz_collection.get('distros').each { distro -> + distros.each { distro -> // INSTALL JOBS: // -------------------------------------------------------------- if ((gz_collection_name == "citadel") || (gz_collection_name == "fortress")) { diff --git a/jenkins-scripts/dsl/tools/setup_local_generation.bash b/jenkins-scripts/dsl/tools/setup_local_generation.bash new file mode 100755 index 000000000..9b14f9b59 --- /dev/null +++ b/jenkins-scripts/dsl/tools/setup_local_generation.bash @@ -0,0 +1,22 @@ +#!/bin/bash +set -e + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +SNAKEYAML_PATH="lib/snakeyaml.jar" +JOB_DSL_VER="1.77" + +pushd "${SCRIPT_DIR}" >/dev/null +echo " * Download job-dsl-core jar" +curl -sSL https://repo.jenkins-ci.org/public/org/jenkins-ci/plugins/job-dsl-core/${JOB_DSL_VER}/job-dsl-core-${JOB_DSL_VER}-standalone.jar -o jobdsl.jar +echo " * Download snakeyaml jar" +# need to inject the yaml module into the jar file. lib subdir is needed to ineject into +# tar in the right place. +mkdir -p lib +curl -sSL https://repo1.maven.org/maven2/org/yaml/snakeyaml/1.32/snakeyaml-1.32.jar -o ${SNAKEYAML_PATH} +echo " * Inject snakeyaml in job-dsl-core jar" +jar uf jobdsl.jar ${SNAKEYAML_PATH} +rm -fr ${SNAKEYAML_PATH} +rmdir lib +echo " >> Run 'java -jar ${PWD}/jobdsl.jar ' to generate locally the Jenkins XML config" +popd >/dev/null From 509ab501999c0374487d37f4acd9ea9e63144779 Mon Sep 17 00:00:00 2001 From: "Addisu Z. Taddese" Date: Mon, 29 May 2023 04:49:07 -0500 Subject: [PATCH 367/540] Enable -fPIC for ign-gazebo6 (#941) --- jenkins-scripts/docker/ignition-abichecker.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/docker/ignition-abichecker.bash b/jenkins-scripts/docker/ignition-abichecker.bash index 66b24559a..1383bfd2a 100644 --- a/jenkins-scripts/docker/ignition-abichecker.bash +++ b/jenkins-scripts/docker/ignition-abichecker.bash @@ -92,7 +92,7 @@ then fi fi -if [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-gazebo" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -ge 7 ]] || \ +if [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-gazebo" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -ge 6 ]] || \ [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-gui" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -ge 7 ]] || \ [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-launch" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -ge 6 ]] || \ [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-sensors" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -ge 7 ]] || \ From 00bcde8ca1ed08681881e0c3e8e6df723d0d6483 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Mon, 29 May 2023 18:55:28 +0200 Subject: [PATCH 368/540] Fix deprecation on github actions: use GITHUB_OUTPUT instead of set-output (#944) * Migrate github action to GITHUB_OUTPUT * Force test of DSL code Signed-off-by: Jose Luis Rivero --- .github/workflows/ci.yaml | 10 +++++----- jenkins-scripts/dsl/test.dsl | 1 + 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2afcf0b03..6f41679b5 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -9,29 +9,29 @@ jobs: name: Diff for DSL code steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 - name: Idenfify files changed in this PR id: files run: | git diff --name-only origin/${{ github.base_ref }}...origin/${{ github.head_ref }} - echo "::set-output name=changed-files::$(git diff --name-only origin/${{ github.base_ref }}...origin/${{ github.head_ref }}| tr '\n' ' ')" + echo "changed-files=$(git diff --name-only origin/${{ github.base_ref }}...origin/${{ github.head_ref }}| tr '\n' ' ')" >> $GITHUB_OUTPUT - name: Run testing on changed config files id: dsl_check run: | for changed_file in ${{ steps.files.outputs.changed-files }}; do if [[ ${changed_file} != ${changed_file/dsl\/*} ]]; then echo "+ Detected at leat one config file: ${changed_file}." - echo "::set-output name=run_job::true" + echo "run_job=true" >> $GITHUB_OUTPUT break else - echo "::set-output name=run_job::false" + echo "run_job=false" >> $GITHUB_OUTPUT fi done - name: Checkout if: steps.dsl_check.outputs.run_job == 'true' - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 2 - uses: actions/setup-java@v3 diff --git a/jenkins-scripts/dsl/test.dsl b/jenkins-scripts/dsl/test.dsl index 7d63fffb5..151f2d209 100644 --- a/jenkins-scripts/dsl/test.dsl +++ b/jenkins-scripts/dsl/test.dsl @@ -3,6 +3,7 @@ import javaposse.jobdsl.dsl.Job Globals.default_emails = "jrivero@osrfoundation.org" +// Jobs def ignition_ci_job = job("_test_job_from_dsl") OSRFLinuxBase.create(ignition_ci_job) def ignition_ci_job_mac = job("_test_job_from_dsl_mac") From 44fb01e28e3722835ac1fdd165ba470dff4e3513 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Mon, 29 May 2023 18:55:51 +0200 Subject: [PATCH 369/540] Add a parser for OpenGL problems on Windows (#943) * Add a parser for OpenGL problems on Windows Signed-off-by: Jose Luis Rivero * Update OSRFWinBase.groovy --------- Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/_configs_/OSRFWinBase.groovy | 7 +++++++ jenkins-scripts/parser_rules/opengl_problem.parser | 1 + 2 files changed, 8 insertions(+) create mode 100644 jenkins-scripts/parser_rules/opengl_problem.parser diff --git a/jenkins-scripts/dsl/_configs_/OSRFWinBase.groovy b/jenkins-scripts/dsl/_configs_/OSRFWinBase.groovy index 4c9bbffcd..5884872d7 100644 --- a/jenkins-scripts/dsl/_configs_/OSRFWinBase.groovy +++ b/jenkins-scripts/dsl/_configs_/OSRFWinBase.groovy @@ -24,6 +24,13 @@ class OSRFWinBase extends OSRFBase git clone https://github.com/gazebo-tooling/release-tools scripts -b %RTOOLS_BRANCH% """.stripIndent()) } + + publishers { + consoleParsing { + projectRules('scripts/jenkins-scripts/parser_rules/opengl_problem.parser') + unstableOnWarning() + } + } } } } diff --git a/jenkins-scripts/parser_rules/opengl_problem.parser b/jenkins-scripts/parser_rules/opengl_problem.parser new file mode 100644 index 000000000..b3b971ff7 --- /dev/null +++ b/jenkins-scripts/parser_rules/opengl_problem.parser @@ -0,0 +1 @@ +warning /.*Ogre::RenderingAPIException::RenderingAPIException: OpenGL .* is not supported in GLRenderSystem::initialiseContext.*/ From 484f0798887b453308ae5f0201f68f4ce719fe58 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 1 Jun 2023 14:07:30 +0200 Subject: [PATCH 370/540] Add nightly generation and harmonic libs to gz-collections.yaml (#940) * Move nightly configuration to use the gz-collections.yaml file Implement libs inside harmonic to keep the nightly configuration running. * The change break the local generation due to the lack of the sneakyaml jar. Add a helper tool to do it: * Use the helper script in github actions * Update README.md with instructions instructions * Fix small typo in nightly output --------- Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/gz-collections.yaml | 95 +++++++++++++-- jenkins-scripts/dsl/ignition_collection.dsl | 122 ++++++++------------ 2 files changed, 133 insertions(+), 84 deletions(-) diff --git a/jenkins-scripts/dsl/gz-collections.yaml b/jenkins-scripts/dsl/gz-collections.yaml index ee3af72f2..1b65b2e85 100644 --- a/jenkins-scripts/dsl/gz-collections.yaml +++ b/jenkins-scripts/dsl/gz-collections.yaml @@ -6,9 +6,7 @@ # collections: # - name: gz-collection1 # libs: -# - gz-lib: -# target_branch: gz-lib1 -# current_branch: main +# ... # ci: # - linux: # pr_distro: @@ -23,35 +21,112 @@ # jobs: # - job1 # - job2 -# nightly: -# - linux: -# nightly_distros: -# - distro1 +# packaging: +# ... spec_version: 0 collections: - name: 'citadel' + libs: ci: linux: reference_distro: - bionic + packaging: - name: 'fortress' + libs: ci: linux: reference_distro: - focal + packaging: - name: 'garden' + libs: ci: linux: reference_distro: - focal + packaging: - name: 'harmonic' + libs: + - name: gz-cmake + major_version: 3 + repo: + current_branch: gz-cmake3 + - name: gz-tools + major_version: 2 + repo: + current_branch: gz-tools2 + - name: gz-utils + major_version: 2 + repo: + current_branch: gz-utils2 + - name: gz-math + major_version: 7 + repo: + current_branch: gz-math7 + - name: gz-plugin + major_version: 2 + repo: + current_branch: gz-plugin2 + - name: gz-common + major_version: 5 + repo: + current_branch: gz-common5 + - name: gz-msgs + major_version: 10 + repo: + current_branch: main + - name: gz-rendering + major_version: 8 + repo: + current_branch: main + - name: sdformat + major_version: 13 + repo: + current_branch: sdf13 + - name: gz-fuel-tools + major_version: 9 + repo: + current_branch: main + - name: gz-transport + major_version: 13 + repo: + current_branch: main + - name: gz-gui + major_version: 8 + repo: + current_branch: main + - name: gz-sensors + major_version: 8 + repo: + current_branch: main + - name: gz-physics + major_version: 6 + repo: + current_branch: gz-physics6 + - name: gz-sim + major_version: 8 + repo: + current_branch: main + - name: gz-launch + major_version: 7 + repo: + current_branch: main + - name: gz-harmonic + major_version: 1 + repo: + current_branch: main ci: linux: reference_distro: - jammy - nightly: + packaging: linux: - nightly_distros: - - jammy + package_name: + ignore_major_version: + - gz-harmonic + nightly: + distros: + - jammy diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index 77231077c..5dcb6bd4b 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -12,32 +12,17 @@ file = readFileFromWorkspace("scripts/jenkins-scripts/dsl/gz-collections.yaml") gz_collections_yaml = new Yaml().load(file) gz_nightly = 'harmonic' -gz_collections = [ - [ name : 'harmonic', - distros : [ 'focal' ], - // These are the branches currently targeted at the upcoming collection - // They're in topological order - nightly_jobs: [ - 'cmake' : [ debbuild: 'gz-cmake3' , branch: 'gz-cmake3' ], - 'tools' : [ debbuild: 'gz-tools2' , branch: 'gz-tools2' ], - 'utils' : [ debbuild: 'gz-utils2' , branch: 'gz-utils2' ], - 'math' : [ debbuild: 'gz-math7' , branch: 'gz-math7' ], - 'plugin' : [ debbuild: 'gz-plugin2' , branch: 'gz-plugin2' ], - 'common' : [ debbuild: 'gz-common5' , branch: 'gz-common5' ], - 'msgs' : [ debbuild: 'gz-msgs10' , branch: 'main' ], - 'rendering' : [ debbuild: 'gz-rendering8' , branch: 'main' ], - 'sdformat' : [ debbuild: 'sdformat13' , branch: 'sdf13' ], - 'fuel-tools': [ debbuild: 'gz-fuel-tools9' , branch: 'main' ], - 'transport' : [ debbuild: 'gz-transport13' , branch: 'main' ], - 'gui' : [ debbuild: 'gz-gui8' , branch: 'main' ], - 'sensors' : [ debbuild: 'gz-sensors8' , branch: 'main' ], - 'physics' : [ debbuild: 'gz-physics6' , branch: 'gz-physics6' ], - 'gazebo' : [ debbuild: 'gz-sim8' , branch: 'main' ], - 'launch' : [ debbuild: 'gz-launch7' , branch: 'main' ], - 'garden' : [ debbuild: 'gz-garden' , branch: 'main' ], - ], - ], -] + +String get_debbuilder_name(parsed_yaml_lib, parsed_yaml_packaging) +{ + major_version = parsed_yaml_lib.major_version + + ignore_major_version = parsed_yaml_packaging.linux?.package_name?.ignore_major_version + if (ignore_major_version && ignore_major_version.contains(parsed_yaml_lib.name)) + major_version = "" + + return parsed_yaml_lib.name + major_version + "-debbuilder" +} gz_collection_jobs = [ @@ -525,15 +510,14 @@ gz_collections_yaml.collections.each { collection -> dashboardView("ign-${gz_collection_name}") { jobs { - gz_collection_jobs["${gz_collection_name}"].each { jobname -> - name(jobname) - } - if (gz_collection_name == gz_nightly) { - // add nightly debbuild jobs too - gz_collections.find { it.get('name') == gz_nightly }.get('nightly_jobs').each { job -> - name(job.getValue().get('debbuild') + '-debbuilder') - } + gz_collection_jobs["${gz_collection_name}"].each { jobname -> + name(jobname) + } + if (collection.packaging?.linux?.nightly) { + collection.libs.each { lib -> + name(get_debbuilder_name(lib, collection.packaging)) } + } } columns { @@ -570,27 +554,13 @@ gz_collections_yaml.collections.each { collection -> } // NIGHTLY GENERATION -def get_nightly_branch(collection_data, ign_package) +def get_nightly_branch(nightly_collection, lib) { - try { - if (collection_data.get(ign_package)) - return collection_data.get(ign_package).get('branch') - } catch(Exception e) { - return 'not_enabled_in_DSL' - } - return 'not_enabled_in_DSL' + return nightly_collection.libs.find { it.name == lib }.repo.current_branch } -collection_data = [] -list_of_pkgs = "" - -collection_data = gz_collections.find { it.get('name') == gz_nightly } -collection_data = collection_data.get('nightly_jobs') - -collection_data.each { job -> - debbuild = job.getValue().get('debbuild') - list_of_pkgs = "${list_of_pkgs} ${debbuild}" -} +nightly_collection = gz_collections_yaml.collections + .find { it.name == gz_nightly } def nightly_scheduler_job = job("ignition-${gz_nightly}-nightly-scheduler") OSRFUNIXBase.create(nightly_scheduler_job) @@ -601,8 +571,12 @@ nightly_scheduler_job.with parameters { - stringParam('NIGHTLY_PACKAGES',"${list_of_pkgs}", - 'space separated list of packages to build') + stringParam('NIGHTLY_PACKAGES', + nightly_collection.libs.collect{ + get_debbuilder_name(it,nightly_collection.packaging) + .replace("-debbuilder","") + }.join(" "), + 'space separated list of packages to build') booleanParam('DRY_RUN',false, 'run a testing run with no effects') @@ -612,22 +586,22 @@ nightly_scheduler_job.with cron(Globals.CRON_START_NIGHTLY) } - cmake_branch = get_nightly_branch(collection_data, 'cmake') - common_branch = get_nightly_branch(collection_data, 'common') - fuel_tools_branch = get_nightly_branch(collection_data, 'fuel-tools') - gazebo_branch = get_nightly_branch(collection_data, 'gazebo') - gui_branch = get_nightly_branch(collection_data, 'gui') - launch_branch = get_nightly_branch(collection_data, 'launch') - math_branch = get_nightly_branch(collection_data, 'math') - msgs_branch = get_nightly_branch(collection_data, 'msgs') - physics_branch = get_nightly_branch(collection_data, 'physics') - plugin_branch = get_nightly_branch(collection_data, 'plugin') - rendering_branch = get_nightly_branch(collection_data, 'rendering') - sensors_branch = get_nightly_branch(collection_data, 'sensors') - sdformat_branch = get_nightly_branch(collection_data, 'sdformat') - tools_branch = get_nightly_branch(collection_data, 'tools') - transport_branch = get_nightly_branch(collection_data, 'transport') - utils_branch = get_nightly_branch(collection_data, 'utils') + cmake_branch = get_nightly_branch(nightly_collection, 'gz-cmake') + common_branch = get_nightly_branch(nightly_collection, 'gz-common') + fuel_tools_branch = get_nightly_branch(nightly_collection, 'gz-fuel-tools') + sim_branch = get_nightly_branch(nightly_collection, 'gz-sim') + gui_branch = get_nightly_branch(nightly_collection, 'gz-gui') + launch_branch = get_nightly_branch(nightly_collection, 'gz-launch') + math_branch = get_nightly_branch(nightly_collection, 'gz-math') + msgs_branch = get_nightly_branch(nightly_collection, 'gz-msgs') + physics_branch = get_nightly_branch(nightly_collection, 'gz-physics') + plugin_branch = get_nightly_branch(nightly_collection, 'gz-plugin') + rendering_branch = get_nightly_branch(nightly_collection, 'gz-rendering') + sensors_branch = get_nightly_branch(nightly_collection, 'gz-sensors') + sdformat_branch = get_nightly_branch(nightly_collection, 'sdformat') + tools_branch = get_nightly_branch(nightly_collection, 'gz-tools') + transport_branch = get_nightly_branch(nightly_collection, 'gz-transport') + utils_branch = get_nightly_branch(nightly_collection, 'gz-utils') steps { shell("""\ @@ -649,8 +623,8 @@ nightly_scheduler_job.with src_branch="${common_branch}" elif [[ "\${n}" != "\${n/fuel-tools/}" ]]; then src_branch="${fuel_tools_branch}" - elif [[ "\${n}" != "\${n/gazebo/}" ]]; then - src_branch="${gazebo_branch}" + elif [[ "\${n}" != "\${n/sim/}" ]]; then + src_branch="${sim_branch}" elif [[ "\${n}" != "\${n/gui/}" ]]; then src_branch="${gui_branch}" elif [[ "\${n}" != "\${n/launch/}" ]]; then @@ -670,7 +644,7 @@ nightly_scheduler_job.with elif [[ "\${n}" != "\${n/sdformat/}" ]]; then src_branch="${sdformat_branch}" elif [[ "\${n}" != "\${n/sim/}" ]]; then - src_branch="${gazebo_branch}" + src_branch="${sim_branch}" elif [[ "\${n}" != "\${n/transport/}" ]]; then src_branch="${transport_branch}" elif [[ "\${n}" != "\${n/tools/}" ]]; then @@ -681,7 +655,7 @@ nightly_scheduler_job.with src_branch="main" fi - echo "releasing \${n} (from branch \${src_branch}" + echo "releasing \${n} (from branch \${src_branch})" python3 ./scripts/release.py \${dry_run_str} "\${n}" nightly "\${PASS}" --release-repo-branch main --nightly-src-branch \${src_branch} --upload-to-repo nightly > log || echo "MARK_AS_UNSTABLE" echo " - done" done From e565edd737623ce0ac6932dcafe2204d2d468357 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 2 Jun 2023 15:13:45 +0200 Subject: [PATCH 371/540] Rename compilation scripts from ign_ to gz_ (#947) * Move code to use gz_ prefix instead of ign_ in *compilation.bash * Move code to use gz_ prefix instead of ign_ in ign_launch test Signed-off-by: Jose Luis Rivero --- .../{ign_cmake-compilation.bash => gz_cmake-compilation.bash} | 0 ...ign_common-compilation.bash => gz_common-compilation.bash} | 0 ...-tools-compilation.bash => gz_fuel-tools-compilation.bash} | 0 ...ign_gazebo-compilation.bash => gz_gazebo-compilation.bash} | 0 .../{ign_gui-compilation.bash => gz_gui-compilation.bash} | 0 ...ign_launch-compilation.bash => gz_launch-compilation.bash} | 0 ...-install-test-job.bash => gz_launch-install-test-job.bash} | 0 .../{ign_math-compilation.bash => gz_math-compilation.bash} | 0 .../{ign_msgs-compilation.bash => gz_msgs-compilation.bash} | 0 ...n_physics-compilation.bash => gz_physics-compilation.bash} | 0 ...ign_plugin-compilation.bash => gz_plugin-compilation.bash} | 0 ...ndering-compilation.bash => gz_rendering-compilation.bash} | 0 .../{ign_rndf-compilation.bash => gz_rndf-compilation.bash} | 0 ...n_sensors-compilation.bash => gz_sensors-compilation.bash} | 0 .../{ign_tools-compilation.bash => gz_tools-compilation.bash} | 0 ...ansport-compilation.bash => gz_transport-compilation.bash} | 0 .../{ign_utils-compilation.bash => gz_utils-compilation.bash} | 0 jenkins-scripts/dsl/ignition.dsl | 4 ++-- jenkins-scripts/dsl/ignition_collection.dsl | 4 ++-- jenkins-scripts/dsl/ros_gz_bridge.dsl | 2 +- 20 files changed, 5 insertions(+), 5 deletions(-) rename jenkins-scripts/docker/{ign_cmake-compilation.bash => gz_cmake-compilation.bash} (100%) rename jenkins-scripts/docker/{ign_common-compilation.bash => gz_common-compilation.bash} (100%) rename jenkins-scripts/docker/{ign_fuel-tools-compilation.bash => gz_fuel-tools-compilation.bash} (100%) rename jenkins-scripts/docker/{ign_gazebo-compilation.bash => gz_gazebo-compilation.bash} (100%) rename jenkins-scripts/docker/{ign_gui-compilation.bash => gz_gui-compilation.bash} (100%) rename jenkins-scripts/docker/{ign_launch-compilation.bash => gz_launch-compilation.bash} (100%) rename jenkins-scripts/docker/{ign_launch-install-test-job.bash => gz_launch-install-test-job.bash} (100%) rename jenkins-scripts/docker/{ign_math-compilation.bash => gz_math-compilation.bash} (100%) rename jenkins-scripts/docker/{ign_msgs-compilation.bash => gz_msgs-compilation.bash} (100%) rename jenkins-scripts/docker/{ign_physics-compilation.bash => gz_physics-compilation.bash} (100%) rename jenkins-scripts/docker/{ign_plugin-compilation.bash => gz_plugin-compilation.bash} (100%) rename jenkins-scripts/docker/{ign_rendering-compilation.bash => gz_rendering-compilation.bash} (100%) rename jenkins-scripts/docker/{ign_rndf-compilation.bash => gz_rndf-compilation.bash} (100%) rename jenkins-scripts/docker/{ign_sensors-compilation.bash => gz_sensors-compilation.bash} (100%) rename jenkins-scripts/docker/{ign_tools-compilation.bash => gz_tools-compilation.bash} (100%) rename jenkins-scripts/docker/{ign_transport-compilation.bash => gz_transport-compilation.bash} (100%) rename jenkins-scripts/docker/{ign_utils-compilation.bash => gz_utils-compilation.bash} (100%) diff --git a/jenkins-scripts/docker/ign_cmake-compilation.bash b/jenkins-scripts/docker/gz_cmake-compilation.bash similarity index 100% rename from jenkins-scripts/docker/ign_cmake-compilation.bash rename to jenkins-scripts/docker/gz_cmake-compilation.bash diff --git a/jenkins-scripts/docker/ign_common-compilation.bash b/jenkins-scripts/docker/gz_common-compilation.bash similarity index 100% rename from jenkins-scripts/docker/ign_common-compilation.bash rename to jenkins-scripts/docker/gz_common-compilation.bash diff --git a/jenkins-scripts/docker/ign_fuel-tools-compilation.bash b/jenkins-scripts/docker/gz_fuel-tools-compilation.bash similarity index 100% rename from jenkins-scripts/docker/ign_fuel-tools-compilation.bash rename to jenkins-scripts/docker/gz_fuel-tools-compilation.bash diff --git a/jenkins-scripts/docker/ign_gazebo-compilation.bash b/jenkins-scripts/docker/gz_gazebo-compilation.bash similarity index 100% rename from jenkins-scripts/docker/ign_gazebo-compilation.bash rename to jenkins-scripts/docker/gz_gazebo-compilation.bash diff --git a/jenkins-scripts/docker/ign_gui-compilation.bash b/jenkins-scripts/docker/gz_gui-compilation.bash similarity index 100% rename from jenkins-scripts/docker/ign_gui-compilation.bash rename to jenkins-scripts/docker/gz_gui-compilation.bash diff --git a/jenkins-scripts/docker/ign_launch-compilation.bash b/jenkins-scripts/docker/gz_launch-compilation.bash similarity index 100% rename from jenkins-scripts/docker/ign_launch-compilation.bash rename to jenkins-scripts/docker/gz_launch-compilation.bash diff --git a/jenkins-scripts/docker/ign_launch-install-test-job.bash b/jenkins-scripts/docker/gz_launch-install-test-job.bash similarity index 100% rename from jenkins-scripts/docker/ign_launch-install-test-job.bash rename to jenkins-scripts/docker/gz_launch-install-test-job.bash diff --git a/jenkins-scripts/docker/ign_math-compilation.bash b/jenkins-scripts/docker/gz_math-compilation.bash similarity index 100% rename from jenkins-scripts/docker/ign_math-compilation.bash rename to jenkins-scripts/docker/gz_math-compilation.bash diff --git a/jenkins-scripts/docker/ign_msgs-compilation.bash b/jenkins-scripts/docker/gz_msgs-compilation.bash similarity index 100% rename from jenkins-scripts/docker/ign_msgs-compilation.bash rename to jenkins-scripts/docker/gz_msgs-compilation.bash diff --git a/jenkins-scripts/docker/ign_physics-compilation.bash b/jenkins-scripts/docker/gz_physics-compilation.bash similarity index 100% rename from jenkins-scripts/docker/ign_physics-compilation.bash rename to jenkins-scripts/docker/gz_physics-compilation.bash diff --git a/jenkins-scripts/docker/ign_plugin-compilation.bash b/jenkins-scripts/docker/gz_plugin-compilation.bash similarity index 100% rename from jenkins-scripts/docker/ign_plugin-compilation.bash rename to jenkins-scripts/docker/gz_plugin-compilation.bash diff --git a/jenkins-scripts/docker/ign_rendering-compilation.bash b/jenkins-scripts/docker/gz_rendering-compilation.bash similarity index 100% rename from jenkins-scripts/docker/ign_rendering-compilation.bash rename to jenkins-scripts/docker/gz_rendering-compilation.bash diff --git a/jenkins-scripts/docker/ign_rndf-compilation.bash b/jenkins-scripts/docker/gz_rndf-compilation.bash similarity index 100% rename from jenkins-scripts/docker/ign_rndf-compilation.bash rename to jenkins-scripts/docker/gz_rndf-compilation.bash diff --git a/jenkins-scripts/docker/ign_sensors-compilation.bash b/jenkins-scripts/docker/gz_sensors-compilation.bash similarity index 100% rename from jenkins-scripts/docker/ign_sensors-compilation.bash rename to jenkins-scripts/docker/gz_sensors-compilation.bash diff --git a/jenkins-scripts/docker/ign_tools-compilation.bash b/jenkins-scripts/docker/gz_tools-compilation.bash similarity index 100% rename from jenkins-scripts/docker/ign_tools-compilation.bash rename to jenkins-scripts/docker/gz_tools-compilation.bash diff --git a/jenkins-scripts/docker/ign_transport-compilation.bash b/jenkins-scripts/docker/gz_transport-compilation.bash similarity index 100% rename from jenkins-scripts/docker/ign_transport-compilation.bash rename to jenkins-scripts/docker/gz_transport-compilation.bash diff --git a/jenkins-scripts/docker/ign_utils-compilation.bash b/jenkins-scripts/docker/gz_utils-compilation.bash similarity index 100% rename from jenkins-scripts/docker/ign_utils-compilation.bash rename to jenkins-scripts/docker/gz_utils-compilation.bash diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index 1a5097814..b953fc56f 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -471,7 +471,7 @@ gz_software.each { gz_sw -> export ARCH=${arch} - /bin/bash -xe ./scripts/jenkins-scripts/docker/ign_${software_name}-compilation.bash + /bin/bash -xe ./scripts/jenkins-scripts/docker/gz_${software_name}-compilation.bash """.stripIndent()) } // end of steps } // end of ci_any_job @@ -579,7 +579,7 @@ void generate_ci_job(gz_ci_job, gz_sw, branch, distro, arch, export BUILDING_EXTRA_MAKETEST_PARAMS="${extra_test}" export DISTRO=${distro} export ARCH=${arch} - /bin/bash -xe ./scripts/jenkins-scripts/docker/ign_${software_name}-compilation.bash + /bin/bash -xe ./scripts/jenkins-scripts/docker/gz_${software_name}-compilation.bash """.stripIndent()) } } diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index 5dcb6bd4b..d512e30a7 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -341,7 +341,7 @@ void generate_install_job(prefix, gz_collection_name, distro, arch) } def dev_package = "${prefix}-${gz_collection_name}" - def job_name = 'ign_launch-install-test-job.bash' + def job_name = 'gz_launch-install-test-job.bash' label Globals.nontest_label("gpu-reliable") @@ -432,7 +432,7 @@ gz_collections_yaml.collections.each { collection -> if [[ \${JENKINS_NODE_TAG} == 'gpu-reliable' ]]; then export ENABLE_GZ_SIM_RUNTIME_TEST=true fi - /bin/bash -x ./scripts/jenkins-scripts/docker/ign_launch-install-test-job.bash + /bin/bash -x ./scripts/jenkins-scripts/docker/gz_launch-install-test-job.bash """.stripIndent()) } } diff --git a/jenkins-scripts/dsl/ros_gz_bridge.dsl b/jenkins-scripts/dsl/ros_gz_bridge.dsl index c20e53ad8..52e726962 100644 --- a/jenkins-scripts/dsl/ros_gz_bridge.dsl +++ b/jenkins-scripts/dsl/ros_gz_bridge.dsl @@ -174,7 +174,7 @@ install_test_job.with export INSTALL_JOB_PKG=ros-\${ROS_DISTRO}-ros-gz\${GZ_VERSION}-sim export ROS2=true - /bin/bash -x ./scripts/jenkins-scripts/docker/ign_launch-install-test-job.bash + /bin/bash -x ./scripts/jenkins-scripts/docker/gz_launch-install-test-job.bash """.stripIndent()) } } From 85f31bb040b98ddc63e4c7b3dec05f9a3251ceeb Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 2 Jun 2023 18:15:16 +0200 Subject: [PATCH 372/540] Unofficial ros_gz: README for forking gbp and script for renaming (#898) * README instructions about how to fork gbp * rename script to modify all the bloom templates consistently --------- Signed-off-by: Jose Luis Rivero --- bloom/ros_gz/README.md | 63 ++++++++++++++++++++++++++++ bloom/ros_gz/rename-ros_gz-pkgs.bash | 54 ++++++++++++++++++++++++ 2 files changed, 117 insertions(+) create mode 100644 bloom/ros_gz/README.md create mode 100755 bloom/ros_gz/rename-ros_gz-pkgs.bash diff --git a/bloom/ros_gz/README.md b/bloom/ros_gz/README.md new file mode 100644 index 000000000..04ac073bb --- /dev/null +++ b/bloom/ros_gz/README.md @@ -0,0 +1,63 @@ +# Release new version of Gazebo unofficial wrappers + +1. Background + * Upstream versions released using this tutorial +2. Initial setup + * Create the alternative -release repository + * Create a custom track in tracks.yaml + +## 1. Background + +Each ROS release defines one version of Gazebo supported officially through +all the ROS packages. The different combinations of ROS <-> Gazebo can be +found in the [REP-3](http://www.ros.org/reps/rep-2000.html). Some examples: + + * ROS 2 Foxy: Citadel + * ROS 2 Humble: Fortress + +Some use cases require the use of alternative combinations of ROS and Gazebo +versions. The `ros_gz` code is usually prepared to be compatible with +different versions of Gazebo, especially the latest ones. + +Although using the officially supported version is the recommended way +specially for non experienced users, some use cases might need to use a +newer version of Gazebo than the one selected in REP-2000. + +### Upstream versions released using this tutorial + +The `gbp -release repository` hosts the latest version released by the +maintainers of `ros_gz`. When using these instructions to release a new custom +version the version of `ros_gz` released will be the latest one existing in the +official `gbp -release repository`. The version would be the same but the +release number will start on 1000. + +## 2. Initial setup + +To release a modified version of `ros_gz` which supports a different major +version of gazebo, before running bloom some actions need to be taken: + +## 2. Initial setup + +### 2.1 Create the alternative -release repository + +For a new official wrappers the notation used below correspond to `ros_ign-release`: + + 1. Fork (manually or using gh) current gbp repository: + https://github.com/ros2-gbp/ros_ign-release + + 1. Clone the new repo, go to the directory and run rename-gazebo-ros-pkgs.bash + - Usage: *$ rename-ros_gz-pkgs.bash * + + +### 2.2 Create a custom track in tracks.yml + +Copy the existing ROS 2 yaml track information and rename it to `${ros2}_gz${version}`. +For example, the `humble` track to be used as base for Garden would be `humble_gzgarden`. + +New versioning requires bumping to large numbers. Set: + +``` + release_inc: '1000' +``` + +All non ubuntu generators can be removed. diff --git a/bloom/ros_gz/rename-ros_gz-pkgs.bash b/bloom/ros_gz/rename-ros_gz-pkgs.bash new file mode 100755 index 000000000..66b3a7953 --- /dev/null +++ b/bloom/ros_gz/rename-ros_gz-pkgs.bash @@ -0,0 +1,54 @@ +#!/bin/bash +# +# This script should be run on a fork of ros_ign-release/ros_gz-release repository +# and will modify control.em bloom templates for: +# +# - Rename the Package name modifying -gz- by -gz$DISTRO- +# - Define a conflict on current official name: $(Package) +# + +if [[ ${#} -lt 2 ]]; then + echo "Usage: ${0} > " + exit 1 +fi + +# Safety check for ros2-gbp repo +if [[ -n $(git config --get remote.origin.url | grep git@github.com:ros2-gbp/ros_ign-release.git) ]]; then + echo "This script refuses to modify the ros2-gbp repository. You probably don't want this." + exit 1 +fi + +GZ_RELEASE=${1} +ROS_DISTROS=${*:2} + +PKGS="ros_gz ros_gz_bridge ros_gz_image ros_gz_interfaces ros_gz_sim ros_gz_sim_demos" + +for pkg in ${PKGS}; do + for distro in ${ROS_DISTROS}; do + echo " - Processing $pkg in $distro" + if ! git checkout "debian/$distro/$pkg"; then + echo "The branch debian/$distro/$pkg was not found in the repo" + echo "Did you forget to run git fetch?" + exit 1 + fi + # Add GZ_VERSION env variable in rules file + sed -i -e "/export DH_VERBOSE/a\export GZ_VERSION=${GZ_RELEASE}" debian/rules.em + git commit debian/rules.em -m "Export GZ_VERSION in rules file" + git push origin "debian/$distro/$pkg" + if grep 'Package.replace' debian/control.em; then + echo " + skip ${pkg} for ${distro}: seems to have changes in place" + continue + fi + # Modify package name. Note that regexp can not be stricker than + # *-gz since there is a package named package ros-gz + sed -i -e "s/Package: @(Package)/Package: @(Package.replace('-gz','-gz${GZ_RELEASE}'))/" debian/control.em + sed -i -e "s/Source: @(Package)/Source: @(Package.replace('-gz','-gz${GZ_RELEASE}'))/" debian/control.em + sed -i -e "s/@(Package)/@(Package.replace('-gz','-gz${GZ_RELEASE}'))/" debian/changelog.em + git commit debian/control.em debian/changelog.em -m "Patch name to release ${GZ_RELEASE} version" + # Include conflict with initial package name in ROS + sed -i -e '/^Depends/a\Conflicts: \@(Package)' debian/control.em + + git commit debian/control.em -m "Set up a conflict with official ROS packages" + git push origin "debian/$distro/$pkg" + done +done From 40e234ecf686706b87aada6bc6743f98ec3963a3 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 10 May 2023 17:35:53 +0200 Subject: [PATCH 373/540] Move release-bloom.py wrapper to parent dir Signed-off-by: Jose Luis Rivero --- bloom/{ros_gazebo_pkgs => }/release-bloom.py | 0 bloom/ros_gazebo_pkgs/ros_gazebo_pkgs-release.py.bash | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename bloom/{ros_gazebo_pkgs => }/release-bloom.py (100%) diff --git a/bloom/ros_gazebo_pkgs/release-bloom.py b/bloom/release-bloom.py similarity index 100% rename from bloom/ros_gazebo_pkgs/release-bloom.py rename to bloom/release-bloom.py diff --git a/bloom/ros_gazebo_pkgs/ros_gazebo_pkgs-release.py.bash b/bloom/ros_gazebo_pkgs/ros_gazebo_pkgs-release.py.bash index 07e97cebd..721954fb1 100755 --- a/bloom/ros_gazebo_pkgs/ros_gazebo_pkgs-release.py.bash +++ b/bloom/ros_gazebo_pkgs/ros_gazebo_pkgs-release.py.bash @@ -14,5 +14,5 @@ if [[ ${1%-*} != "${1}" ]]; then fi for p in gazebo-dev gazebo-msgs gazebo-plugins gazebo-ros gazebo-ros-control gazebo-ros-pkgs; do - ./release-bloom.py ${p} $(for i in $@; do echo -n "$i "; done) + ../release-bloom.py ${p} $(for i in $@; do echo -n "$i "; done) done From 7983f756588dde854dce8fd31a014b9dfd3cf05b Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 10 May 2023 17:40:34 +0200 Subject: [PATCH 374/540] Modernize the release-bloom script * Python3 modifications * Update ROS2 distros Signed-off-by: Jose Luis Rivero --- bloom/release-bloom.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/bloom/release-bloom.py b/bloom/release-bloom.py index d93c41f2a..e6d353e57 100755 --- a/bloom/release-bloom.py +++ b/bloom/release-bloom.py @@ -1,10 +1,11 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 from __future__ import print_function +import argparse import subprocess import sys -import urllib -import argparse +import urllib.request +import urllib.parse USAGE = 'release-bloom.py [--ignition-version version_name] ' JENKINS_URL = 'http://build.osrfoundation.org' @@ -77,13 +78,10 @@ def call_jenkins_jobs(argv): params['ROS2'] = True ubuntu_per_ros_distro = UBUNTU_DISTROS_IN_ROS2 - if args.ignition_version: - params['IGNITION_VERSION'] = args.ignition_version - if not args.release_version: args.release_version = 0 params['RELEASE_VERSION'] = args.release_version - params_query = urllib.urlencode(params) + params_query = urllib.parse.urlencode(params) base_url = '%s/job/%s/buildWithParameters?%s' % \ (JENKINS_URL, JOB_NAME_PATTERN % (args.package), params_query) @@ -93,7 +91,7 @@ def call_jenkins_jobs(argv): (base_url, arch, ubuntu_distro, args.ros_distro) print('Accessing: %s' % (url)) if not DRY_RUN: - urllib.urlopen(url) + urllib.request.urlopen(url) if __name__ == '__main__': From f0a31c06118ac6f94d6e65f7516c9efe09c43095 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 18 May 2023 00:32:05 +0200 Subject: [PATCH 375/540] Update ROS versions and remove ignition Signed-off-by: Jose Luis Rivero --- bloom/release-bloom.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/bloom/release-bloom.py b/bloom/release-bloom.py index e6d353e57..9785b0d58 100755 --- a/bloom/release-bloom.py +++ b/bloom/release-bloom.py @@ -7,16 +7,15 @@ import urllib.request import urllib.parse -USAGE = 'release-bloom.py [--ignition-version version_name] ' +USAGE = 'release-bloom.py ' JENKINS_URL = 'http://build.osrfoundation.org' JOB_NAME_PATTERN = '%s-bloom-debbuilder' UBUNTU_ARCHS = ['amd64'] -# not releasing for precise by default -UBUNTU_DISTROS_IN_ROS = {'melodic': ['bionic'], - 'noetic': ['focal']} +UBUNTU_DISTROS_IN_ROS = {'noetic': ['focal']} UBUNTU_DISTROS_IN_ROS2 = {'foxy': ['focal'], - 'rolling': ['focal']} + 'humble': ['jammy'], + 'rolling': ['jammy']} DRY_RUN = False def parse_args(argv): @@ -35,9 +34,6 @@ def parse_args(argv): help='Release version suffix; usually 1 (e.g., 1') parser.add_argument('--upload-to-repo', dest='upload_to_repository', default="stable", help='OSRF repo to upload: stable | prerelease | nightly') - parser.add_argument('--ignition-version', action='store', default=None, - help='Gazebo version to use in the binary packages') - args = parser.parse_args() DRY_RUN = args.dry_run return args From d0639f9519c6c26f1e32f380a157ba86edf57c14 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 18 May 2023 13:45:39 +0200 Subject: [PATCH 376/540] Script to release a new version of the ros_gz wrappers Signed-off-by: Jose Luis Rivero --- bloom/ros_gz/Dockerfile | 26 +++++++++++++++++++ bloom/ros_gz/README.md | 32 ++++++++++++++++++++++++ bloom/ros_gz/_rosdep_wrapper.bash | 6 +++++ bloom/ros_gz/bloom_from_special_env.bash | 24 ++++++++++++++++++ bloom/ros_gz/ros_gz-release.py.bash | 18 +++++++++++++ 5 files changed, 106 insertions(+) create mode 100644 bloom/ros_gz/Dockerfile create mode 100755 bloom/ros_gz/_rosdep_wrapper.bash create mode 100755 bloom/ros_gz/bloom_from_special_env.bash create mode 100755 bloom/ros_gz/ros_gz-release.py.bash diff --git a/bloom/ros_gz/Dockerfile b/bloom/ros_gz/Dockerfile new file mode 100644 index 000000000..3257be2f0 --- /dev/null +++ b/bloom/ros_gz/Dockerfile @@ -0,0 +1,26 @@ +FROM ros:humble-ros-base-jammy + +ENV LANG C +ENV LC_ALL C +ARG DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3-bloom \ + wget \ + && rm -rf /var/lib/apt/lists/* + +# Install osrf-rosdep for Garden +RUN sudo bash -c \ + 'wget https://raw.githubusercontent.com/osrf/osrf-rosdep/master/gz/00-gazebo.list -O /etc/ros/rosdep/sources.list.d/00-gazebo.list' +# Replace Fortress with Garden to make bloom to work +# since it does not support arbitrary environemnt variables +RUN sudo bash -c \ + 'wget https://raw.githubusercontent.com/osrf/osrf-rosdep/master/gz/replace_fortress_with_garden/00-replace-gz-fortress-with-garden.list -O /etc/ros/rosdep/sources.list.d/00-replace-gz-fortress-with-garden.list' + +# Not for using bloom but useful for rosdep calls +ENV GZ_VERSION=garden +ENV IGNITION_VERSION=garden + +WORKDIR /tmp +COPY _rosdep_wrapper.bash . +ENTRYPOINT ["/tmp/_rosdep_wrapper.bash"] diff --git a/bloom/ros_gz/README.md b/bloom/ros_gz/README.md index 04ac073bb..74299a412 100644 --- a/bloom/ros_gz/README.md +++ b/bloom/ros_gz/README.md @@ -5,6 +5,9 @@ 2. Initial setup * Create the alternative -release repository * Create a custom track in tracks.yaml +3. Run a new release + * Prerequisites + * Bloom a new release ## 1. Background @@ -61,3 +64,32 @@ New versioning requires bumping to large numbers. Set: ``` All non ubuntu generators can be removed. + +## 3. Run a new release + +To execute a new release of the ros_gz unofficial wrappers there are mainly two +steps to do: use bloom to generate new metadata in the -release repo fork and +use ros_gz-release.py script to trigger the builds builds.osrfoundation.org. + +The new version will be the latest one released in the rosdistro index of the +official ros_gz packages. + +### Prerequisites + +The host system to release should have `docker` and `rocker` installed. + +### 3.1 Bloom a new release + +Blooming a new release of the ros_gz unofficial wrappers requires some changes +to be done in the releasing enviroment affecting the rosdep rules. To facilitate +this, there is a `Dockerfile` that provides the needed modifications and a script +that encapsulates the bloom arguments to be passed and the use of this enviroment. + +``` +./bloom_from_special_env.bash +``` + +The script will create the docker enviroment with the rosdep modifications needed +and invoke rocker with `--home` and `--user` flags to pass the credentials and +customatizations needed for the bloom call. It will run the `bloom-release` command +with the arguments required for the ros_gz wrappers. diff --git a/bloom/ros_gz/_rosdep_wrapper.bash b/bloom/ros_gz/_rosdep_wrapper.bash new file mode 100755 index 000000000..0d4da8f71 --- /dev/null +++ b/bloom/ros_gz/_rosdep_wrapper.bash @@ -0,0 +1,6 @@ +#!/bin/bash +set -e + +rosdep update + +exec "$@" diff --git a/bloom/ros_gz/bloom_from_special_env.bash b/bloom/ros_gz/bloom_from_special_env.bash new file mode 100755 index 000000000..f8a20b000 --- /dev/null +++ b/bloom/ros_gz/bloom_from_special_env.bash @@ -0,0 +1,24 @@ +#!/bin/bash + +if ! command rocker &> /dev/null +then + echo "Please install the rocker app https://github.com/osrf/rocker" + exit 1 +fi + +# TODO: update URL for the release repo +BLOOM_CMD="/usr/bin/bloom-release --no-pull-request \ + --rosdistro humble \ + --track humble_gzgarden \ + --override-release-repository-url \ + https://github.com/j-rivero/ros_ign-release ros_gz" +# +TAG_NAME=${TAG_NAME:-ros_gz_fortress_garden_release} + +echo " * Build the docker release environment" +docker build . -t "${TAG_NAME}" +echo " * Using rocker to enter in the release environment" +rocker --home --user "${TAG_NAME}" "${BLOOM_CMD}" +echo " * Exit the docker release environment" +echo " * Restoring the rosdep cache in the user" +rosdep update diff --git a/bloom/ros_gz/ros_gz-release.py.bash b/bloom/ros_gz/ros_gz-release.py.bash new file mode 100755 index 000000000..18b959c6c --- /dev/null +++ b/bloom/ros_gz/ros_gz-release.py.bash @@ -0,0 +1,18 @@ +#!/bin/bash +# Launch all the suite of ros_gazebo_pkgs: +# Usage: ros_gz-release.py.bash +# + +if [[ $# -lt 2 ]]; then + echo "$0 'other arguments used in release.py'" + exit 1 +fi + +if [[ ${1%-*} != "${1}" ]]; then + echo "Version should not contain a revision number. Use -r argument" + exit 1 +fi + +for p in ros-gz ros-gz-bridge ros-gz-image ros-gz-interfaces ros-gz-sim ros-gz-sim-demos; do + ../release-bloom.py "${p}" $(for i in $@; do echo -n "$i "; done) +done From 3e56674977ee4d303bba5cda3e57c1ec4a74655d Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 2 Jun 2023 13:01:21 +0200 Subject: [PATCH 377/540] Make nightly_labeler to run during the nightly generation This way we can be sure that any node killed during the nightly generation is being replaced by a new one. Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/_configs_/Globals.groovy | 5 ++++- jenkins-scripts/dsl/core.dsl | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/jenkins-scripts/dsl/_configs_/Globals.groovy b/jenkins-scripts/dsl/_configs_/Globals.groovy index 0ea44c616..81279ed9b 100644 --- a/jenkins-scripts/dsl/_configs_/Globals.groovy +++ b/jenkins-scripts/dsl/_configs_/Globals.groovy @@ -12,8 +12,11 @@ class Globals static CRON_EVERY_THREE_DAYS = 'H H * * H/3' static CRON_ON_WEEKEND = 'H H * * 6-7' - static CRON_PREPARE_NIGHTLY = '45 8 * * *' static CRON_START_NIGHTLY = '0 9 * * *' + // Run nightly scheduler during the nightly creation to be sure + // that any possible node killed is replaced. Starting -15min + // before CRON_NIGHTLY_NODES and evert 20min for 3 hours + static CRON_NIGHTLY_NODES = '45/20 8-11 * * *' // Only one -E regex can be passed, so make a regex that matches both // _ign_TEST and _gz_TEST diff --git a/jenkins-scripts/dsl/core.dsl b/jenkins-scripts/dsl/core.dsl index 6800dd94f..8422a27bf 100644 --- a/jenkins-scripts/dsl/core.dsl +++ b/jenkins-scripts/dsl/core.dsl @@ -105,7 +105,7 @@ nightly_labeler.with label Globals.nontest_label("master") triggers { - cron(Globals.CRON_PREPARE_NIGHTLY) + cron(Globals.CRON_NIGHTLY_NODES) } steps From b59f397baffca64a533062c5f0b89d839a2ec86c Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 6 Jun 2023 18:22:05 +0200 Subject: [PATCH 378/540] Unofficial ros_gz: add a smoke test to detect ABI breakages (#950) Smoke test designed to mainly identify extreme breakages in the software with the eyes set on any possible ABI disruptive change that ROS dependencies of the wrappers could execute. To do so, the testing does: * Reproduce ros_gz talker/listener and check for expected output * Execute ros2 launch ros_gz_sim gz_sim.launch.py gz_args:=shapes.sdf and leave it for 3 min checking that indeed it has been running for 3min. --- .../ros_gz_bridge-release.py.bash | 18 ------- .../docker/ros_gz-install-test-job.bash | 50 +++++++++++++++++++ jenkins-scripts/dsl/ros_gz_bridge.dsl | 6 ++- 3 files changed, 54 insertions(+), 20 deletions(-) delete mode 100755 bloom/ros_gazebo_pkgs/ros_gz_bridge-release.py.bash create mode 100644 jenkins-scripts/docker/ros_gz-install-test-job.bash diff --git a/bloom/ros_gazebo_pkgs/ros_gz_bridge-release.py.bash b/bloom/ros_gazebo_pkgs/ros_gz_bridge-release.py.bash deleted file mode 100755 index 0e5534821..000000000 --- a/bloom/ros_gazebo_pkgs/ros_gz_bridge-release.py.bash +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -# Launch all the suite of ros_gazebo_pkgs: -# Usage: ros_gz_bridge-release.py.bash -# - -if [[ $# -lt 2 ]]; then - echo "$0 'other arguments used in release.py'" - exit 1 -fi - -if [[ ${1%-*} != "${1}" ]]; then - echo "Version should not contain a revision number. Use -r argument" - exit 1 -fi - -for p in ros-ign-image ros-ign-bridge ros-ign-gazebo-demos ros-ign-point-cloud ros-ign ros-ign-gazebo; do - ./release-bloom.py "${p}" $(for i in $@; do echo -n "$i "; done) -done diff --git a/jenkins-scripts/docker/ros_gz-install-test-job.bash b/jenkins-scripts/docker/ros_gz-install-test-job.bash new file mode 100644 index 000000000..1421d0003 --- /dev/null +++ b/jenkins-scripts/docker/ros_gz-install-test-job.bash @@ -0,0 +1,50 @@ +#!/bin/bash -x + +# Knowing Script dir beware of symlink +[[ -L ${0} ]] && SCRIPT_DIR=$(readlink ${0}) || SCRIPT_DIR=${0} +SCRIPT_DIR="${SCRIPT_DIR%/*}" + +[[ -z ${ENABLE_GZ_SIM_RUNTIME_TEST} ]] && ENABLE_GZ_SIM_RUNTIME_TEST=true + +export GPU_SUPPORT_NEEDED=true + +if [[ -z ${ROS_DISTRO} ]]; then + echo "ROS_DISTRO is empty. Can not run test" + exit 1 +fi + +export INSTALL_JOB_POSTINSTALL_HOOK=""" +. /opt/ros/${ROS_DISTRO}/setup.bash + +echo '# BEGIN SECTION: ros_gz talker/listener' +ros2 run ros_gz_bridge parameter_bridge /chatter@std_msgs/msg/String@gz.msgs.StringMsg 2>/dev/null & +sleep 1 +ros2 topic echo /chatter > /tmp/echo_chatter & +sleep 1 +gz topic -t /chatter -m gz.msgs.StringMsg -p 'data:\"Hello\"' & +sleep 1 +if ! grep -v Hello /tmp/echo_chatter; then + echo 'chatter log file does not contain the expected Hello string' + exit 1 +fi +echo '# END SECTION' + +echo '# BEGIN SECTION: test gz_sim via ros2 launch' +TEST_START=\`date +%s\` +# preserve-status did not work here +timeout 180 ros2 launch ros_gz_sim gz_sim.launch.py gz_args:=shapes.sdf || true +TEST_END=\`date +%s\` +DIFF=\$((\$TEST_END-\$TEST_START)) + +if [ \$DIFF -lt 180 ]; then + echo 'The test took less than 180s. Something bad happened' + exit 1 +fi +""" + +# Need bc to proper testing and parsing the time +# TODO: fix gz-sim-cli dependency on ros_gz package +# to avoid hardcoded version on gz-sim +export DEPENDENCY_PKGS="${DEPENDENCY_PKGS} wget ros-${ROS_DISTRO}-ros-base gz-sim7-cli" + +. ${SCRIPT_DIR}/lib/generic-install-base.bash diff --git a/jenkins-scripts/dsl/ros_gz_bridge.dsl b/jenkins-scripts/dsl/ros_gz_bridge.dsl index 52e726962..1dc1b01fb 100644 --- a/jenkins-scripts/dsl/ros_gz_bridge.dsl +++ b/jenkins-scripts/dsl/ros_gz_bridge.dsl @@ -172,9 +172,11 @@ install_test_job.with shell("""\ #!/bin/bash -xe - export INSTALL_JOB_PKG=ros-\${ROS_DISTRO}-ros-gz\${GZ_VERSION}-sim + export INSTALL_JOB_PKG=ros-\${ROS_DISTRO}-ros-gz\${GZ_VERSION} + export INSTALL_JOB_REPOS=\${OSRF_REPOS_TO_USE} + export USE_ROS_REPO=true export ROS2=true - /bin/bash -x ./scripts/jenkins-scripts/docker/gz_launch-install-test-job.bash + /bin/bash -x ./scripts/jenkins-scripts/docker/ros_gz-install-test-job.bash """.stripIndent()) } } From 296086efd9d34d0beda9297fcd6c46d5483ca86e Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 7 Jun 2023 16:10:33 +0200 Subject: [PATCH 379/540] Added new DSL to generate gz-cmake Harmonic -ci-pr_any- job based on gz-collections.yaml configuration (#951) * Added new DSL to generate Harmonic -ci-pr_any- jobs based on yaml configuration. * Reduce libs to just gz-cmake to test * Add ci metadata to gz-collections for harmonic * Make BUILDING_SOFTWARE_DIRECTORY variable to respect previous values Signed-off-by: Jose Luis Rivero --- .../docker/gz_cmake-compilation.bash | 2 +- .../docker/gz_common-compilation.bash | 2 +- .../docker/gz_fuel-tools-compilation.bash | 2 +- .../docker/gz_gazebo-compilation.bash | 2 +- .../docker/gz_gui-compilation.bash | 2 +- .../docker/gz_launch-compilation.bash | 2 +- .../docker/gz_math-compilation.bash | 2 +- .../docker/gz_msgs-compilation.bash | 2 +- .../docker/gz_physics-compilation.bash | 2 +- .../docker/gz_plugin-compilation.bash | 2 +- .../docker/gz_rendering-compilation.bash | 2 +- .../docker/gz_rndf-compilation.bash | 2 +- .../docker/gz_sensors-compilation.bash | 2 +- .../docker/gz_tools-compilation.bash | 2 +- .../docker/gz_transport-compilation.bash | 2 +- .../docker/gz_utils-compilation.bash | 2 +- .../docker/sdformat-compilation.bash | 2 +- jenkins-scripts/dsl/gazebo_libs.dsl | 78 +++++++++++++++++++ jenkins-scripts/dsl/gz-collections.yaml | 27 +++++++ 19 files changed, 122 insertions(+), 17 deletions(-) create mode 100644 jenkins-scripts/dsl/gazebo_libs.dsl diff --git a/jenkins-scripts/docker/gz_cmake-compilation.bash b/jenkins-scripts/docker/gz_cmake-compilation.bash index b956bfd5b..100f50d10 100644 --- a/jenkins-scripts/docker/gz_cmake-compilation.bash +++ b/jenkins-scripts/docker/gz_cmake-compilation.bash @@ -14,7 +14,7 @@ if [[ -z ${DISTRO} ]]; then exit 1 fi -export BUILDING_SOFTWARE_DIRECTORY="ign-cmake" +export BUILDING_SOFTWARE_DIRECTORY="${BUILDING_SOFTWARE_DIRECTORY:-ign-cmake}" export BUILDING_DEPENDENCIES="pkg-config" # Enable long-running ign-cmake tests in CI. diff --git a/jenkins-scripts/docker/gz_common-compilation.bash b/jenkins-scripts/docker/gz_common-compilation.bash index 2f1ecd1f0..cd66219e2 100644 --- a/jenkins-scripts/docker/gz_common-compilation.bash +++ b/jenkins-scripts/docker/gz_common-compilation.bash @@ -14,7 +14,7 @@ if [[ -z ${DISTRO} ]]; then exit 1 fi -export BUILDING_SOFTWARE_DIRECTORY="ign-common" +export BUILDING_SOFTWARE_DIRECTORY="${BUILDING_SOFTWARE_DIRECTORY:-ign-common}" export BUILDING_PKG_DEPENDENCIES_VAR_NAME="GZ_COMMON_DEPENDENCIES" # Identify GZ_COMMON_MAJOR_VERSION to help with dependency resolution diff --git a/jenkins-scripts/docker/gz_fuel-tools-compilation.bash b/jenkins-scripts/docker/gz_fuel-tools-compilation.bash index 7bed0ae12..d51f3c5da 100644 --- a/jenkins-scripts/docker/gz_fuel-tools-compilation.bash +++ b/jenkins-scripts/docker/gz_fuel-tools-compilation.bash @@ -14,7 +14,7 @@ if [[ -z ${DISTRO} ]]; then exit 1 fi -export BUILDING_SOFTWARE_DIRECTORY="ign-fuel-tools" +export BUILDING_SOFTWARE_DIRECTORY="${BUILDING_SOFTWARE_DIRECTORY:-ign-fuel-tools}" export BUILDING_PKG_DEPENDENCIES_VAR_NAME="GZ_FUEL_TOOLS_DEPENDENCIES" export BUILDING_JOB_REPOSITORIES="stable" diff --git a/jenkins-scripts/docker/gz_gazebo-compilation.bash b/jenkins-scripts/docker/gz_gazebo-compilation.bash index 86caabcbc..c1f5b1272 100644 --- a/jenkins-scripts/docker/gz_gazebo-compilation.bash +++ b/jenkins-scripts/docker/gz_gazebo-compilation.bash @@ -14,7 +14,7 @@ if [[ -z ${DISTRO} ]]; then exit 1 fi -export BUILDING_SOFTWARE_DIRECTORY="ign-gazebo" +export BUILDING_SOFTWARE_DIRECTORY="${BUILDING_SOFTWARE_DIRECTORY:-ign-gazebo}" export BUILDING_PKG_DEPENDENCIES_VAR_NAME="GZ_SIM_DEPENDENCIES" # Identify GZ_SIM_MAJOR_VERSION to help with dependency resolution diff --git a/jenkins-scripts/docker/gz_gui-compilation.bash b/jenkins-scripts/docker/gz_gui-compilation.bash index 4c00c02d2..81fa4f5a8 100644 --- a/jenkins-scripts/docker/gz_gui-compilation.bash +++ b/jenkins-scripts/docker/gz_gui-compilation.bash @@ -14,7 +14,7 @@ if [[ -z ${DISTRO} ]]; then exit 1 fi -export BUILDING_SOFTWARE_DIRECTORY="ign-gui" +export BUILDING_SOFTWARE_DIRECTORY="${BUILDING_SOFTWARE_DIRECTORY:-ign-gui}" export BUILDING_PKG_DEPENDENCIES_VAR_NAME="GZ_GUI_DEPENDENCIES" # Identify GZ_GUI_MAJOR_VERSION to help with dependency resolution diff --git a/jenkins-scripts/docker/gz_launch-compilation.bash b/jenkins-scripts/docker/gz_launch-compilation.bash index 7e5923e1c..f333c28e2 100644 --- a/jenkins-scripts/docker/gz_launch-compilation.bash +++ b/jenkins-scripts/docker/gz_launch-compilation.bash @@ -14,7 +14,7 @@ if [[ -z ${DISTRO} ]]; then exit 1 fi -export BUILDING_SOFTWARE_DIRECTORY="ign-launch" +export BUILDING_SOFTWARE_DIRECTORY="${BUILDING_SOFTWARE_DIRECTORY:-ign-launch}" export BUILDING_PKG_DEPENDENCIES_VAR_NAME="GZ_LAUNCH_DEPENDENCIES" # Identify GZ_LAUNCH_MAJOR_VERSION to help with dependency resolution diff --git a/jenkins-scripts/docker/gz_math-compilation.bash b/jenkins-scripts/docker/gz_math-compilation.bash index f18a769f3..387168a18 100644 --- a/jenkins-scripts/docker/gz_math-compilation.bash +++ b/jenkins-scripts/docker/gz_math-compilation.bash @@ -14,7 +14,7 @@ if [[ -z ${DISTRO} ]]; then exit 1 fi -export BUILDING_SOFTWARE_DIRECTORY="ign-math" +export BUILDING_SOFTWARE_DIRECTORY="${BUILDING_SOFTWARE_DIRECTORY:-ign-math}" export BUILDING_JOB_REPOSITORIES="stable" export BUILDING_PKG_DEPENDENCIES_VAR_NAME="GZ_MATH_DEPENDENCIES" diff --git a/jenkins-scripts/docker/gz_msgs-compilation.bash b/jenkins-scripts/docker/gz_msgs-compilation.bash index 8dbcd48e0..2705df184 100644 --- a/jenkins-scripts/docker/gz_msgs-compilation.bash +++ b/jenkins-scripts/docker/gz_msgs-compilation.bash @@ -14,7 +14,7 @@ if [[ -z ${DISTRO} ]]; then exit 1 fi -export BUILDING_SOFTWARE_DIRECTORY="ign-msgs" +export BUILDING_SOFTWARE_DIRECTORY="${BUILDING_SOFTWARE_DIRECTORY:-ign-msgs}" export BUILDING_PKG_DEPENDENCIES_VAR_NAME="GZ_MSGS_DEPENDENCIES" # Identify GZ_MSGS_MAJOR_VERSION to help with dependency resolution diff --git a/jenkins-scripts/docker/gz_physics-compilation.bash b/jenkins-scripts/docker/gz_physics-compilation.bash index d57594a5c..b634e8cbb 100644 --- a/jenkins-scripts/docker/gz_physics-compilation.bash +++ b/jenkins-scripts/docker/gz_physics-compilation.bash @@ -14,7 +14,7 @@ if [[ -z ${DISTRO} ]]; then exit 1 fi -export BUILDING_SOFTWARE_DIRECTORY="ign-physics" +export BUILDING_SOFTWARE_DIRECTORY="${BUILDING_SOFTWARE_DIRECTORY:-ign-physics}" export BUILDING_PKG_DEPENDENCIES_VAR_NAME="GZ_PHYSICS_DEPENDENCIES" # Identify GZ_PHYSICS_MAJOR_VERSION to help with dependency resolution diff --git a/jenkins-scripts/docker/gz_plugin-compilation.bash b/jenkins-scripts/docker/gz_plugin-compilation.bash index f6362e131..621330640 100644 --- a/jenkins-scripts/docker/gz_plugin-compilation.bash +++ b/jenkins-scripts/docker/gz_plugin-compilation.bash @@ -14,7 +14,7 @@ if [[ -z ${DISTRO} ]]; then exit 1 fi -export BUILDING_SOFTWARE_DIRECTORY="ign-plugin" +export BUILDING_SOFTWARE_DIRECTORY="${BUILDING_SOFTWARE_DIRECTORY:-ign-plugin}" export BUILDING_PKG_DEPENDENCIES_VAR_NAME="GZ_PLUGIN_DEPENDENCIES" # Identify GZ_PLUGIN_MAJOR_VERSION to help with dependency resolution diff --git a/jenkins-scripts/docker/gz_rendering-compilation.bash b/jenkins-scripts/docker/gz_rendering-compilation.bash index a350093b2..420bd7a19 100644 --- a/jenkins-scripts/docker/gz_rendering-compilation.bash +++ b/jenkins-scripts/docker/gz_rendering-compilation.bash @@ -14,7 +14,7 @@ if [[ -z ${DISTRO} ]]; then exit 1 fi -export BUILDING_SOFTWARE_DIRECTORY="ign-rendering" +export BUILDING_SOFTWARE_DIRECTORY="${BUILDING_SOFTWARE_DIRECTORY:-ign-rendering}" export BUILDING_PKG_DEPENDENCIES_VAR_NAME="GZ_RENDERING_DEPENDENCIES" # Identify GZ_RENDERING_MAJOR_VERSION to help with dependency resolution diff --git a/jenkins-scripts/docker/gz_rndf-compilation.bash b/jenkins-scripts/docker/gz_rndf-compilation.bash index 6799fc31d..899a2b03a 100644 --- a/jenkins-scripts/docker/gz_rndf-compilation.bash +++ b/jenkins-scripts/docker/gz_rndf-compilation.bash @@ -14,7 +14,7 @@ if [[ -z ${DISTRO} ]]; then exit 1 fi -export BUILDING_SOFTWARE_DIRECTORY="ign-rndf" +export BUILDING_SOFTWARE_DIRECTORY="${BUILDING_SOFTWARE_DIRECTORY:-ign-rndf}" export BUILDING_PKG_DEPENDENCIES_VAR_NAME="GZ_RNDF_DEPENDENCIES" export BUILDING_JOB_REPOSITORIES="stable" diff --git a/jenkins-scripts/docker/gz_sensors-compilation.bash b/jenkins-scripts/docker/gz_sensors-compilation.bash index 16c740653..5f291b3e7 100644 --- a/jenkins-scripts/docker/gz_sensors-compilation.bash +++ b/jenkins-scripts/docker/gz_sensors-compilation.bash @@ -14,7 +14,7 @@ if [[ -z ${DISTRO} ]]; then exit 1 fi -export BUILDING_SOFTWARE_DIRECTORY="ign-sensors" +export BUILDING_SOFTWARE_DIRECTORY="${BUILDING_SOFTWARE_DIRECTORY:-ign-sensors}" export BUILDING_PKG_DEPENDENCIES_VAR_NAME="GZ_SENSORS_DEPENDENCIES" # Identify GZ_SENSORS_MAJOR_VERSION to help with dependency resolution diff --git a/jenkins-scripts/docker/gz_tools-compilation.bash b/jenkins-scripts/docker/gz_tools-compilation.bash index caceddc42..56b82b415 100644 --- a/jenkins-scripts/docker/gz_tools-compilation.bash +++ b/jenkins-scripts/docker/gz_tools-compilation.bash @@ -14,7 +14,7 @@ if [[ -z ${DISTRO} ]]; then exit 1 fi -export BUILDING_SOFTWARE_DIRECTORY="ign-tools" +export BUILDING_SOFTWARE_DIRECTORY="${BUILDING_SOFTWARE_DIRECTORY:-ign-tools}" export BUILDING_JOB_REPOSITORIES="stable" export BUILDING_DEPENDENCIES="ruby" diff --git a/jenkins-scripts/docker/gz_transport-compilation.bash b/jenkins-scripts/docker/gz_transport-compilation.bash index c6152eea9..7cd6a6b36 100644 --- a/jenkins-scripts/docker/gz_transport-compilation.bash +++ b/jenkins-scripts/docker/gz_transport-compilation.bash @@ -14,7 +14,7 @@ if [[ -z ${DISTRO} ]]; then exit 1 fi -export BUILDING_SOFTWARE_DIRECTORY="ign-transport" +export BUILDING_SOFTWARE_DIRECTORY="${BUILDING_SOFTWARE_DIRECTORY:-ign-transport}" export BUILDING_PKG_DEPENDENCIES_VAR_NAME="GZ_TRANSPORT_DEPENDENCIES" # Identify GZ_TRANSPORT_MAJOR_VERSION to help with dependency resolution diff --git a/jenkins-scripts/docker/gz_utils-compilation.bash b/jenkins-scripts/docker/gz_utils-compilation.bash index a30db9f35..13d2e09b3 100644 --- a/jenkins-scripts/docker/gz_utils-compilation.bash +++ b/jenkins-scripts/docker/gz_utils-compilation.bash @@ -14,7 +14,7 @@ if [[ -z ${DISTRO} ]]; then exit 1 fi -export BUILDING_SOFTWARE_DIRECTORY="ign-utils" +export BUILDING_SOFTWARE_DIRECTORY="${BUILDING_SOFTWARE_DIRECTORY:-ign-utils}" export BUILDING_PKG_DEPENDENCIES_VAR_NAME="GZ_UTILS_DEPENDENCIES" # Identify GZ_UTILS_MAJOR_VERSION to help with dependency resolution diff --git a/jenkins-scripts/docker/sdformat-compilation.bash b/jenkins-scripts/docker/sdformat-compilation.bash index 8cfb50a1f..6f7b91895 100644 --- a/jenkins-scripts/docker/sdformat-compilation.bash +++ b/jenkins-scripts/docker/sdformat-compilation.bash @@ -16,7 +16,7 @@ fi . "${SCRIPT_DIR}/lib/_sdformat_version_hook.bash" -export BUILDING_SOFTWARE_DIRECTORY="sdformat" +export BUILDING_SOFTWARE_DIRECTORY="${BUILDING_SOFTWARE_DIRECTORY:-sdformat}" if [[ ${SDFORMAT_MAJOR_VERSION} -ge 8 ]]; then export NEED_C17_COMPILER=true diff --git a/jenkins-scripts/dsl/gazebo_libs.dsl b/jenkins-scripts/dsl/gazebo_libs.dsl new file mode 100644 index 000000000..1f65e356b --- /dev/null +++ b/jenkins-scripts/dsl/gazebo_libs.dsl @@ -0,0 +1,78 @@ +import _configs_.* +import javaposse.jobdsl.dsl.Job + +import org.yaml.snakeyaml.Yaml + +// shell command to inject in all bash steps +GLOBAL_SHELL_CMD='' + +// GZ COLLECTIONS +arch = 'amd64' +ENABLE_CPPCHECK = true + +// Jenkins needs the relative path to work and locally the simulation is done +// using a symlink +file = readFileFromWorkspace("scripts/jenkins-scripts/dsl/gz-collections.yaml") +gz_collections_yaml = new Yaml().load(file) + +boolean include_gpu_label_if_needed(job, lib, ci_info) +{ + job.with + { + if (ci_info.requirements.nvidia_gpu.contains(lib.name)) + { + label Globals.nontest_label("gpu-reliable") + + // unstable build if missing valid gpu display + publishers { + consoleParsing { + projectRules('scripts/jenkins-scripts/parser_rules/display_missing.parser') + unstableOnWarning() + failBuildOnError(false) + } + } + } + } +} + +[ 'harmonic' ].each { collection_name -> + collection = gz_collections_yaml.collections. find { it.name == collection_name } + collection.ci.linux.reference_distro.each { distro -> + collection.libs.findAll { ! collection.ci.exclude.contains(it.name) }.each { lib -> + assert(lib.name) + assert(lib.repo.current_branch) + // 1.2.1 Main PR jobs (-ci-pr_any-) (pulling check every 5 minutes) + // -------------------------------------------------------------- + def gz_job_name_prefix = lib.name.replaceAll('-','_') + def gz_ci_job_name = "${gz_job_name_prefix}-ci-pr_any-${distro}-${arch}" + def gz_ci_any_job = job(gz_ci_job_name) + def enable_testing = (collection.ci.tests_disabled?.contains(lib.name)) ? false : true + OSRFLinuxCompilationAnyGitHub.create(gz_ci_any_job, + "gazebosim/${lib.name}", + enable_testing, + ENABLE_CPPCHECK, + [ lib.repo.current_branch ]) + include_gpu_label_if_needed(gz_ci_any_job, lib, collection.ci) + gz_ci_any_job.with + { + if (collection.ci.requirements.large_memory?.contains(lib.name)) + label Globals.nontest_label("large-memory") + + steps + { + shell("""\ + #!/bin/bash -xe + + export DISTRO=${distro} + + ${GLOBAL_SHELL_CMD} + + export BUILDING_SOFTWARE_DIRECTORY=${lib.name} + export ARCH=${arch} + /bin/bash -xe ./scripts/jenkins-scripts/docker/${gz_job_name_prefix}-compilation.bash + """.stripIndent()) + } // end of steps + } // end of ci_any_job + } //en of lib + } // end of distro +} // end of collection diff --git a/jenkins-scripts/dsl/gz-collections.yaml b/jenkins-scripts/dsl/gz-collections.yaml index 1b65b2e85..7d08f3ee8 100644 --- a/jenkins-scripts/dsl/gz-collections.yaml +++ b/jenkins-scripts/dsl/gz-collections.yaml @@ -118,9 +118,36 @@ collections: repo: current_branch: main ci: + exclude: + - gz-harmonic +# Remove the exclusion after gz-cmake testing + - gz-tools + - gz-utils + - gz-math + - gz-plugin + - gz-common + - gz-msgs + - gz-rendering + - gz-fuel-tools + - gz-transport + - gz-gui + - gz-sensors + - gz-physics + - gz-sim + - gz-launch + - sdformat linux: reference_distro: - jammy + requirements: + large_memory: + - gz-physics + nvidia_gpu: + - gz-sim + - gz-gui + - gz-rendering + - gz-sensors + tests_disabled: packaging: linux: package_name: From 0c2504a7c987913529b9d9351e8783e57810e322 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 7 Jun 2023 16:34:04 +0200 Subject: [PATCH 380/540] Fix detect_cmake_major_version.py to work with gz branches (#952) Signed-off-by: Jose Luis Rivero --- jenkins-scripts/docker/gz_cmake-compilation.bash | 2 +- jenkins-scripts/docker/gz_common-compilation.bash | 2 +- jenkins-scripts/docker/gz_fuel-tools-compilation.bash | 2 +- jenkins-scripts/docker/gz_gazebo-compilation.bash | 2 +- jenkins-scripts/docker/gz_gui-compilation.bash | 2 +- jenkins-scripts/docker/gz_launch-compilation.bash | 2 +- jenkins-scripts/docker/gz_math-compilation.bash | 2 +- jenkins-scripts/docker/gz_msgs-compilation.bash | 2 +- jenkins-scripts/docker/gz_physics-compilation.bash | 2 +- jenkins-scripts/docker/gz_plugin-compilation.bash | 2 +- jenkins-scripts/docker/gz_rendering-compilation.bash | 2 +- jenkins-scripts/docker/gz_sensors-compilation.bash | 2 +- jenkins-scripts/docker/gz_tools-compilation.bash | 2 +- jenkins-scripts/docker/gz_transport-compilation.bash | 2 +- jenkins-scripts/docker/gz_utils-compilation.bash | 2 +- 15 files changed, 15 insertions(+), 15 deletions(-) diff --git a/jenkins-scripts/docker/gz_cmake-compilation.bash b/jenkins-scripts/docker/gz_cmake-compilation.bash index 100f50d10..5691b6e34 100644 --- a/jenkins-scripts/docker/gz_cmake-compilation.bash +++ b/jenkins-scripts/docker/gz_cmake-compilation.bash @@ -23,7 +23,7 @@ export BUILDING_EXTRA_CMAKE_PARAMS+=" -DBUILDSYSTEM_TESTING=True" # Identify GZ_CMAKE_MAJOR_VERSION to help with dependency resolution GZ_CMAKE_MAJOR_VERSION=$(\ python3 ${SCRIPT_DIR}/../tools/detect_cmake_major_version.py \ - ${WORKSPACE}/ign-cmake/CMakeLists.txt) + ${WORKSPACE}/${BUILDING_SOFTWARE_DIRECTORY}/CMakeLists.txt) # Check GZ_CMAKE version is integer if ! [[ ${GZ_CMAKE_MAJOR_VERSION} =~ ^-?[0-9]+$ ]]; then diff --git a/jenkins-scripts/docker/gz_common-compilation.bash b/jenkins-scripts/docker/gz_common-compilation.bash index cd66219e2..d91540db9 100644 --- a/jenkins-scripts/docker/gz_common-compilation.bash +++ b/jenkins-scripts/docker/gz_common-compilation.bash @@ -20,7 +20,7 @@ export BUILDING_PKG_DEPENDENCIES_VAR_NAME="GZ_COMMON_DEPENDENCIES" # Identify GZ_COMMON_MAJOR_VERSION to help with dependency resolution GZ_COMMON_MAJOR_VERSION=$(\ python3 ${SCRIPT_DIR}/../tools/detect_cmake_major_version.py \ - ${WORKSPACE}/ign-common/CMakeLists.txt) + ${WORKSPACE}/${BUILDING_SOFTWARE_DIRECTORY}/CMakeLists.txt) # Check GZ_COMMON version is integer if ! [[ ${GZ_COMMON_MAJOR_VERSION} =~ ^-?[0-9]+$ ]]; then diff --git a/jenkins-scripts/docker/gz_fuel-tools-compilation.bash b/jenkins-scripts/docker/gz_fuel-tools-compilation.bash index d51f3c5da..3b1670a7f 100644 --- a/jenkins-scripts/docker/gz_fuel-tools-compilation.bash +++ b/jenkins-scripts/docker/gz_fuel-tools-compilation.bash @@ -21,7 +21,7 @@ export BUILDING_JOB_REPOSITORIES="stable" # Identify GZ_FUEL_TOOLS_MAJOR_VERSION to help with dependency resolution GZ_FUEL_TOOLS_MAJOR_VERSION=$(\ python3 ${SCRIPT_DIR}/../tools/detect_cmake_major_version.py \ - ${WORKSPACE}/ign-fuel-tools/CMakeLists.txt) + ${WORKSPACE}/${BUILDING_SOFTWARE_DIRECTORY}/CMakeLists.txt) # Check GZ_FUEL_TOOLS version is integer if ! [[ ${GZ_FUEL_TOOLS_MAJOR_VERSION} =~ ^-?[0-9]+$ ]]; then diff --git a/jenkins-scripts/docker/gz_gazebo-compilation.bash b/jenkins-scripts/docker/gz_gazebo-compilation.bash index c1f5b1272..1fb98d3a7 100644 --- a/jenkins-scripts/docker/gz_gazebo-compilation.bash +++ b/jenkins-scripts/docker/gz_gazebo-compilation.bash @@ -20,7 +20,7 @@ export BUILDING_PKG_DEPENDENCIES_VAR_NAME="GZ_SIM_DEPENDENCIES" # Identify GZ_SIM_MAJOR_VERSION to help with dependency resolution GZ_SIM_MAJOR_VERSION=$(\ python3 ${SCRIPT_DIR}/../tools/detect_cmake_major_version.py \ - ${WORKSPACE}/ign-gazebo/CMakeLists.txt) + ${WORKSPACE}/${BUILDING_SOFTWARE_DIRECTORY}/CMakeLists.txt) # Check IGN_GAZEBO version is integer if ! [[ ${GZ_SIM_MAJOR_VERSION} =~ ^-?[0-9]+$ ]]; then diff --git a/jenkins-scripts/docker/gz_gui-compilation.bash b/jenkins-scripts/docker/gz_gui-compilation.bash index 81fa4f5a8..9670fb7b1 100644 --- a/jenkins-scripts/docker/gz_gui-compilation.bash +++ b/jenkins-scripts/docker/gz_gui-compilation.bash @@ -20,7 +20,7 @@ export BUILDING_PKG_DEPENDENCIES_VAR_NAME="GZ_GUI_DEPENDENCIES" # Identify GZ_GUI_MAJOR_VERSION to help with dependency resolution GZ_GUI_MAJOR_VERSION=$(\ python3 ${SCRIPT_DIR}/../tools/detect_cmake_major_version.py \ - ${WORKSPACE}/ign-gui/CMakeLists.txt) + ${WORKSPACE}/${BUILDING_SOFTWARE_DIRECTORY}/CMakeLists.txt) # Check GZ_GUI_MAJOR_VERSION version is integer if ! [[ ${GZ_GUI_MAJOR_VERSION} =~ ^-?[0-9]+$ ]]; then diff --git a/jenkins-scripts/docker/gz_launch-compilation.bash b/jenkins-scripts/docker/gz_launch-compilation.bash index f333c28e2..189fc96af 100644 --- a/jenkins-scripts/docker/gz_launch-compilation.bash +++ b/jenkins-scripts/docker/gz_launch-compilation.bash @@ -20,7 +20,7 @@ export BUILDING_PKG_DEPENDENCIES_VAR_NAME="GZ_LAUNCH_DEPENDENCIES" # Identify GZ_LAUNCH_MAJOR_VERSION to help with dependency resolution GZ_LAUNCH_MAJOR_VERSION=$(\ python3 ${SCRIPT_DIR}/../tools/detect_cmake_major_version.py \ - ${WORKSPACE}/ign-launch/CMakeLists.txt) + ${WORKSPACE}/${BUILDING_SOFTWARE_DIRECTORY}/CMakeLists.txt) # Check GZ_LAUNCH version is integer if ! [[ ${GZ_LAUNCH_MAJOR_VERSION} =~ ^-?[0-9]+$ ]]; then diff --git a/jenkins-scripts/docker/gz_math-compilation.bash b/jenkins-scripts/docker/gz_math-compilation.bash index 387168a18..716f3663c 100644 --- a/jenkins-scripts/docker/gz_math-compilation.bash +++ b/jenkins-scripts/docker/gz_math-compilation.bash @@ -21,7 +21,7 @@ export BUILDING_PKG_DEPENDENCIES_VAR_NAME="GZ_MATH_DEPENDENCIES" # Identify GZ_MATH_MAJOR_VERSION to help with dependency resolution GZ_MATH_MAJOR_VERSION=$(\ python3 ${SCRIPT_DIR}/../tools/detect_cmake_major_version.py \ - ${WORKSPACE}/ign-math/CMakeLists.txt) + ${WORKSPACE}/${BUILDING_SOFTWARE_DIRECTORY}/CMakeLists.txt) # Check GZ_MATH version is integer if ! [[ ${GZ_MATH_MAJOR_VERSION} =~ ^-?[0-9]+$ ]]; then diff --git a/jenkins-scripts/docker/gz_msgs-compilation.bash b/jenkins-scripts/docker/gz_msgs-compilation.bash index 2705df184..cc083a426 100644 --- a/jenkins-scripts/docker/gz_msgs-compilation.bash +++ b/jenkins-scripts/docker/gz_msgs-compilation.bash @@ -20,7 +20,7 @@ export BUILDING_PKG_DEPENDENCIES_VAR_NAME="GZ_MSGS_DEPENDENCIES" # Identify GZ_MSGS_MAJOR_VERSION to help with dependency resolution GZ_MSGS_MAJOR_VERSION=$(\ python3 ${SCRIPT_DIR}/../tools/detect_cmake_major_version.py \ - ${WORKSPACE}/ign-msgs/CMakeLists.txt) + ${WORKSPACE}/${BUILDING_SOFTWARE_DIRECTORY}/CMakeLists.txt) # Check GZ_MSGS version is integer if ! [[ ${GZ_MSGS_MAJOR_VERSION} =~ ^-?[0-9]+$ ]]; then diff --git a/jenkins-scripts/docker/gz_physics-compilation.bash b/jenkins-scripts/docker/gz_physics-compilation.bash index b634e8cbb..ef9788ea4 100644 --- a/jenkins-scripts/docker/gz_physics-compilation.bash +++ b/jenkins-scripts/docker/gz_physics-compilation.bash @@ -20,7 +20,7 @@ export BUILDING_PKG_DEPENDENCIES_VAR_NAME="GZ_PHYSICS_DEPENDENCIES" # Identify GZ_PHYSICS_MAJOR_VERSION to help with dependency resolution GZ_PHYSICS_MAJOR_VERSION=$(\ python3 ${SCRIPT_DIR}/../tools/detect_cmake_major_version.py \ - ${WORKSPACE}/ign-physics/CMakeLists.txt) + ${WORKSPACE}/${BUILDING_SOFTWARE_DIRECTORY}/CMakeLists.txt) # Check GZ_PHYSICS version is integer if ! [[ ${GZ_PHYSICS_MAJOR_VERSION} =~ ^-?[0-9]+$ ]]; then diff --git a/jenkins-scripts/docker/gz_plugin-compilation.bash b/jenkins-scripts/docker/gz_plugin-compilation.bash index 621330640..b3b5f68f4 100644 --- a/jenkins-scripts/docker/gz_plugin-compilation.bash +++ b/jenkins-scripts/docker/gz_plugin-compilation.bash @@ -20,7 +20,7 @@ export BUILDING_PKG_DEPENDENCIES_VAR_NAME="GZ_PLUGIN_DEPENDENCIES" # Identify GZ_PLUGIN_MAJOR_VERSION to help with dependency resolution GZ_PLUGIN_MAJOR_VERSION=$(\ python3 ${SCRIPT_DIR}/../tools/detect_cmake_major_version.py \ - ${WORKSPACE}/ign-plugin/CMakeLists.txt) + ${WORKSPACE}/${BUILDING_SOFTWARE_DIRECTORY}/CMakeLists.txt) # Check GZ_PLUGIN version is integer if ! [[ ${GZ_PLUGIN_MAJOR_VERSION} =~ ^-?[0-9]+$ ]]; then diff --git a/jenkins-scripts/docker/gz_rendering-compilation.bash b/jenkins-scripts/docker/gz_rendering-compilation.bash index 420bd7a19..caf362f03 100644 --- a/jenkins-scripts/docker/gz_rendering-compilation.bash +++ b/jenkins-scripts/docker/gz_rendering-compilation.bash @@ -20,7 +20,7 @@ export BUILDING_PKG_DEPENDENCIES_VAR_NAME="GZ_RENDERING_DEPENDENCIES" # Identify GZ_RENDERING_MAJOR_VERSION to help with dependency resolution GZ_RENDERING_MAJOR_VERSION=$(\ python3 ${SCRIPT_DIR}/../tools/detect_cmake_major_version.py \ - ${WORKSPACE}/ign-rendering/CMakeLists.txt) + ${WORKSPACE}/${BUILDING_SOFTWARE_DIRECTORY}/CMakeLists.txt) # Check GZ_RENDERING version is integer if ! [[ ${GZ_RENDERING_MAJOR_VERSION} =~ ^-?[0-9]+$ ]]; then diff --git a/jenkins-scripts/docker/gz_sensors-compilation.bash b/jenkins-scripts/docker/gz_sensors-compilation.bash index 5f291b3e7..45a16efb2 100644 --- a/jenkins-scripts/docker/gz_sensors-compilation.bash +++ b/jenkins-scripts/docker/gz_sensors-compilation.bash @@ -20,7 +20,7 @@ export BUILDING_PKG_DEPENDENCIES_VAR_NAME="GZ_SENSORS_DEPENDENCIES" # Identify GZ_SENSORS_MAJOR_VERSION to help with dependency resolution GZ_SENSORS_MAJOR_VERSION=$(\ python3 ${SCRIPT_DIR}/../tools/detect_cmake_major_version.py \ - ${WORKSPACE}/ign-sensors/CMakeLists.txt) + ${WORKSPACE}/${BUILDING_SOFTWARE_DIRECTORY}/CMakeLists.txt) # Check GZ_SENSORS version is integer if ! [[ ${GZ_SENSORS_MAJOR_VERSION} =~ ^-?[0-9]+$ ]]; then diff --git a/jenkins-scripts/docker/gz_tools-compilation.bash b/jenkins-scripts/docker/gz_tools-compilation.bash index 56b82b415..e5f3073b2 100644 --- a/jenkins-scripts/docker/gz_tools-compilation.bash +++ b/jenkins-scripts/docker/gz_tools-compilation.bash @@ -20,7 +20,7 @@ export BUILDING_DEPENDENCIES="ruby" GZ_TOOLS_MAJOR_VERSION=$(\ python3 ${SCRIPT_DIR}/../tools/detect_cmake_major_version.py \ - ${WORKSPACE}/ign-tools/CMakeLists.txt) + ${WORKSPACE}/${BUILDING_SOFTWARE_DIRECTORY}/CMakeLists.txt) # Check GZ_TOOLS version is integer if ! [[ ${GZ_TOOLS_MAJOR_VERSION} =~ ^-?[0-9]+$ ]]; then diff --git a/jenkins-scripts/docker/gz_transport-compilation.bash b/jenkins-scripts/docker/gz_transport-compilation.bash index 7cd6a6b36..d047aa176 100644 --- a/jenkins-scripts/docker/gz_transport-compilation.bash +++ b/jenkins-scripts/docker/gz_transport-compilation.bash @@ -20,7 +20,7 @@ export BUILDING_SOFTWARE_DIRECTORY="${BUILDING_SOFTWARE_DIRECTORY:-ign-transport # Identify GZ_TRANSPORT_MAJOR_VERSION to help with dependency resolution GZ_TRANSPORT_MAJOR_VERSION=$(\ python3 ${SCRIPT_DIR}/../tools/detect_cmake_major_version.py \ - ${WORKSPACE}/ign-transport/CMakeLists.txt) + ${WORKSPACE}/${BUILDING_SOFTWARE_DIRECTORY}/CMakeLists.txt) # Check GZ_TRANSPORT version is integer if ! [[ ${GZ_TRANSPORT_MAJOR_VERSION} =~ ^-?[0-9]+$ ]]; then diff --git a/jenkins-scripts/docker/gz_utils-compilation.bash b/jenkins-scripts/docker/gz_utils-compilation.bash index 13d2e09b3..c1264f720 100644 --- a/jenkins-scripts/docker/gz_utils-compilation.bash +++ b/jenkins-scripts/docker/gz_utils-compilation.bash @@ -20,7 +20,7 @@ export BUILDING_PKG_DEPENDENCIES_VAR_NAME="GZ_UTILS_DEPENDENCIES" # Identify GZ_UTILS_MAJOR_VERSION to help with dependency resolution GZ_UTILS_MAJOR_VERSION=$(\ python3 ${SCRIPT_DIR}/../tools/detect_cmake_major_version.py \ - ${WORKSPACE}/ign-utils/CMakeLists.txt) + ${WORKSPACE}/${BUILDING_SOFTWARE_DIRECTORY}/CMakeLists.txt) # Check GZ_UTILS version is integer if ! [[ ${GZ_UTILS_MAJOR_VERSION} =~ ^-?[0-9]+$ ]]; then From 00e4d2c0ddb9a290ffb7db75dcb092f612f9c33a Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 8 Jun 2023 18:14:32 +0200 Subject: [PATCH 381/540] Enable Harmonic -pr_any-jammy for all libs (#953) * Enable Harmonic -pr_any-jammy for all libs * Use gz_sim instead of gz_gazebo Signed-off-by: Jose Luis Rivero --- ...-compilation.bash => gz_sim-compilation.bash} | 0 jenkins-scripts/dsl/gz-collections.yaml | 16 ---------------- jenkins-scripts/dsl/ignition.dsl | 2 +- 3 files changed, 1 insertion(+), 17 deletions(-) rename jenkins-scripts/docker/{gz_gazebo-compilation.bash => gz_sim-compilation.bash} (100%) diff --git a/jenkins-scripts/docker/gz_gazebo-compilation.bash b/jenkins-scripts/docker/gz_sim-compilation.bash similarity index 100% rename from jenkins-scripts/docker/gz_gazebo-compilation.bash rename to jenkins-scripts/docker/gz_sim-compilation.bash diff --git a/jenkins-scripts/dsl/gz-collections.yaml b/jenkins-scripts/dsl/gz-collections.yaml index 7d08f3ee8..d93dcaefc 100644 --- a/jenkins-scripts/dsl/gz-collections.yaml +++ b/jenkins-scripts/dsl/gz-collections.yaml @@ -120,22 +120,6 @@ collections: ci: exclude: - gz-harmonic -# Remove the exclusion after gz-cmake testing - - gz-tools - - gz-utils - - gz-math - - gz-plugin - - gz-common - - gz-msgs - - gz-rendering - - gz-fuel-tools - - gz-transport - - gz-gui - - gz-sensors - - gz-physics - - gz-sim - - gz-launch - - sdformat linux: reference_distro: - jammy diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index b953fc56f..0507b9aaa 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -471,7 +471,7 @@ gz_software.each { gz_sw -> export ARCH=${arch} - /bin/bash -xe ./scripts/jenkins-scripts/docker/gz_${software_name}-compilation.bash + /bin/bash -xe ./scripts/jenkins-scripts/docker/gz_${gz_sw}-compilation.bash """.stripIndent()) } // end of steps } // end of ci_any_job From 54889fac595d33f4e480a630ad0eb071174c05ae Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 8 Jun 2023 19:17:57 +0200 Subject: [PATCH 382/540] Change fuel-tools compilation script name to use always underscores (#954) Signed-off-by: Jose Luis Rivero --- ...-tools-compilation.bash => gz_fuel_tools-compilation.bash} | 0 jenkins-scripts/dsl/ignition.dsl | 4 ++-- 2 files changed, 2 insertions(+), 2 deletions(-) rename jenkins-scripts/docker/{gz_fuel-tools-compilation.bash => gz_fuel_tools-compilation.bash} (100%) diff --git a/jenkins-scripts/docker/gz_fuel-tools-compilation.bash b/jenkins-scripts/docker/gz_fuel_tools-compilation.bash similarity index 100% rename from jenkins-scripts/docker/gz_fuel-tools-compilation.bash rename to jenkins-scripts/docker/gz_fuel_tools-compilation.bash diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index 0507b9aaa..e511d0275 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -471,7 +471,7 @@ gz_software.each { gz_sw -> export ARCH=${arch} - /bin/bash -xe ./scripts/jenkins-scripts/docker/gz_${gz_sw}-compilation.bash + /bin/bash -xe ./scripts/jenkins-scripts/docker/gz_${gz_sw.replaceAll('-','_')}-compilation.bash """.stripIndent()) } // end of steps } // end of ci_any_job @@ -579,7 +579,7 @@ void generate_ci_job(gz_ci_job, gz_sw, branch, distro, arch, export BUILDING_EXTRA_MAKETEST_PARAMS="${extra_test}" export DISTRO=${distro} export ARCH=${arch} - /bin/bash -xe ./scripts/jenkins-scripts/docker/gz_${software_name}-compilation.bash + /bin/bash -xe ./scripts/jenkins-scripts/docker/gz_${software_name.replaceAll('-','_')}-compilation.bash """.stripIndent()) } } From e08ac93593f549a0eef21a76176cd47246e31ec5 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 8 Jun 2023 22:18:26 +0200 Subject: [PATCH 383/540] Use gazebo-release/ros_ign-release for releasing ros_gz unofficial (#955) * Fix rocker call * Use gazebo-release repository * Include instructions for releasing Signed-off-by: Jose Luis Rivero --- bloom/ros_gz/README.md | 32 +++++++++++++++++------- bloom/ros_gz/bloom_from_special_env.bash | 6 ++--- 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/bloom/ros_gz/README.md b/bloom/ros_gz/README.md index 74299a412..393aa4c2d 100644 --- a/bloom/ros_gz/README.md +++ b/bloom/ros_gz/README.md @@ -1,13 +1,13 @@ # Release new version of Gazebo unofficial wrappers 1. Background - * Upstream versions released using this tutorial + * Upstream versions released using this tutorial 2. Initial setup - * Create the alternative -release repository - * Create a custom track in tracks.yaml + * Create the alternative -release repository + * Create a custom track in tracks.yaml 3. Run a new release - * Prerequisites - * Bloom a new release + * Prerequisites + * Bloom a new release ## 1. Background @@ -39,8 +39,6 @@ release number will start on 1000. To release a modified version of `ros_gz` which supports a different major version of gazebo, before running bloom some actions need to be taken: -## 2. Initial setup - ### 2.1 Create the alternative -release repository For a new official wrappers the notation used below correspond to `ros_ign-release`: @@ -49,7 +47,8 @@ For a new official wrappers the notation used below correspond to `ros_ign-relea https://github.com/ros2-gbp/ros_ign-release 1. Clone the new repo, go to the directory and run rename-gazebo-ros-pkgs.bash - - Usage: *$ rename-ros_gz-pkgs.bash * + - Usage: `$ rename-ros_gz-pkgs.bash ` + - Example: `$ rename-ros_gz-pkgs.bash garden humble` ### 2.2 Create a custom track in tracks.yml @@ -63,7 +62,7 @@ New versioning requires bumping to large numbers. Set: release_inc: '1000' ``` -All non ubuntu generators can be removed. +Debian, rhel and fedora generators can be removed. ## 3. Run a new release @@ -93,3 +92,18 @@ The script will create the docker enviroment with the rosdep modifications neede and invoke rocker with `--home` and `--user` flags to pass the credentials and customatizations needed for the bloom call. It will run the `bloom-release` command with the arguments required for the ros_gz wrappers. + +### 3.2 Launching jobs in the osrf buildfarm + +The previous step generates the metadata needed to build the debians but there is +a final step to call the jobs inside the buildfarm that will create the debians. + +To do so, for simulating the calls to be done to the buildfarm: + +``` +./ros_gz-release.py.bash --dry-run -r +# Example for 0.244.11-1001 version for ROS Humble in ros_ign-release +./ros_gz-release.py.bash 0.244.11 https://github.com/gazebo-release/ros_ign-release humble foo --dry-run -r 1001 +``` + +For releasing directly, just remove the `--dry-run` argument. diff --git a/bloom/ros_gz/bloom_from_special_env.bash b/bloom/ros_gz/bloom_from_special_env.bash index f8a20b000..e6f17d876 100755 --- a/bloom/ros_gz/bloom_from_special_env.bash +++ b/bloom/ros_gz/bloom_from_special_env.bash @@ -1,6 +1,6 @@ #!/bin/bash -if ! command rocker &> /dev/null +if ! command rocker --version &> /dev/null then echo "Please install the rocker app https://github.com/osrf/rocker" exit 1 @@ -11,8 +11,8 @@ BLOOM_CMD="/usr/bin/bloom-release --no-pull-request \ --rosdistro humble \ --track humble_gzgarden \ --override-release-repository-url \ - https://github.com/j-rivero/ros_ign-release ros_gz" -# + https://github.com/gazebo-release/ros_ign-release ros_gz" + TAG_NAME=${TAG_NAME:-ros_gz_fortress_garden_release} echo " * Build the docker release environment" From ac61d75b82cdb2fc44dfeb553b459e83977c1f9b Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 9 Jun 2023 17:54:46 +0200 Subject: [PATCH 384/540] Add gzgarden bridge periodic install job for CI (#956) Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/ros_gz_bridge.dsl | 92 +++++++++++++++++---------- 1 file changed, 59 insertions(+), 33 deletions(-) diff --git a/jenkins-scripts/dsl/ros_gz_bridge.dsl b/jenkins-scripts/dsl/ros_gz_bridge.dsl index 1dc1b01fb..245c5c671 100644 --- a/jenkins-scripts/dsl/ros_gz_bridge.dsl +++ b/jenkins-scripts/dsl/ros_gz_bridge.dsl @@ -137,46 +137,72 @@ bridge_packages.each { pkg -> } } -def install_test_job = -job("ros_gz_bridge-install-pkg_unofficial-any-manual") -OSRFLinuxInstall.create(install_test_job) -install_test_job.with +void generate_install_test_job(Job install_test_job) +{ + OSRFLinuxInstall.create(install_test_job) + install_test_job.with + { + parameters { + stringParam("LINUX_DISTRO", 'ubuntu', "Linux distribution to build packages for") + stringParam("OSRF_REPOS_TO_USE", 'stable', "Repositories to add to the testing install") + labelParam('JENKINS_NODE_TAG') { + description('Jenkins node or group to run the build on') + defaultValue('gpu-reliable') + } + } + + // Designed to be run manually. No triggers. + label Globals.nontest_label("gpu-reliable") + + steps { + systemGroovyCommand("""\ + build.setDescription( + '' + build.buildVariableResolver.resolve('LINUX_DISTRO') + '/' + + build.buildVariableResolver.resolve('DISTRO') + '::' + + build.buildVariableResolver.resolve('ROS_DISTRO') + '::' + + build.buildVariableResolver.resolve('GZ_VERSION') + '' + + '
' + + 'RTOOLS_BRANCH: ' + build.buildVariableResolver.resolve('RTOOLS_BRANCH')); + """.stripIndent()) + + shell("""\ + #!/bin/bash -xe + + export INSTALL_JOB_PKG=ros-\${ROS_DISTRO}-ros-gz\${GZ_VERSION} + export INSTALL_JOB_REPOS=\${OSRF_REPOS_TO_USE} + export USE_ROS_REPO=true + export ROS2=true + /bin/bash -x ./scripts/jenkins-scripts/docker/ros_gz-install-test-job.bash + """.stripIndent()) + } + } +} + +def manual_install_test_job = job("ros_gz_bridge-install-pkg_unofficial-any-manual") +generate_install_test_job(manual_install_test_job) + +manual_install_test_job.with { parameters { - stringParam("LINUX_DISTRO", 'ubuntu', "Linux distribution to build packages for") stringParam("DISTRO", '', "Linux release inside LINUX_DISTRO to build packages for") stringParam("ARCH", '', "Architecture to build packages for") + stringParam("ROS_DISTRO", '', "ROS distribution") + stringParam("GZ_VERSION", '', "Gazebo version") + } +} + +def periodic_install_test_job = job("ros_gzgarden_bridge-install-pkg_humble-ci-jammy-amd64") +generate_install_test_job(periodic_install_test_job) +periodic_install_test_job.with +{ + parameters { + stringParam("DISTRO", 'jammy', "Linux release inside LINUX_DISTRO to build packages for") + stringParam("ARCH", 'amd64', "Architecture to build packages for") stringParam("ROS_DISTRO", 'humble', "ROS distribution") stringParam("GZ_VERSION", 'garden', "Gazebo version") - stringParam("OSRF_REPOS_TO_USE", 'stable prerelease', "Repositories to add to the testing install") - labelParam('JENKINS_NODE_TAG') { - description('Jenkins node or group to run the build on') - defaultValue('gpu-reliable') - } } - // Designed to be run manually. No triggers. - label Globals.nontest_label("gpu-reliable") - - steps { - systemGroovyCommand("""\ - build.setDescription( - '' + build.buildVariableResolver.resolve('LINUX_DISTRO') + '/' + - build.buildVariableResolver.resolve('DISTRO') + '::' + - build.buildVariableResolver.resolve('ROS_DISTRO') + '::' + - build.buildVariableResolver.resolve('GZ_VERSION') + '' + - '
' + - 'RTOOLS_BRANCH: ' + build.buildVariableResolver.resolve('RTOOLS_BRANCH')); - """.stripIndent()) - - shell("""\ - #!/bin/bash -xe - - export INSTALL_JOB_PKG=ros-\${ROS_DISTRO}-ros-gz\${GZ_VERSION} - export INSTALL_JOB_REPOS=\${OSRF_REPOS_TO_USE} - export USE_ROS_REPO=true - export ROS2=true - /bin/bash -x ./scripts/jenkins-scripts/docker/ros_gz-install-test-job.bash - """.stripIndent()) + triggers { + scm('@daily') } } From 29e579074d05bfb909a817f4292231913e9a8c28 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Mon, 12 Jun 2023 15:58:26 +0200 Subject: [PATCH 385/540] Fix cron syntax in Jenkins: use gnu_mcron compatible expression (#957) Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/_configs_/Globals.groovy | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/jenkins-scripts/dsl/_configs_/Globals.groovy b/jenkins-scripts/dsl/_configs_/Globals.groovy index 81279ed9b..3e2283ff7 100644 --- a/jenkins-scripts/dsl/_configs_/Globals.groovy +++ b/jenkins-scripts/dsl/_configs_/Globals.groovy @@ -12,11 +12,13 @@ class Globals static CRON_EVERY_THREE_DAYS = 'H H * * H/3' static CRON_ON_WEEKEND = 'H H * * 6-7' - static CRON_START_NIGHTLY = '0 9 * * *' // Run nightly scheduler during the nightly creation to be sure // that any possible node killed is replaced. Starting -15min // before CRON_NIGHTLY_NODES and evert 20min for 3 hours - static CRON_NIGHTLY_NODES = '45/20 8-11 * * *' + static CRON_NIGHTLY_NODES = '*/20 9-11 * * *' + // Start the nightly generation 10 minutes after the nigthly node + // initial generation + static CRON_START_NIGHTLY = '10 9 * * *' // Only one -E regex can be passed, so make a regex that matches both // _ign_TEST and _gz_TEST From fd72e63af63c240cdac175394f814e495d9ed726 Mon Sep 17 00:00:00 2001 From: Crola1702 Date: Fri, 16 Jun 2023 09:10:26 -0500 Subject: [PATCH 386/540] Fix nvml error discovery in log Signed-off-by: Crola1702 --- jenkins-scripts/dsl/_configs_/OSRFUNIXBase.groovy | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/jenkins-scripts/dsl/_configs_/OSRFUNIXBase.groovy b/jenkins-scripts/dsl/_configs_/OSRFUNIXBase.groovy index f7462aa49..067325b4f 100644 --- a/jenkins-scripts/dsl/_configs_/OSRFUNIXBase.groovy +++ b/jenkins-scripts/dsl/_configs_/OSRFUNIXBase.groovy @@ -53,11 +53,12 @@ class OSRFUNIXBase extends OSRFBase def node = build.getBuiltOn() def old_labels = node.getLabelString() - println("# BEGIN SECTION: NVIDIA MISMATCH RECOVERY") + println("Checking if nvidia mismatch error is present in log") if (!(build.getLog(1000) =~ "nvml error: driver/library version mismatch")) { println(" NVIDIA driver/library version mismatch not detected in the log - Not performing any recovery automatic recovery step") return 1; } else { + println("# BEGIN SECTION: NVIDIA MISMATCH RECOVERY") try { println(" PROBLEM: NVIDIA driver/library version mismatch was detected in the log. Try to automatically resolve it:") println("Removing labels and adding 'recovery-process' label to node") @@ -85,7 +86,7 @@ class OSRFUNIXBase extends OSRFBase // Add the new regex to naginator tag // There is no need to specify checkRegexp and maxSchedule because they are the default values HelperRetryFailures.create(job, [ - regexpForRerun: "nvml error driver/library version mismatch", + regexpForRerun: "nvml error: driver/library version mismatch", delay: 70 ]) } From 06ff9904fd6b33c40668953074cda4257a91b563 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 14 Jun 2023 10:34:24 +0200 Subject: [PATCH 387/540] Parametrize the tooling to support different ROS_DISTRO values Signed-off-by: Jose Luis Rivero --- bloom/release-bloom.py | 1 + bloom/ros_gz/Dockerfile | 4 +++- bloom/ros_gz/bloom_from_special_env.bash | 19 ++++++++++++++----- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/bloom/release-bloom.py b/bloom/release-bloom.py index 9785b0d58..dba69fbe6 100755 --- a/bloom/release-bloom.py +++ b/bloom/release-bloom.py @@ -15,6 +15,7 @@ UBUNTU_DISTROS_IN_ROS = {'noetic': ['focal']} UBUNTU_DISTROS_IN_ROS2 = {'foxy': ['focal'], 'humble': ['jammy'], + 'iron': ['jammy'], 'rolling': ['jammy']} DRY_RUN = False diff --git a/bloom/ros_gz/Dockerfile b/bloom/ros_gz/Dockerfile index 3257be2f0..295aaca8f 100644 --- a/bloom/ros_gz/Dockerfile +++ b/bloom/ros_gz/Dockerfile @@ -1,4 +1,6 @@ -FROM ros:humble-ros-base-jammy +ARG ROS_DISTRO=humble + +FROM ros:$ROS_DISTRO-ros-base ENV LANG C ENV LC_ALL C diff --git a/bloom/ros_gz/bloom_from_special_env.bash b/bloom/ros_gz/bloom_from_special_env.bash index e6f17d876..d4632c218 100755 --- a/bloom/ros_gz/bloom_from_special_env.bash +++ b/bloom/ros_gz/bloom_from_special_env.bash @@ -1,5 +1,14 @@ #!/bin/bash +if [[ $# -lt 1 ]]; then + echo "Usage: $0 " + exit 1 +fi + +ROS_DISTRO=${1} +# Might be used with testing proposes to change gbp-release repo +GBP_ORG=${GBP_ORG:-gazebo-release} + if ! command rocker --version &> /dev/null then echo "Please install the rocker app https://github.com/osrf/rocker" @@ -8,15 +17,15 @@ fi # TODO: update URL for the release repo BLOOM_CMD="/usr/bin/bloom-release --no-pull-request \ - --rosdistro humble \ - --track humble_gzgarden \ + --rosdistro ${ROS_DISTRO}\ + --track ${ROS_DISTRO}_gzgarden \ --override-release-repository-url \ - https://github.com/gazebo-release/ros_ign-release ros_gz" + https://github.com/${GBP_ORG}/ros_ign-release ros_gz" -TAG_NAME=${TAG_NAME:-ros_gz_fortress_garden_release} +TAG_NAME=${TAG_NAME:-ros_gz-${ROS_DISTRO}-fortress-garden-release} echo " * Build the docker release environment" -docker build . -t "${TAG_NAME}" +docker build . -t "${TAG_NAME}" --build-arg ROS_DISTRO="${ROS_DISTRO}" echo " * Using rocker to enter in the release environment" rocker --home --user "${TAG_NAME}" "${BLOOM_CMD}" echo " * Exit the docker release environment" From 4af08089dca91e5ee7fe330f6c307fae9f62e27e Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 14 Jun 2023 17:06:26 +0200 Subject: [PATCH 388/540] Add a double check for correct metadata Signed-off-by: Jose Luis Rivero --- bloom/ros_gz/Dockerfile | 3 +++ bloom/ros_gz/_check_metadata.bash | 29 ++++++++++++++++++++++++ bloom/ros_gz/bloom_from_special_env.bash | 5 +++- 3 files changed, 36 insertions(+), 1 deletion(-) create mode 100755 bloom/ros_gz/_check_metadata.bash diff --git a/bloom/ros_gz/Dockerfile b/bloom/ros_gz/Dockerfile index 295aaca8f..2d98e3a60 100644 --- a/bloom/ros_gz/Dockerfile +++ b/bloom/ros_gz/Dockerfile @@ -7,6 +7,8 @@ ENV LC_ALL C ARG DEBIAN_FRONTEND=noninteractive RUN apt-get update && apt-get install -y \ + coreutils \ + dpkg-dev \ python3-bloom \ wget \ && rm -rf /var/lib/apt/lists/* @@ -24,5 +26,6 @@ ENV GZ_VERSION=garden ENV IGNITION_VERSION=garden WORKDIR /tmp +COPY _check_metadata.bash . COPY _rosdep_wrapper.bash . ENTRYPOINT ["/tmp/_rosdep_wrapper.bash"] diff --git a/bloom/ros_gz/_check_metadata.bash b/bloom/ros_gz/_check_metadata.bash new file mode 100755 index 000000000..f47d6a144 --- /dev/null +++ b/bloom/ros_gz/_check_metadata.bash @@ -0,0 +1,29 @@ +#!/bin/bash + +set -e + + +if [[ $# -lt 2 ]]; then + echo "Usage: $0 " + exit 1 +fi + +RELEASE_REPO_URL=${1} +ROS_DISTRO=${2} + +TMP_DIR=$(mktemp -d) +git clone "${RELEASE_REPO_URL}" "${TMP_DIR}" +pushd "${TMP_DIR}" 2> /dev/null || exit +cd "${TMP_DIR}" || exit +DEBIAN_LATEST_TAG=$(git for-each-ref --sort=creatordate --format '%(refname)' refs/tags | \ + grep "debian/ros-${ROS_DISTRO}" | \ + tail -1 | sed 's:refs/tags/::') +git checkout "${DEBIAN_LATEST_TAG}" +dpkg-parsechangelog -SSource +if ! dpkg-parsechangelog -SSource | grep -q 'gzgarden' > /dev/null; then + echo "Latest changelog entry does not have gzgarden name" + echo "in the repo tag ${DEBIAN_LATEST_TAG} for repo ${RELEASE_REPO_URL}" + echo "Maybe the rename in the bloom templates was not done." + exit 1 +fi +popd 2>/dev/null || exit diff --git a/bloom/ros_gz/bloom_from_special_env.bash b/bloom/ros_gz/bloom_from_special_env.bash index d4632c218..d60281a9d 100755 --- a/bloom/ros_gz/bloom_from_special_env.bash +++ b/bloom/ros_gz/bloom_from_special_env.bash @@ -8,6 +8,7 @@ fi ROS_DISTRO=${1} # Might be used with testing proposes to change gbp-release repo GBP_ORG=${GBP_ORG:-gazebo-release} +RELEASE_REPO_URL=https://github.com/${GBP_ORG}/ros_ign-release if ! command rocker --version &> /dev/null then @@ -20,7 +21,7 @@ BLOOM_CMD="/usr/bin/bloom-release --no-pull-request \ --rosdistro ${ROS_DISTRO}\ --track ${ROS_DISTRO}_gzgarden \ --override-release-repository-url \ - https://github.com/${GBP_ORG}/ros_ign-release ros_gz" + ${RELEASE_REPO_URL} ros_gz" TAG_NAME=${TAG_NAME:-ros_gz-${ROS_DISTRO}-fortress-garden-release} @@ -31,3 +32,5 @@ rocker --home --user "${TAG_NAME}" "${BLOOM_CMD}" echo " * Exit the docker release environment" echo " * Restoring the rosdep cache in the user" rosdep update +echo " * Running safety check in generated metadata" +docker run "${TAG_NAME}" /tmp/_check_metadata.bash "${RELEASE_REPO_URL}" "${ROS_DISTRO}" From 8238d40eaaa3b557b27fad34b4c704b489a2496d Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 14 Jun 2023 19:14:46 +0200 Subject: [PATCH 389/540] Add a check for metadata in ros_gz-release.py.bash Signed-off-by: Jose Luis Rivero --- bloom/ros_gz/_check_metadata.bash | 24 ++++++++++++++---------- bloom/ros_gz/ros_gz-release.py.bash | 9 ++++++++- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/bloom/ros_gz/_check_metadata.bash b/bloom/ros_gz/_check_metadata.bash index f47d6a144..e4bc4eb09 100755 --- a/bloom/ros_gz/_check_metadata.bash +++ b/bloom/ros_gz/_check_metadata.bash @@ -11,19 +11,23 @@ fi RELEASE_REPO_URL=${1} ROS_DISTRO=${2} +echo " * Checking that ${RELEASE_REPO_URL} has a correct gzgarden metadata for ROS ${ROS_DISTRO}" + TMP_DIR=$(mktemp -d) -git clone "${RELEASE_REPO_URL}" "${TMP_DIR}" -pushd "${TMP_DIR}" 2> /dev/null || exit -cd "${TMP_DIR}" || exit +git clone -q "${RELEASE_REPO_URL}" "${TMP_DIR}" +pushd "${TMP_DIR}" 2> /dev/null > /dev/null || exit DEBIAN_LATEST_TAG=$(git for-each-ref --sort=creatordate --format '%(refname)' refs/tags | \ grep "debian/ros-${ROS_DISTRO}" | \ tail -1 | sed 's:refs/tags/::') -git checkout "${DEBIAN_LATEST_TAG}" -dpkg-parsechangelog -SSource -if ! dpkg-parsechangelog -SSource | grep -q 'gzgarden' > /dev/null; then - echo "Latest changelog entry does not have gzgarden name" - echo "in the repo tag ${DEBIAN_LATEST_TAG} for repo ${RELEASE_REPO_URL}" - echo "Maybe the rename in the bloom templates was not done." +git checkout -q "${DEBIAN_LATEST_TAG}" +if ! dpkg-parsechangelog -SSource 2> /dev/null | grep -q 'gzgarden'; then + echo " !! Latest changelog entry does not have gzgarden name" + echo " !! in the repo tag '${DEBIAN_LATEST_TAG}' for repo '${RELEASE_REPO_URL}'" + echo -n " !! Latest changelog is:" + dpkg-parsechangelog -SSource 2> /dev/null || exit + echo " !! Maybe the rename in the bloom templates was not done." exit 1 fi -popd 2>/dev/null || exit +popd >/dev/null || exit + +echo " + Changelog has correct gzgarden metadata for ${DEBIAN_LATEST_TAG}" diff --git a/bloom/ros_gz/ros_gz-release.py.bash b/bloom/ros_gz/ros_gz-release.py.bash index 18b959c6c..473eaed3d 100755 --- a/bloom/ros_gz/ros_gz-release.py.bash +++ b/bloom/ros_gz/ros_gz-release.py.bash @@ -1,7 +1,12 @@ #!/bin/bash # Launch all the suite of ros_gazebo_pkgs: # Usage: ros_gz-release.py.bash -# + +# Knowing Script dir beware of symlink +[[ -L ${0} ]] && SCRIPT_DIR=$(readlink ${0}) || SCRIPT_DIR=${0} +SCRIPT_DIR="${SCRIPT_DIR%/*}" + +set -e if [[ $# -lt 2 ]]; then echo "$0 'other arguments used in release.py'" @@ -13,6 +18,8 @@ if [[ ${1%-*} != "${1}" ]]; then exit 1 fi +"${SCRIPT_DIR}/_check_metadata.bash" "${2}" "${3}" + for p in ros-gz ros-gz-bridge ros-gz-image ros-gz-interfaces ros-gz-sim ros-gz-sim-demos; do ../release-bloom.py "${p}" $(for i in $@; do echo -n "$i "; done) done From 746fe8f7cd601bdcb2afb75f37cc6836c49f153f Mon Sep 17 00:00:00 2001 From: Crola1702 Date: Tue, 20 Jun 2023 07:56:28 -0500 Subject: [PATCH 390/540] Fix typos and section organization Signed-off-by: Crola1702 --- jenkins-scripts/dsl/_configs_/OSRFUNIXBase.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jenkins-scripts/dsl/_configs_/OSRFUNIXBase.groovy b/jenkins-scripts/dsl/_configs_/OSRFUNIXBase.groovy index 067325b4f..fc7e8adfa 100644 --- a/jenkins-scripts/dsl/_configs_/OSRFUNIXBase.groovy +++ b/jenkins-scripts/dsl/_configs_/OSRFUNIXBase.groovy @@ -55,7 +55,7 @@ class OSRFUNIXBase extends OSRFBase println("Checking if nvidia mismatch error is present in log") if (!(build.getLog(1000) =~ "nvml error: driver/library version mismatch")) { - println(" NVIDIA driver/library version mismatch not detected in the log - Not performing any recovery automatic recovery step") + println(" NVIDIA driver/library version mismatch not detected in the log - Not performing any automatic recovery steps") return 1; } else { println("# BEGIN SECTION: NVIDIA MISMATCH RECOVERY") @@ -69,8 +69,8 @@ class OSRFUNIXBase extends OSRFBase node.setLabelString(old_labels) throw ex } + println("# END SECTION: NVIDIA MISMATCH RECOVERY") } - println("# END SECTION: NVIDIA MISMATCH RECOVERY") '''.stripIndent() ) shell("""sudo shutdown -r +1""") From f19f679f64345ec95ae113b9e2008d23affc850c Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Mon, 19 Jun 2023 19:31:34 +0200 Subject: [PATCH 391/540] Fix the calls to gz-gazebo compilation script Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/ignition.dsl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index e511d0275..8f5bad13a 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -568,6 +568,8 @@ void generate_ci_job(gz_ci_job, gz_sw, branch, distro, arch, { if (gz_sw == 'physics') label Globals.nontest_label("large-memory") + if (gz_sw == 'gazebo') + gz_sw = 'sim' steps { shell("""\ @@ -579,7 +581,7 @@ void generate_ci_job(gz_ci_job, gz_sw, branch, distro, arch, export BUILDING_EXTRA_MAKETEST_PARAMS="${extra_test}" export DISTRO=${distro} export ARCH=${arch} - /bin/bash -xe ./scripts/jenkins-scripts/docker/gz_${software_name.replaceAll('-','_')}-compilation.bash + /bin/bash -xe ./scripts/jenkins-scripts/docker/gz_${gz_sw.replaceAll('-','_')}-compilation.bash """.stripIndent()) } } From 930d33c22894d043664385fc0842fe5d12d29077 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 28 Jun 2023 16:41:34 +0200 Subject: [PATCH 392/540] Change Dockerfile MAINTAINER command (deprecated) in favour of LABEL Signed-off-by: Jose Luis Rivero --- jenkins-scripts/docker/lib/docker_generate_dockerfile.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash b/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash index 02434b758..043b470e6 100644 --- a/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash +++ b/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash @@ -134,7 +134,7 @@ cat > Dockerfile << DELIM_DOCKER # Docker file to run build.sh FROM ${FROM_VALUE} -MAINTAINER Jose Luis Rivero +LABEL maintainer="Jose Luis Rivero " # setup environment ENV LANG C From d924412e672072667bb0af6a8a302116e57b385d Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 28 Jun 2023 16:43:26 +0200 Subject: [PATCH 393/540] Scape $USER in Dockerfile to avoid bash expansion Keep Dockerfile variables in placce. Should help with the local reproducibility. Signed-off-by: Jose Luis Rivero --- jenkins-scripts/docker/lib/docker_generate_dockerfile.bash | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash b/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash index 043b470e6..9b8831017 100644 --- a/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash +++ b/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash @@ -456,8 +456,8 @@ RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers RUN chown -R \$USER:\$USER /home/\$USER # permit access to USER variable inside docker -ENV USER ${USER} -USER $USER +ENV USER \$USER +USER \$USER # Must use sudo where necessary from this point on DELIM_DOCKER_USER From f3f1b50400930814b40fe4937a9ca447e924666b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Crist=C3=B3bal=20Arroyo?= Date: Fri, 30 Jun 2023 11:03:30 -0500 Subject: [PATCH 394/540] Add new job to run outdated gazebo jenkins jobs (#962) * Add nightly-runner job and script * Add nightly-runner to test.dsl --------- Signed-off-by: Crola1702 Co-authored-by: Jose Luis Rivero --- jenkins-scripts/dsl/_configs_/Globals.groovy | 1 + jenkins-scripts/dsl/core.dsl | 17 ++ jenkins-scripts/dsl/test.dsl | 13 + .../tools/outdated-job-runner.groovy | 228 ++++++++++++++++++ 4 files changed, 259 insertions(+) create mode 100644 jenkins-scripts/tools/outdated-job-runner.groovy diff --git a/jenkins-scripts/dsl/_configs_/Globals.groovy b/jenkins-scripts/dsl/_configs_/Globals.groovy index 3e2283ff7..63ac935cd 100644 --- a/jenkins-scripts/dsl/_configs_/Globals.groovy +++ b/jenkins-scripts/dsl/_configs_/Globals.groovy @@ -11,6 +11,7 @@ class Globals static gazebodistro_branch = false static CRON_EVERY_THREE_DAYS = 'H H * * H/3' + static CRON_HOURLY = '* H * * *' static CRON_ON_WEEKEND = 'H H * * 6-7' // Run nightly scheduler during the nightly creation to be sure // that any possible node killed is replaced. Starting -15min diff --git a/jenkins-scripts/dsl/core.dsl b/jenkins-scripts/dsl/core.dsl index 8422a27bf..8c273c291 100644 --- a/jenkins-scripts/dsl/core.dsl +++ b/jenkins-scripts/dsl/core.dsl @@ -126,3 +126,20 @@ nightly_labeler.with } } } + +// ------------------------------------------------------------------- +def outdated_job_runner = job("_outdated_job_runner") +OSRFBase.create(outdated_job_runner) +outdated_job_runner.with +{ + label Globals.nontest_label("master") + + triggers { + cron(Globals.CRON_HOURLY) + } + + steps + { + systemGroovyCommand(readFileFromWorkspace('scripts/jenkins-scripts/tools/outdated-job-runner.groovy')) + } +} diff --git a/jenkins-scripts/dsl/test.dsl b/jenkins-scripts/dsl/test.dsl index 151f2d209..66661adb7 100644 --- a/jenkins-scripts/dsl/test.dsl +++ b/jenkins-scripts/dsl/test.dsl @@ -16,3 +16,16 @@ OSRFLinuxCompilationAnyGitHub.create(ignition_ci_pr_job, false, false, ['main']) + +// ------------------------------------------------------------------- +def outdated_job_runner = job("_test_outdated_job_runner") +OSRFBase.create(outdated_job_runner) +outdated_job_runner.with +{ + label Globals.nontest_label("master") + + steps + { + systemGroovyCommand(readFileFromWorkspace('scripts/jenkins-scripts/tools/outdated-job-runner.groovy')) + } +} diff --git a/jenkins-scripts/tools/outdated-job-runner.groovy b/jenkins-scripts/tools/outdated-job-runner.groovy new file mode 100644 index 000000000..10adc753b --- /dev/null +++ b/jenkins-scripts/tools/outdated-job-runner.groovy @@ -0,0 +1,228 @@ +/* + * This script is used to run a job that checks if there are any + * outdated jobs in the buildfarm. It also runs the jobs + * if there is any node available. + */ + +import hudson.model.*; +import jenkins.model.Jenkins; +import java.util.Date; + +/* + * Schedule a job to run immediately + */ +def runJob(String jobName) { + def job = Jenkins.instance.getItemByFullName(jobName) + job.scheduleBuild(0) +} + +/* + * Find available nodes to run jobs per OS using labels + * win: windows + * osx: macos + * linux: docker + */ +def findAvailableNodes() { + def availableNodes = [osx: [], win: [], docker: []] + + Jenkins.instance.nodes.each { node -> + if (node.computer.online && node.computer.countIdle() > 0) { + if (node.getLabelString().contains("win")) { + availableNodes.win << node.name + } else if (node.getLabelString().contains("osx")) { + availableNodes.osx << node.name + } else if (node.getLabelString().contains("docker")) { + availableNodes.docker << node.name + } + } + } + + return availableNodes +} + +/* + * For each node in each OS, run a job if there are jobs to run + */ +def runJobsInAvailableNodes(LinkedHashMap outdatedJobs) { + def availableNodes = findAvailableNodes() + + availableNodes.each { os -> + os.value.each { node -> + if (outdatedJobs[os.key].size() > 0) { + try { + def jobName = outdatedJobs[os.key].remove(0) + println("Running job ${jobName}") + runJob(jobName) + } catch (Exception e) { + println("Error running job ${jobName}") + println(e) + println("MARK_AS_UNSTABLE") + } + } + } + } +} + +// TODO (Cristobal): This is a hardcoded list of jobs that we're tracking. We should get this list dynamically +// Currently is the result of https://github.com/osrf/buildfarmer/blob/main/common.py#L51 minus debbuilders +def trackedJobsList = [ + 'ign_cmake-gz-3-win', + 'ign_cmake-ign-2-win', + 'ign_common-gz-5-win', + 'ign_common-ign-3-win', + 'ign_common-ign-4-win', + 'ign_fuel-tools-gz-8-win', + 'ign_fuel-tools-ign-4-win', + 'ign_fuel-tools-ign-7-win', + 'ign_gazebo-gz-7-win', + 'ign_gazebo-ign-3-win', + 'ign_gazebo-ign-6-win', + 'ign_gui-gz-7-win', + 'ign_gui-ign-3-win', + 'ign_gui-ign-6-win', + 'ign_launch-gz-6-win', + 'ign_launch-ign-2-win', + 'ign_launch-ign-5-win', + 'ign_math-gz-7-win', + 'ign_math-ign-6-win', + 'ign_msgs-gz-9-win', + 'ign_msgs-ign-5-win', + 'ign_msgs-ign-8-win', + 'ign_physics-gz-6-win', + 'ign_physics-ign-2-win', + 'ign_physics-ign-5-win', + 'ign_plugin-gz-2-win', + 'ign_plugin-ign-1-win', + 'ign_rendering-gz-7-win', + 'ign_rendering-ign-3-win', + 'ign_rendering-ign-6-win', + 'ign_sensors-gz-7-win', + 'ign_sensors-ign-3-win', + 'ign_sensors-ign-6-win', + 'ign_tools-gz-2-win', + 'ign_tools-ign-1-win', + 'ign_transport-gz-12-win', + 'ign_transport-ign-11-win', + 'ign_transport-ign-8-win', + 'ign_utils-gz-2-win', + 'ign_utils-ign-1-win', + 'ignition_cmake-ci-gz-cmake3-focal-amd64', + 'ignition_cmake-ci-gz-cmake3-homebrew-amd64', + 'ignition_cmake-ci-ign-cmake2-focal-amd64', + 'ignition_cmake-ci-ign-cmake2-homebrew-amd64', + 'ignition_common-ci-gz-common5-focal-amd64', + 'ignition_common-ci-gz-common5-homebrew-amd64', + 'ignition_common-ci-ign-common3-focal-amd64', + 'ignition_common-ci-ign-common3-homebrew-amd64', + 'ignition_common-ci-ign-common4-focal-amd64', + 'ignition_common-ci-ign-common4-homebrew-amd64', + 'ignition_fuel-tools-ci-gz-fuel-tools8-focal-amd64', + 'ignition_fuel-tools-ci-gz-fuel-tools8-homebrew-amd64', + 'ignition_fuel-tools-ci-ign-fuel-tools4-focal-amd64', + 'ignition_fuel-tools-ci-ign-fuel-tools4-homebrew-amd64', + 'ignition_fuel-tools-ci-ign-fuel-tools7-focal-amd64', + 'ignition_fuel-tools-ci-ign-fuel-tools7-homebrew-amd64', + 'ignition_gazebo-ci-gz-sim7-focal-amd64', + 'ignition_gazebo-ci-gz-sim7-homebrew-amd64', + 'ignition_gazebo-ci-ign-gazebo3-focal-amd64', + 'ignition_gazebo-ci-ign-gazebo3-homebrew-amd64', + 'ignition_gazebo-ci-ign-gazebo6-focal-amd64', + 'ignition_gazebo-ci-ign-gazebo6-homebrew-amd64', + 'ignition_gui-ci-gz-gui7-focal-amd64', + 'ignition_gui-ci-gz-gui7-homebrew-amd64', + 'ignition_gui-ci-ign-gui3-focal-amd64', + 'ignition_gui-ci-ign-gui3-homebrew-amd64', + 'ignition_gui-ci-ign-gui6-focal-amd64', + 'ignition_gui-ci-ign-gui6-homebrew-amd64', + 'ignition_launch-ci-gz-launch6-focal-amd64', + 'ignition_launch-ci-gz-launch6-homebrew-amd64', + 'ignition_launch-ci-ign-launch2-focal-amd64', + 'ignition_launch-ci-ign-launch2-homebrew-amd64', + 'ignition_launch-ci-ign-launch5-focal-amd64', + 'ignition_launch-ci-ign-launch5-homebrew-amd64', + 'ignition_math-ci-gz-math7-focal-amd64', + 'ignition_math-ci-gz-math7-homebrew-amd64', + 'ignition_math-ci-ign-math6-focal-amd64', + 'ignition_math-ci-ign-math6-homebrew-amd64', + 'ignition_msgs-ci-gz-msgs9-focal-amd64', + 'ignition_msgs-ci-gz-msgs9-homebrew-amd64', + 'ignition_msgs-ci-ign-msgs5-focal-amd64', + 'ignition_msgs-ci-ign-msgs5-homebrew-amd64', + 'ignition_msgs-ci-ign-msgs8-focal-amd64', + 'ignition_msgs-ci-ign-msgs8-homebrew-amd64', + 'ignition_physics-ci-gz-physics6-focal-amd64', + 'ignition_physics-ci-gz-physics6-homebrew-amd64', + 'ignition_physics-ci-ign-physics2-focal-amd64', + 'ignition_physics-ci-ign-physics2-homebrew-amd64', + 'ignition_physics-ci-ign-physics5-focal-amd64', + 'ignition_physics-ci-ign-physics5-homebrew-amd64', + 'ignition_plugin-ci-gz-plugin2-focal-amd64', + 'ignition_plugin-ci-gz-plugin2-homebrew-amd64', + 'ignition_plugin-ci-ign-plugin1-focal-amd64', + 'ignition_plugin-ci-ign-plugin1-homebrew-amd64', + 'ignition_rendering-ci-gz-rendering7-focal-amd64', + 'ignition_rendering-ci-gz-rendering7-homebrew-amd64', + 'ignition_rendering-ci-ign-rendering3-focal-amd64', + 'ignition_rendering-ci-ign-rendering3-homebrew-amd64', + 'ignition_rendering-ci-ign-rendering6-focal-amd64', + 'ignition_rendering-ci-ign-rendering6-homebrew-amd64', + 'ignition_sensors-ci-gz-sensors7-focal-amd64', + 'ignition_sensors-ci-gz-sensors7-homebrew-amd64', + 'ignition_sensors-ci-ign-sensors3-focal-amd64', + 'ignition_sensors-ci-ign-sensors3-homebrew-amd64', + 'ignition_sensors-ci-ign-sensors6-focal-amd64', + 'ignition_sensors-ci-ign-sensors6-homebrew-amd64', + 'ignition_tools-ci-gz-tools2-focal-amd64', + 'ignition_tools-ci-gz-tools2-homebrew-amd64', + 'ignition_tools-ci-ign-tools1-focal-amd64', + 'ignition_tools-ci-ign-tools1-homebrew-amd64', + 'ignition_transport-ci-gz-transport12-focal-amd64', + 'ignition_transport-ci-gz-transport12-homebrew-amd64', + 'ignition_transport-ci-ign-transport11-focal-amd64', + 'ignition_transport-ci-ign-transport11-homebrew-amd64', + 'ignition_transport-ci-ign-transport8-focal-amd64', + 'ignition_transport-ci-ign-transport8-homebrew-amd64', + 'ignition_utils-ci-gz-utils2-focal-amd64', + 'ignition_utils-ci-gz-utils2-homebrew-amd64', + 'ignition_utils-ci-ign-utils1-focal-amd64', + 'ignition_utils-ci-ign-utils1-homebrew-amd64', + 'sdformat-ci-sdformat12-focal-amd64', + 'sdformat-ci-sdformat12-homebrew-amd64', + 'sdformat-ci-sdformat13-focal-amd64', + 'sdformat-ci-sdformat13-homebrew-amd64', + 'sdformat-ci-sdformat9-focal-amd64', + 'sdformat-ci-sdformat9-homebrew-amd64', + 'sdformat-sdf-12-win', + 'sdformat-sdf-13-win', + 'sdformat-sdf-9-win', +]; +def trackedJobs = new LinkedHashSet(trackedJobsList); + +def jenkinsJobs = Hudson.instance + +def jobsToRun = [osx: [], win: [], docker: []] + +long eightDaysAgoMillis = System.currentTimeMillis() - 8 * 24 * 60 * 60 * 1000; // 8 days ago in milis +Date eightDaysAgoDate = new Date(eightDaysAgoMillis); + +jenkinsJobs.getItems(Project).each { project -> + // Filter jobs that have not been updated in the last 8 days + if (!project.disabled && trackedJobs.contains(project.displayName) && project.lastBuild.getTime().before(eightDaysAgoDate)) { + if (project.displayName.contains('homebrew')) { + jobsToRun.osx << project.displayName + } else if (project.displayName.contains('win')) { + jobsToRun.win << project.displayName + } else { + jobsToRun.docker << project.displayName + } + } +} + +println('\n--- Jobs to run: ---') +println('OSX: ' + jobsToRun.osx) +println('WIN: ' + jobsToRun.win) +println('docker: ' + jobsToRun.docker) + +println('\n--- Running jobs: ---') +runJobsInAvailableNodes(jobsToRun) + From 7bee49508d2859d535dccf58e764a2abe9ee7ba9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Crist=C3=B3bal=20Arroyo?= Date: Fri, 30 Jun 2023 13:13:10 -0500 Subject: [PATCH 395/540] Make outdated_job_runner run it every hour instead of every minute (#965) Signed-off-by: Crola1702 --- jenkins-scripts/dsl/_configs_/Globals.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/_configs_/Globals.groovy b/jenkins-scripts/dsl/_configs_/Globals.groovy index 63ac935cd..9ef38635d 100644 --- a/jenkins-scripts/dsl/_configs_/Globals.groovy +++ b/jenkins-scripts/dsl/_configs_/Globals.groovy @@ -11,7 +11,7 @@ class Globals static gazebodistro_branch = false static CRON_EVERY_THREE_DAYS = 'H H * * H/3' - static CRON_HOURLY = '* H * * *' + static CRON_HOURLY = 'H H * * *' static CRON_ON_WEEKEND = 'H H * * 6-7' // Run nightly scheduler during the nightly creation to be sure // that any possible node killed is replaced. Starting -15min From 9f1096f751aacf438d9fb24687dd90ccff097b9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Crist=C3=B3bal=20Arroyo?= Date: Mon, 3 Jul 2023 10:15:00 -0500 Subject: [PATCH 396/540] Fix CRON_HOURLY string (#966) Signed-off-by: Crola1702 --- jenkins-scripts/dsl/_configs_/Globals.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/_configs_/Globals.groovy b/jenkins-scripts/dsl/_configs_/Globals.groovy index 9ef38635d..1386da8b6 100644 --- a/jenkins-scripts/dsl/_configs_/Globals.groovy +++ b/jenkins-scripts/dsl/_configs_/Globals.groovy @@ -11,7 +11,7 @@ class Globals static gazebodistro_branch = false static CRON_EVERY_THREE_DAYS = 'H H * * H/3' - static CRON_HOURLY = 'H H * * *' + static CRON_HOURLY = 'H * * * *' static CRON_ON_WEEKEND = 'H H * * 6-7' // Run nightly scheduler during the nightly creation to be sure // that any possible node killed is replaced. Starting -15min From 74d37c6349a7b09332291e1960e79a08ccf2fdfd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Crist=C3=B3bal=20Arroyo?= Date: Mon, 10 Jul 2023 09:44:35 -0500 Subject: [PATCH 397/540] Add harmonic jobs to outdated runner (#968) Signed-off-by: Crola1702 --- .../tools/outdated-job-runner.groovy | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/jenkins-scripts/tools/outdated-job-runner.groovy b/jenkins-scripts/tools/outdated-job-runner.groovy index 10adc753b..494085677 100644 --- a/jenkins-scripts/tools/outdated-job-runner.groovy +++ b/jenkins-scripts/tools/outdated-job-runner.groovy @@ -71,20 +71,25 @@ def trackedJobsList = [ 'ign_common-gz-5-win', 'ign_common-ign-3-win', 'ign_common-ign-4-win', + 'ign_fuel-tools-ci-win', 'ign_fuel-tools-gz-8-win', 'ign_fuel-tools-ign-4-win', 'ign_fuel-tools-ign-7-win', + 'ign_gazebo-ci-win', 'ign_gazebo-gz-7-win', 'ign_gazebo-ign-3-win', 'ign_gazebo-ign-6-win', + 'ign_gui-ci-win', 'ign_gui-gz-7-win', 'ign_gui-ign-3-win', 'ign_gui-ign-6-win', + 'ign_launch-ci-win', 'ign_launch-gz-6-win', 'ign_launch-ign-2-win', 'ign_launch-ign-5-win', 'ign_math-gz-7-win', 'ign_math-ign-6-win', + 'ign_msgs-ci-win', 'ign_msgs-gz-9-win', 'ign_msgs-ign-5-win', 'ign_msgs-ign-8-win', @@ -93,14 +98,17 @@ def trackedJobsList = [ 'ign_physics-ign-5-win', 'ign_plugin-gz-2-win', 'ign_plugin-ign-1-win', + 'ign_rendering-ci-win', 'ign_rendering-gz-7-win', 'ign_rendering-ign-3-win', 'ign_rendering-ign-6-win', + 'ign_sensors-ci-win', 'ign_sensors-gz-7-win', 'ign_sensors-ign-3-win', 'ign_sensors-ign-6-win', 'ign_tools-gz-2-win', 'ign_tools-ign-1-win', + 'ign_transport-ci-win', 'ign_transport-gz-12-win', 'ign_transport-ign-11-win', 'ign_transport-ign-8-win', @@ -122,24 +130,32 @@ def trackedJobsList = [ 'ignition_fuel-tools-ci-ign-fuel-tools4-homebrew-amd64', 'ignition_fuel-tools-ci-ign-fuel-tools7-focal-amd64', 'ignition_fuel-tools-ci-ign-fuel-tools7-homebrew-amd64', + 'ignition_fuel-tools-ci-main-focal-amd64', + 'ignition_fuel-tools-ci-main-homebrew-amd64', 'ignition_gazebo-ci-gz-sim7-focal-amd64', 'ignition_gazebo-ci-gz-sim7-homebrew-amd64', 'ignition_gazebo-ci-ign-gazebo3-focal-amd64', 'ignition_gazebo-ci-ign-gazebo3-homebrew-amd64', 'ignition_gazebo-ci-ign-gazebo6-focal-amd64', 'ignition_gazebo-ci-ign-gazebo6-homebrew-amd64', + 'ignition_gazebo-ci-main-focal-amd64', + 'ignition_gazebo-ci-main-homebrew-amd64', 'ignition_gui-ci-gz-gui7-focal-amd64', 'ignition_gui-ci-gz-gui7-homebrew-amd64', 'ignition_gui-ci-ign-gui3-focal-amd64', 'ignition_gui-ci-ign-gui3-homebrew-amd64', 'ignition_gui-ci-ign-gui6-focal-amd64', 'ignition_gui-ci-ign-gui6-homebrew-amd64', + 'ignition_gui-ci-main-focal-amd64', + 'ignition_gui-ci-main-homebrew-amd64', 'ignition_launch-ci-gz-launch6-focal-amd64', 'ignition_launch-ci-gz-launch6-homebrew-amd64', 'ignition_launch-ci-ign-launch2-focal-amd64', 'ignition_launch-ci-ign-launch2-homebrew-amd64', 'ignition_launch-ci-ign-launch5-focal-amd64', 'ignition_launch-ci-ign-launch5-homebrew-amd64', + 'ignition_launch-ci-main-focal-amd64', + 'ignition_launch-ci-main-homebrew-amd64', 'ignition_math-ci-gz-math7-focal-amd64', 'ignition_math-ci-gz-math7-homebrew-amd64', 'ignition_math-ci-ign-math6-focal-amd64', @@ -150,6 +166,8 @@ def trackedJobsList = [ 'ignition_msgs-ci-ign-msgs5-homebrew-amd64', 'ignition_msgs-ci-ign-msgs8-focal-amd64', 'ignition_msgs-ci-ign-msgs8-homebrew-amd64', + 'ignition_msgs-ci-main-focal-amd64', + 'ignition_msgs-ci-main-homebrew-amd64', 'ignition_physics-ci-gz-physics6-focal-amd64', 'ignition_physics-ci-gz-physics6-homebrew-amd64', 'ignition_physics-ci-ign-physics2-focal-amd64', @@ -166,12 +184,16 @@ def trackedJobsList = [ 'ignition_rendering-ci-ign-rendering3-homebrew-amd64', 'ignition_rendering-ci-ign-rendering6-focal-amd64', 'ignition_rendering-ci-ign-rendering6-homebrew-amd64', + 'ignition_rendering-ci-main-focal-amd64', + 'ignition_rendering-ci-main-homebrew-amd64', 'ignition_sensors-ci-gz-sensors7-focal-amd64', 'ignition_sensors-ci-gz-sensors7-homebrew-amd64', 'ignition_sensors-ci-ign-sensors3-focal-amd64', 'ignition_sensors-ci-ign-sensors3-homebrew-amd64', 'ignition_sensors-ci-ign-sensors6-focal-amd64', 'ignition_sensors-ci-ign-sensors6-homebrew-amd64', + 'ignition_sensors-ci-main-focal-amd64', + 'ignition_sensors-ci-main-homebrew-amd64', 'ignition_tools-ci-gz-tools2-focal-amd64', 'ignition_tools-ci-gz-tools2-homebrew-amd64', 'ignition_tools-ci-ign-tools1-focal-amd64', @@ -182,6 +204,8 @@ def trackedJobsList = [ 'ignition_transport-ci-ign-transport11-homebrew-amd64', 'ignition_transport-ci-ign-transport8-focal-amd64', 'ignition_transport-ci-ign-transport8-homebrew-amd64', + 'ignition_transport-ci-main-focal-amd64', + 'ignition_transport-ci-main-homebrew-amd64', 'ignition_utils-ci-gz-utils2-focal-amd64', 'ignition_utils-ci-gz-utils2-homebrew-amd64', 'ignition_utils-ci-ign-utils1-focal-amd64', From be6a8d1ac62cb509ea18219a52d84cdce77ce920 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 21 Jul 2023 16:49:50 +0200 Subject: [PATCH 398/540] Change gz-collections.yaml spec to support ci configs (#970) * Change YAML spec to support ci configurations as an independent category inside the index and refer to the configurations by name when used for libs. * Change gazebo_libs code to support new format * Use configuration for existing ignition_collection --------- Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/gazebo_libs.dsl | 146 ++++++++++++++------ jenkins-scripts/dsl/gz-collections.yaml | 85 ++++++++---- jenkins-scripts/dsl/ignition_collection.dsl | 7 +- 3 files changed, 167 insertions(+), 71 deletions(-) diff --git a/jenkins-scripts/dsl/gazebo_libs.dsl b/jenkins-scripts/dsl/gazebo_libs.dsl index 1f65e356b..b0b038eff 100644 --- a/jenkins-scripts/dsl/gazebo_libs.dsl +++ b/jenkins-scripts/dsl/gazebo_libs.dsl @@ -7,7 +7,6 @@ import org.yaml.snakeyaml.Yaml GLOBAL_SHELL_CMD='' // GZ COLLECTIONS -arch = 'amd64' ENABLE_CPPCHECK = true // Jenkins needs the relative path to work and locally the simulation is done @@ -15,14 +14,26 @@ ENABLE_CPPCHECK = true file = readFileFromWorkspace("scripts/jenkins-scripts/dsl/gz-collections.yaml") gz_collections_yaml = new Yaml().load(file) -boolean include_gpu_label_if_needed(job, lib, ci_info) +void generate_label_by_requirements(job, lib_name, requirements) { + + if (requirements.nvidia_gpu.contains(lib_name) && + requirements.large_memory.contains(lib_name)) { + println ("ERROR: more than one label is generated by requirements") + exit(1) + } + + label = requirements.nvidia_gpu.contains(lib_name) ? "gpu-reliable" : null + if (! label) + label = requirements.large_memory.contains(lib_name) ? "large-memory" : null + if (! label) + return + job.with { - if (ci_info.requirements.nvidia_gpu.contains(lib.name)) - { - label Globals.nontest_label("gpu-reliable") + label Globals.nontest_label(label) + if (requirements.nvidia_gpu.contains(lib_name)) { // unstable build if missing valid gpu display publishers { consoleParsing { @@ -35,44 +46,89 @@ boolean include_gpu_label_if_needed(job, lib, ci_info) } } -[ 'harmonic' ].each { collection_name -> - collection = gz_collections_yaml.collections. find { it.name == collection_name } - collection.ci.linux.reference_distro.each { distro -> - collection.libs.findAll { ! collection.ci.exclude.contains(it.name) }.each { lib -> - assert(lib.name) - assert(lib.repo.current_branch) - // 1.2.1 Main PR jobs (-ci-pr_any-) (pulling check every 5 minutes) - // -------------------------------------------------------------- - def gz_job_name_prefix = lib.name.replaceAll('-','_') - def gz_ci_job_name = "${gz_job_name_prefix}-ci-pr_any-${distro}-${arch}" - def gz_ci_any_job = job(gz_ci_job_name) - def enable_testing = (collection.ci.tests_disabled?.contains(lib.name)) ? false : true - OSRFLinuxCompilationAnyGitHub.create(gz_ci_any_job, - "gazebosim/${lib.name}", - enable_testing, - ENABLE_CPPCHECK, - [ lib.repo.current_branch ]) - include_gpu_label_if_needed(gz_ci_any_job, lib, collection.ci) - gz_ci_any_job.with +boolean is_testing_enabled(lib_name, ci_config) +{ + return ! ci_config.tests_disabled?.contains(lib_name) +} + + +/* + * Generate an index that facilitates the operations with the yaml values, + * avoiding to parse them several times. + * + * ci_configs_by_lib index structure: + * lib_name : [ ci_config_name : [ branches ] ] + * + * The index main keys are the lib names (i.e: gz-cmake) and associated them + * another map of CI configuration names supported as keys (i.e: jammy) with the + * list of associated branches for that CI configuration (i.e [gz-cmake3, gz-cmake4]) + * as values. + */ +void generate_ciconfigs_by_lib(config, configs_per_lib_index) +{ + config.collections.each { collection -> + // TODO(jrivero): limit to harmonic for testing proposes + if (collection.name != 'harmonic') + return + + collection.libs.each { lib -> + def libName = lib.name + def branch = lib.repo.current_branch + collection.ci.configs.each { config_name -> + configs_per_lib_index["$libName"]["${config_name}"] = configs_per_lib_index["$libName"]["${config_name}"]?: [] + configs_per_lib_index["$libName"]["${config_name}"].contains(branch) ?: configs_per_lib_index["$libName"]["${config_name}"] << branch + } + } + } +} + +def configs_per_lib_index = [:].withDefault { [:] } +generate_ciconfigs_by_lib(gz_collections_yaml, configs_per_lib_index) + +// Generate PR jobs: 1 per ci configuration on each lib +configs_per_lib_index.each { lib_name, lib_configs -> + lib_configs.each { ci_configs -> + def config_name = ci_configs.getKey() + def ci_config = gz_collections_yaml.ci_configs.find{ it.name == config_name } + def branch_names = ci_configs.getValue() + if (ci_config.exclude.contains(lib_name)) + return + assert(lib_name) + assert(branch_names) + assert(ci_config) + + if (ci_config.exclude.contains(lib_name)) + return + + // 1.2.1 Main PR jobs (-ci-pr_any-) (pulling check every 5 minutes) + // -------------------------------------------------------------- + def distro = ci_config.system.version + def arch = ci_config.system.arch + def gz_job_name_prefix = lib_name.replaceAll('-','_') + def gz_ci_job_name = "${gz_job_name_prefix}-ci-pr_any-${distro}-${arch}" + def gz_ci_any_job = job(gz_ci_job_name) + OSRFLinuxCompilationAnyGitHub.create(gz_ci_any_job, + "gazebosim/${lib_name}", + is_testing_enabled(lib_name, ci_config), + ENABLE_CPPCHECK, + branch_names) + generate_label_by_requirements(gz_ci_any_job, lib_name, ci_config.requirements) + gz_ci_any_job.with + { + steps { - if (collection.ci.requirements.large_memory?.contains(lib.name)) - label Globals.nontest_label("large-memory") - - steps - { - shell("""\ - #!/bin/bash -xe - - export DISTRO=${distro} - - ${GLOBAL_SHELL_CMD} - - export BUILDING_SOFTWARE_DIRECTORY=${lib.name} - export ARCH=${arch} - /bin/bash -xe ./scripts/jenkins-scripts/docker/${gz_job_name_prefix}-compilation.bash - """.stripIndent()) - } // end of steps - } // end of ci_any_job - } //en of lib - } // end of distro -} // end of collection + shell("""\ + #!/bin/bash -xe + + export DISTRO=${distro} + + ${GLOBAL_SHELL_CMD} + + export BUILDING_SOFTWARE_DIRECTORY=${lib_name} + export ARCH=${arch} + /bin/bash -xe ./scripts/jenkins-scripts/docker/${gz_job_name_prefix}-compilation.bash + """.stripIndent()) + } // end of steps + } // end of ci_any_job + } //en of lib_configs +} // end of lib diff --git a/jenkins-scripts/dsl/gz-collections.yaml b/jenkins-scripts/dsl/gz-collections.yaml index d93dcaefc..1c9589536 100644 --- a/jenkins-scripts/dsl/gz-collections.yaml +++ b/jenkins-scripts/dsl/gz-collections.yaml @@ -29,23 +29,20 @@ collections: - name: 'citadel' libs: ci: - linux: - reference_distro: - - bionic + configs: + - bionic packaging: - name: 'fortress' libs: ci: - linux: - reference_distro: - - focal + configs: + - focal packaging: - name: 'garden' libs: ci: - linux: - reference_distro: - - focal + configs: + - focal packaging: - name: 'harmonic' libs: @@ -118,20 +115,8 @@ collections: repo: current_branch: main ci: - exclude: - - gz-harmonic - linux: - reference_distro: - - jammy - requirements: - large_memory: - - gz-physics - nvidia_gpu: - - gz-sim - - gz-gui - - gz-rendering - - gz-sensors - tests_disabled: + configs: + - jammy packaging: linux: package_name: @@ -140,4 +125,56 @@ collections: nightly: distros: - jammy - +ci_configs: + - name: bionic + system: + so: linux + distribution: ubuntu + version: bionic + arch: amd64 + exclude: + - ign-citadel + requirements: + large_memory: + - ign-physics + nvidia_gpu: + - ign-gazebo + - ign-gui + - ign-rendering + - ign-sensors + tests_disabled: + - name: focal + system: + so: linux + distribution: ubuntu + version: focal + arch: amd64 + requirements: + large_memory: + - gz-physics + nvidia_gpu: + - gz-sim + - gz-gui + - gz-rendering + - gz-sensors + exclude: + - gz-fortress + - gz-garden + tests_disabled: + - name: jammy + system: + so: linux + distribution: ubuntu + version: jammy + arch: amd64 + requirements: + large_memory: + - gz-physics + nvidia_gpu: + - gz-sim + - gz-gui + - gz-rendering + - gz-sensors + exclude: + - gz-harmonic + tests_disabled: diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index d512e30a7..04ce54833 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -365,7 +365,6 @@ void generate_install_job(prefix, gz_collection_name, distro, arch) // Testing compilation from source gz_collections_yaml.collections.each { collection -> gz_collection_name = collection.name - distros = collection.ci.linux.reference_distro // COLCON - Windows def gz_win_ci_job = job("ign_${gz_collection_name}-ci-win") @@ -382,7 +381,11 @@ gz_collections_yaml.collections.each { collection -> } Globals.gazebodistro_branch = false - distros.each { distro -> + collection.ci.configs.each { ci_config_name -> + ci_config = gz_collections_yaml.ci_configs.find { it.name == ci_config_name } + distro = ci_config.system.version + arch = ci_config.system.arch + // INSTALL JOBS: // -------------------------------------------------------------- if ((gz_collection_name == "citadel") || (gz_collection_name == "fortress")) { From 1fa30bed5dba014ab0000e59cbb700a7f0310fc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Steven!=20Ragnar=C3=B6k?= Date: Thu, 27 Jul 2023 09:13:46 -0700 Subject: [PATCH 399/540] Make display detection more robust. (#916) * Refactor X11 display detection. * Add a short wait to allow lightdm to start X11 after being restarted. * Use systemctl directly instead of relying on wrapper. --- jenkins-scripts/lib/check_graphic_card.bash | 25 ++++++++++++--------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/jenkins-scripts/lib/check_graphic_card.bash b/jenkins-scripts/lib/check_graphic_card.bash index 4ce65fb7d..1528ab240 100644 --- a/jenkins-scripts/lib/check_graphic_card.bash +++ b/jenkins-scripts/lib/check_graphic_card.bash @@ -3,16 +3,19 @@ GRAPHIC_CARD_PKG="" export_display_variable() { - # Hack to found the current display (if available) two steps: - # Check for /tmp/.X11-unix/ socket and check if the process is running - for i in $(ls /tmp/.X11-unix/ | sed -e 's@^X@:@') + # Check for an active X11 display socket. + for i in $(find /tmp/.X11-unix -type s) do - # grep can fail so let's disable the fail or error during its call - set +e - ps aux | grep bin/X.*$i | grep -v grep - set -e - if [ $? -eq 0 ] ; then - export DISPLAY=$i + # If a process is running with the open socket then + # lsof will exit successfully. + if lsof -Fp $i ; then + # Strip the path and leading X from the X11 socket + # but check that the resulting string is numeric and + # non-empty before exporting. + DISPLAY=:$(basename $i | sed -n -E 's/^X([0-9]+)/:\1/p') + if [ -n "$DISPLAY" ]; then + export DISPLAY + fi fi done } @@ -79,7 +82,9 @@ if $GPU_SUPPORT_NEEDED; then if [[ ${DISPLAY} == "" ]]; then echo "GPU support needed by the script but DISPLAY var is empty" # Try to restart lightdm. It should stop the script in the case of failure - sudo service lightdm restart + sudo systemctl restart lightdm + # Wait for lightdm service to restart X11. + sleep 5 # Second try to get display variable export_display_variable if [[ ${DISPLAY} == "" ]]; then From a2dece0c82ca569b813a2ede3ee486e17ba48dad Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 27 Jul 2023 21:25:39 +0200 Subject: [PATCH 400/540] Need to run lsof with sudo from Jenkins CI (#973) Signed-off-by: Jose Luis Rivero --- jenkins-scripts/lib/check_graphic_card.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/lib/check_graphic_card.bash b/jenkins-scripts/lib/check_graphic_card.bash index 1528ab240..aca156a4e 100644 --- a/jenkins-scripts/lib/check_graphic_card.bash +++ b/jenkins-scripts/lib/check_graphic_card.bash @@ -8,7 +8,7 @@ export_display_variable() do # If a process is running with the open socket then # lsof will exit successfully. - if lsof -Fp $i ; then + if sudo bash -c "lsof -Fp $i" ; then # Strip the path and leading X from the X11 socket # but check that the resulting string is numeric and # non-empty before exporting. From 5c5cc9bb900cf5551795b7611cd2ea5d14ff4a37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Crist=C3=B3bal=20Arroyo?= Date: Fri, 28 Jul 2023 11:03:05 -0500 Subject: [PATCH 401/540] Update outdated job condition from 8 to 4 days (#972) Signed-off-by: Crola1702 --- jenkins-scripts/tools/outdated-job-runner.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/tools/outdated-job-runner.groovy b/jenkins-scripts/tools/outdated-job-runner.groovy index 494085677..bf030d162 100644 --- a/jenkins-scripts/tools/outdated-job-runner.groovy +++ b/jenkins-scripts/tools/outdated-job-runner.groovy @@ -226,7 +226,7 @@ def jenkinsJobs = Hudson.instance def jobsToRun = [osx: [], win: [], docker: []] -long eightDaysAgoMillis = System.currentTimeMillis() - 8 * 24 * 60 * 60 * 1000; // 8 days ago in milis +long eightDaysAgoMillis = System.currentTimeMillis() - 4 * 24 * 60 * 60 * 1000; // 4 days ago in milis Date eightDaysAgoDate = new Date(eightDaysAgoMillis); jenkinsJobs.getItems(Project).each { project -> From 05fbae415abd888affa98a58155032e344b1b2bd Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 28 Jul 2023 18:03:33 +0200 Subject: [PATCH 402/540] Use large-memory nodes for nightlies (#969) * Use large-memory nodes for nightlies Signed-off-by: Jose Luis Rivero --- docs/BUILDFARM_NODES.md | 10 ---------- .../tools/label-assignment-backstop.groovy | 11 ++++------- 2 files changed, 4 insertions(+), 17 deletions(-) diff --git a/docs/BUILDFARM_NODES.md b/docs/BUILDFARM_NODES.md index e13b4d96e..a5daf8305 100644 --- a/docs/BUILDFARM_NODES.md +++ b/docs/BUILDFARM_NODES.md @@ -22,16 +22,6 @@ used (can check this in the Jenkins UI) | win | Node is able to run Windows CI | Windows10 system | | win_testing | Node is ready to test a new vcpkg snapshot | Windows10 system | -### Node labels for nightlies - -For generating nightlies controlling the generation order of every library, build.osrfoundation.org use the approach of [using a single node to process the whole ignition family for a given Ubuntu distribution](https://github.com/gazebo-tooling/release-tools/issues/644). Current assignment of nodes is as follow: - -| Label name | Description | -| -------- | ----------- | -| linux-nightly-bionic | r2d2 | -| linux-nightly-focal | optimus | -| linux-nightly-jammy | drogon | - ## Provision of Node labels ### Agents diff --git a/jenkins-scripts/tools/label-assignment-backstop.groovy b/jenkins-scripts/tools/label-assignment-backstop.groovy index cd80915e3..54c0f307b 100644 --- a/jenkins-scripts/tools/label-assignment-backstop.groovy +++ b/jenkins-scripts/tools/label-assignment-backstop.groovy @@ -11,15 +11,13 @@ import jenkins.model.*; import hudson.model.Label; /* - Each label is in a 2-tuple with a label expression that indicates its "pool" - of potential assignees. - The first field must be a single label (no spaces) but the second may be an - arbitrarily complex label expression like "(docker && linux) && !armhf" + Each label is in a 2-tuple with a label tag that indicates its "pool" + of potential assignees. The fields must be a single label (no spaces). */ def nightly_label_prefix = "linux-nightly" def exactly_one_labels = [ - ["${nightly_label_prefix}-focal", "docker"], - ["${nightly_label_prefix}-jammy", "docker"], + ["${nightly_label_prefix}-focal", "large-memory"], + ["${nightly_label_prefix}-jammy", "large-memory"], ] for (tup in exactly_one_labels) { @@ -52,7 +50,6 @@ for (tup in exactly_one_labels) { node.computer.online && node.getLabelString().contains(pool_label) && !node.getLabelString().contains(nightly_label_prefix) && - !node.getLabelString().contains("gpu-nvidia") && !node.getLabelString().contains("test-instance") } From 22bfbb9ade87a758b8365f29cfe79048d5915a52 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 28 Jul 2023 18:35:44 +0200 Subject: [PATCH 403/540] Generate Harmonic CI jobs on Jammy for Harmonic branches (#971) * Generate Harmonic ci jobs on Jammy for Harmonic branches * Need to be explicit on BUILDING_SOFTWARE_DIRECTORY * Include new jobs in Harmonic dashboard Signed-off-by: Jose Luis Rivero --------- Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/gazebo_libs.dsl | 49 ++++++++++++++++++++- jenkins-scripts/dsl/ignition_collection.dsl | 32 +++++++------- 2 files changed, 64 insertions(+), 17 deletions(-) diff --git a/jenkins-scripts/dsl/gazebo_libs.dsl b/jenkins-scripts/dsl/gazebo_libs.dsl index b0b038eff..bdd41acdd 100644 --- a/jenkins-scripts/dsl/gazebo_libs.dsl +++ b/jenkins-scripts/dsl/gazebo_libs.dsl @@ -82,6 +82,40 @@ void generate_ciconfigs_by_lib(config, configs_per_lib_index) } } +void generate_ci_job(gz_ci_job, lib_name, branch, ci_config, + extra_cmake = '', extra_test = '', extra_cmd = '') +{ + def distro = ci_config.system.version + def arch = ci_config.system.arch + + OSRFLinuxCompilation.create(gz_ci_job, is_testing_enabled(lib_name, ci_config)) + OSRFGitHub.create(gz_ci_job, + "gazebosim/${lib_name}", "${branch}") + generate_label_by_requirements(gz_ci_job, lib_name, ci_config.requirements) + gz_ci_job.with + { + if (lib_name == 'physics') + label Globals.nontest_label("large-memory") + if (lib_name == 'gazebo') + lib_name = 'sim' + + steps { + shell("""\ + #!/bin/bash -xe + + ${GLOBAL_SHELL_CMD} + ${extra_cmd} + export BUILDING_EXTRA_CMAKE_PARAMS="${extra_cmake}" + export BUILDING_EXTRA_MAKETEST_PARAMS="${extra_test}" + export BUILDING_SOFTWARE_DIRECTORY="${lib_name}" + export DISTRO=${distro} + export ARCH=${arch} + /bin/bash -xe ./scripts/jenkins-scripts/docker/${lib_name.replaceAll('-','_')}-compilation.bash + """.stripIndent()) + } + } +} + def configs_per_lib_index = [:].withDefault { [:] } generate_ciconfigs_by_lib(gz_collections_yaml, configs_per_lib_index) @@ -100,7 +134,7 @@ configs_per_lib_index.each { lib_name, lib_configs -> if (ci_config.exclude.contains(lib_name)) return - // 1.2.1 Main PR jobs (-ci-pr_any-) (pulling check every 5 minutes) + // Main PR jobs (-ci-pr_any-) (pulling check every 5 minutes) // -------------------------------------------------------------- def distro = ci_config.system.version def arch = ci_config.system.arch @@ -130,5 +164,18 @@ configs_per_lib_index.each { lib_name, lib_configs -> """.stripIndent()) } // end of steps } // end of ci_any_job + + // CI branch jobs (-ci-$branch-) (pulling check every 5 minutes) + branch_names.each { branch_name -> + def gz_ci_job = job("${gz_job_name_prefix}-ci-${branch_name}-${distro}-${arch}") + generate_ci_job(gz_ci_job, lib_name, branch_name, ci_config) + gz_ci_job.with + { + triggers { + scm('@daily') + } + } + } // end_of_branch + } //en of lib_configs } // end of lib diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index 04ce54833..a33652ab9 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -290,38 +290,38 @@ gz_collection_jobs = 'ign_tools-gz-2-win', 'ign_transport-gz-13-win', 'ign_utils-gz-2-win', - 'ignition_cmake-ci-gz-cmake3-focal-amd64', + 'gz_cmake-ci-gz-cmake3-jammy-amd64', + 'gz_common-ci-gz-common5-jammy-amd64', + 'gz_fuel_tools-ci-main-jammy-amd64', + 'gz_gui-ci-main-jammy-amd64', + 'gz_launch-ci-main-jammy-amd64', + 'gz_math-ci-gz-math7-jammy-amd64', + 'gz_msgs-ci-main-jammy-amd64', + 'gz_physics-ci-gz-physics6-jammy-amd64', + 'gz_plugin-ci-gz-plugin2-jammy-amd64', + 'gz_rendering-ci-main-jammy-amd64', + 'gz_sensors-ci-main-jammy-amd64', + 'gz_sim-ci-main-jammy-amd64', + 'gz_tools-ci-gz-tools2-jammy-amd64', + 'gz_transport-ci-main-jammy-amd64', + 'gz_utils-ci-gz-utils2-jammy-amd64', 'ignition_cmake-ci-gz-cmake3-homebrew-amd64', - 'ignition_common-ci-gz-common5-focal-amd64', 'ignition_common-ci-gz-common5-homebrew-amd64', - 'ignition_fuel-tools-ci-gz-fuel-tools9-focal-amd64', 'ignition_fuel-tools-ci-gz-fuel-tools9-homebrew-amd64', 'ignition_harmonic-ci-main-homebrew-amd64', - 'ignition_gazebo-ci-main-focal-amd64', 'ignition_gazebo-ci-main-homebrew-amd64', - 'ignition_gui-ci-main-focal-amd64', 'ignition_gui-ci-main-homebrew-amd64', - 'ignition_launch-ci-main-focal-amd64', 'ignition_launch-ci-main-homebrew-amd64', - 'ignition_math-ci-gz-math7-focal-amd64', 'ignition_math-ci-gz-math7-homebrew-amd64', - 'ignition_msgs-ci-gz-msgs10-focal-amd64', 'ignition_msgs-ci-gz-msgs10-homebrew-amd64', - 'ignition_physics-ci-gz-physics6-focal-amd64', 'ignition_physics-ci-gz-physics6-homebrew-amd64', - 'ignition_plugin-ci-gz-plugin2-focal-amd64', 'ignition_plugin-ci-gz-plugin2-homebrew-amd64', - 'ignition_rendering-ci-main-focal-amd64', 'ignition_rendering-ci-main-homebrew-amd64', - 'ignition_sensors-ci-main-focal-amd64', 'ignition_sensors-ci-main-homebrew-amd64', - 'ignition_tools-ci-gz-tools2-focal-amd64', 'ignition_tools-ci-gz-tools2-homebrew-amd64', - 'ignition_transport-ci-gz-transport13-focal-amd64', 'ignition_transport-ci-gz-transport13-homebrew-amd64', - 'ignition_utils-ci-gz-utils2-focal-amd64', 'ignition_utils-ci-gz-utils2-homebrew-amd64', - 'sdformat-ci-sdformat13-focal-amd64', + 'sdformat-ci-sdf13-jammy-amd64.xml', 'sdformat-ci-sdformat13-homebrew-amd64', 'sdformat-sdf-13-win' ], From 68075165eaa9ceca35e6a60b742b61ab0cb23070 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 28 Jul 2023 21:19:58 +0200 Subject: [PATCH 404/540] Enable nighty mode when using nightly on VERSION instead of UPLOAD_TO_REPO (#975) Signed-off-by: Jose Luis Rivero --- jenkins-scripts/docker/lib/debbuild-base.bash | 2 +- jenkins-scripts/dsl/_configs_/OSRFLinuxBuildPkg.groovy | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/jenkins-scripts/docker/lib/debbuild-base.bash b/jenkins-scripts/docker/lib/debbuild-base.bash index 9ba9cc189..70b3e4591 100644 --- a/jenkins-scripts/docker/lib/debbuild-base.bash +++ b/jenkins-scripts/docker/lib/debbuild-base.bash @@ -1,7 +1,7 @@ #!/bin/bash -x NIGHTLY_MODE=${NIGHTLY_MODE:-false} -if [ "${UPLOAD_TO_REPO}" = "nightly" ]; then +if [ "${VERSION}" = "nightly" ]; then OSRF_REPOS_TO_USE="${OSRF_REPOS_TO_USE:-stable nightly}" NIGHTLY_MODE=true # SOURCE_TARBALL_URI is reused in nightly mode to indicate the branch diff --git a/jenkins-scripts/dsl/_configs_/OSRFLinuxBuildPkg.groovy b/jenkins-scripts/dsl/_configs_/OSRFLinuxBuildPkg.groovy index 9f61a68fe..41d8aac15 100644 --- a/jenkins-scripts/dsl/_configs_/OSRFLinuxBuildPkg.groovy +++ b/jenkins-scripts/dsl/_configs_/OSRFLinuxBuildPkg.groovy @@ -42,7 +42,7 @@ class OSRFLinuxBuildPkg "Package name to be built") stringParam("VERSION", default_params.find{ it.key == "VERSION"}?.value, - "Packages version to be built") + "Packages version to be built or nightly (enable nightly build mode)") stringParam("RELEASE_VERSION", default_params.find{ it.key == "RELEASE_VERSION"}?.value, "Packages release version") From 45cf2431c3e2bbf8e14b0a5c5bf0ba4b2066b524 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Crist=C3=B3bal=20Arroyo?= Date: Mon, 31 Jul 2023 12:01:13 -0500 Subject: [PATCH 405/540] Revert: Robust display detection (#916) (#978) Signed-off-by: Crola1702 --- jenkins-scripts/lib/check_graphic_card.bash | 25 +++++++++------------ 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/jenkins-scripts/lib/check_graphic_card.bash b/jenkins-scripts/lib/check_graphic_card.bash index aca156a4e..4ce65fb7d 100644 --- a/jenkins-scripts/lib/check_graphic_card.bash +++ b/jenkins-scripts/lib/check_graphic_card.bash @@ -3,19 +3,16 @@ GRAPHIC_CARD_PKG="" export_display_variable() { - # Check for an active X11 display socket. - for i in $(find /tmp/.X11-unix -type s) + # Hack to found the current display (if available) two steps: + # Check for /tmp/.X11-unix/ socket and check if the process is running + for i in $(ls /tmp/.X11-unix/ | sed -e 's@^X@:@') do - # If a process is running with the open socket then - # lsof will exit successfully. - if sudo bash -c "lsof -Fp $i" ; then - # Strip the path and leading X from the X11 socket - # but check that the resulting string is numeric and - # non-empty before exporting. - DISPLAY=:$(basename $i | sed -n -E 's/^X([0-9]+)/:\1/p') - if [ -n "$DISPLAY" ]; then - export DISPLAY - fi + # grep can fail so let's disable the fail or error during its call + set +e + ps aux | grep bin/X.*$i | grep -v grep + set -e + if [ $? -eq 0 ] ; then + export DISPLAY=$i fi done } @@ -82,9 +79,7 @@ if $GPU_SUPPORT_NEEDED; then if [[ ${DISPLAY} == "" ]]; then echo "GPU support needed by the script but DISPLAY var is empty" # Try to restart lightdm. It should stop the script in the case of failure - sudo systemctl restart lightdm - # Wait for lightdm service to restart X11. - sleep 5 + sudo service lightdm restart # Second try to get display variable export_display_variable if [[ ${DISPLAY} == "" ]]; then From 554864a7ab002ee99824c38599760e2cee12b615 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 1 Aug 2023 16:22:37 +0200 Subject: [PATCH 406/540] Resurrect "Robust display detection" with a variable fix (#980) * Resurrect changes in (#916) (#978) * Remove extra : in DISPLAY variable Signed-off-by: Jose Luis Rivero --- jenkins-scripts/lib/check_graphic_card.bash | 25 ++++++++++++--------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/jenkins-scripts/lib/check_graphic_card.bash b/jenkins-scripts/lib/check_graphic_card.bash index 4ce65fb7d..c6ca86f64 100644 --- a/jenkins-scripts/lib/check_graphic_card.bash +++ b/jenkins-scripts/lib/check_graphic_card.bash @@ -3,16 +3,19 @@ GRAPHIC_CARD_PKG="" export_display_variable() { - # Hack to found the current display (if available) two steps: - # Check for /tmp/.X11-unix/ socket and check if the process is running - for i in $(ls /tmp/.X11-unix/ | sed -e 's@^X@:@') + # Check for an active X11 display socket. + for i in $(find /tmp/.X11-unix -type s) do - # grep can fail so let's disable the fail or error during its call - set +e - ps aux | grep bin/X.*$i | grep -v grep - set -e - if [ $? -eq 0 ] ; then - export DISPLAY=$i + # If a process is running with the open socket then + # lsof will exit successfully. + if sudo bash -c "lsof -Fp $i" ; then + # Strip the path and leading X from the X11 socket + # but check that the resulting string is numeric and + # non-empty before exporting. + DISPLAY=$(basename $i | sed -n -E 's/^X([0-9]+)/:\1/p') + if [ -n "$DISPLAY" ]; then + export DISPLAY + fi fi done } @@ -79,7 +82,9 @@ if $GPU_SUPPORT_NEEDED; then if [[ ${DISPLAY} == "" ]]; then echo "GPU support needed by the script but DISPLAY var is empty" # Try to restart lightdm. It should stop the script in the case of failure - sudo service lightdm restart + sudo systemctl restart lightdm + # Wait for lightdm service to restart X11. + sleep 5 # Second try to get display variable export_display_variable if [[ ${DISPLAY} == "" ]]; then From 19e4175ffcc906dae9ea8ffc9ab2c23e02997a4b Mon Sep 17 00:00:00 2001 From: "Addisu Z. Taddese" Date: Wed, 2 Aug 2023 19:41:11 -0500 Subject: [PATCH 407/540] Bumps in Harmonic: sdformat14 (#977) Signed-off-by: Addisu Z. Taddese --- jenkins-scripts/dsl/gz-collections.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jenkins-scripts/dsl/gz-collections.yaml b/jenkins-scripts/dsl/gz-collections.yaml index 1c9589536..b7363e9c6 100644 --- a/jenkins-scripts/dsl/gz-collections.yaml +++ b/jenkins-scripts/dsl/gz-collections.yaml @@ -79,9 +79,9 @@ collections: repo: current_branch: main - name: sdformat - major_version: 13 + major_version: 14 repo: - current_branch: sdf13 + current_branch: main - name: gz-fuel-tools major_version: 9 repo: From 30bd297a9c58d7200a6148790827f8e6af137b4d Mon Sep 17 00:00:00 2001 From: "Addisu Z. Taddese" Date: Thu, 3 Aug 2023 12:35:13 -0500 Subject: [PATCH 408/540] Add sdformat14 nightlies (#982) * Add sdformat14 nightlies Signed-off-by: Addisu Z. Taddese * Add sdformat-main to Harmonic collection Signed-off-by: Addisu Z. Taddese --------- Signed-off-by: Addisu Z. Taddese --- jenkins-scripts/dsl/ignition.dsl | 1 + jenkins-scripts/dsl/ignition_collection.dsl | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index 8f5bad13a..128170d1c 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -89,6 +89,7 @@ gz_extra_debbuild = [ 'gui8', 'msgs10', 'rendering8', 'sensors8', + 'sdformat14', 'sim8', 'transport13', 'utils1' // see comment https://github.com/gazebo-tooling/release-tools/pull/431#issuecomment-815099918 diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index a33652ab9..a0b2e26da 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -321,9 +321,9 @@ gz_collection_jobs = 'ignition_tools-ci-gz-tools2-homebrew-amd64', 'ignition_transport-ci-gz-transport13-homebrew-amd64', 'ignition_utils-ci-gz-utils2-homebrew-amd64', - 'sdformat-ci-sdf13-jammy-amd64.xml', - 'sdformat-ci-sdformat13-homebrew-amd64', - 'sdformat-sdf-13-win' + 'sdformat-ci-main-jammy-amd64', + 'sdformat-ci-main-homebrew-amd64', + 'sdformat-sdf-main-win' ], ] From 1d3419c864807ed9b2f647d8b2d88714f1278a09 Mon Sep 17 00:00:00 2001 From: "Addisu Z. Taddese" Date: Thu, 3 Aug 2023 14:49:06 -0500 Subject: [PATCH 409/540] Fix sdformat14 dsl (#983) --- jenkins-scripts/dsl/ignition.dsl | 1 - jenkins-scripts/dsl/sdformat.dsl | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index 128170d1c..8f5bad13a 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -89,7 +89,6 @@ gz_extra_debbuild = [ 'gui8', 'msgs10', 'rendering8', 'sensors8', - 'sdformat14', 'sim8', 'transport13', 'utils1' // see comment https://github.com/gazebo-tooling/release-tools/pull/431#issuecomment-815099918 diff --git a/jenkins-scripts/dsl/sdformat.dsl b/jenkins-scripts/dsl/sdformat.dsl index f8dd47848..30009b06b 100644 --- a/jenkins-scripts/dsl/sdformat.dsl +++ b/jenkins-scripts/dsl/sdformat.dsl @@ -4,7 +4,7 @@ import javaposse.jobdsl.dsl.Job def sdformat_supported_versions = [ 'sdformat6' , 'sdformat9', 'sdformat12', 'sdformat13' ] def sdformat_gz11_versions = [ 'sdformat9', 'sdformat12', 'sdformat13', 'main' ] // nightly and prereleases -def extra_sdformat_debbuilder = [ ] +def extra_sdformat_debbuilder = ['sdformat14'] // Main platform using for quick CI def ci_distro = Globals.get_ci_distro() From 434431db68508c89d62225ff91712edf216b0eb1 Mon Sep 17 00:00:00 2001 From: "Addisu Z. Taddese" Date: Thu, 3 Aug 2023 21:07:41 -0500 Subject: [PATCH 410/540] Add physics7 nightlies (#984) --- jenkins-scripts/dsl/gz-collections.yaml | 4 ++-- jenkins-scripts/dsl/ignition.dsl | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/jenkins-scripts/dsl/gz-collections.yaml b/jenkins-scripts/dsl/gz-collections.yaml index b7363e9c6..d4e2fb9b8 100644 --- a/jenkins-scripts/dsl/gz-collections.yaml +++ b/jenkins-scripts/dsl/gz-collections.yaml @@ -99,9 +99,9 @@ collections: repo: current_branch: main - name: gz-physics - major_version: 6 + major_version: 7 repo: - current_branch: gz-physics6 + current_branch: main - name: gz-sim major_version: 8 repo: diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index 8f5bad13a..f1652ef27 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -87,6 +87,7 @@ gz_extra_debbuild = [ 'gui8', 'fuel-tools9', 'launch7', 'msgs10', + 'physics7', 'rendering8', 'sensors8', 'sim8', From 5183764199953b6bf53555e703e31962dfed63d6 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 4 Aug 2023 19:33:25 +0200 Subject: [PATCH 411/540] Implement pre_setup_script for yaml and limit gz-physics MAJE_JOBS Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/gazebo_libs.dsl | 5 +++++ jenkins-scripts/dsl/gz-collections.yaml | 9 +++++++++ 2 files changed, 14 insertions(+) diff --git a/jenkins-scripts/dsl/gazebo_libs.dsl b/jenkins-scripts/dsl/gazebo_libs.dsl index bdd41acdd..4dcd8c637 100644 --- a/jenkins-scripts/dsl/gazebo_libs.dsl +++ b/jenkins-scripts/dsl/gazebo_libs.dsl @@ -87,6 +87,8 @@ void generate_ci_job(gz_ci_job, lib_name, branch, ci_config, { def distro = ci_config.system.version def arch = ci_config.system.arch + def pre_setup_script = ci_config.pre_setup_script_hook.get(lib_name)?.join('\n') + extra_cmd = [extra_cmd, pre_setup_script].findAll({ it != null }).join() OSRFLinuxCompilation.create(gz_ci_job, is_testing_enabled(lib_name, ci_config)) OSRFGitHub.create(gz_ci_job, @@ -141,6 +143,8 @@ configs_per_lib_index.each { lib_name, lib_configs -> def gz_job_name_prefix = lib_name.replaceAll('-','_') def gz_ci_job_name = "${gz_job_name_prefix}-ci-pr_any-${distro}-${arch}" def gz_ci_any_job = job(gz_ci_job_name) + def pre_setup_script = ci_config.pre_setup_script_hook.get(lib_name)?.join('\n') + def extra_cmd = pre_setup_script ?: "" OSRFLinuxCompilationAnyGitHub.create(gz_ci_any_job, "gazebosim/${lib_name}", is_testing_enabled(lib_name, ci_config), @@ -157,6 +161,7 @@ configs_per_lib_index.each { lib_name, lib_configs -> export DISTRO=${distro} ${GLOBAL_SHELL_CMD} + ${extra_cmd} export BUILDING_SOFTWARE_DIRECTORY=${lib_name} export ARCH=${arch} diff --git a/jenkins-scripts/dsl/gz-collections.yaml b/jenkins-scripts/dsl/gz-collections.yaml index d4e2fb9b8..82ff05115 100644 --- a/jenkins-scripts/dsl/gz-collections.yaml +++ b/jenkins-scripts/dsl/gz-collections.yaml @@ -142,6 +142,9 @@ ci_configs: - ign-gui - ign-rendering - ign-sensors + pre_setup_script_hook: + gz-physics: + - "export MAKE_JOBS=1" tests_disabled: - name: focal system: @@ -160,6 +163,9 @@ ci_configs: exclude: - gz-fortress - gz-garden + pre_setup_script_hook: + ign-physics: + - "export MAKE_JOBS=1" tests_disabled: - name: jammy system: @@ -177,4 +183,7 @@ ci_configs: - gz-sensors exclude: - gz-harmonic + pre_setup_script_hook: + gz-physics: + - "export MAKE_JOBS=1" tests_disabled: From 46554982089d5d9d9fd99050ba5532672460d612 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 4 Aug 2023 19:33:46 +0200 Subject: [PATCH 412/540] Implement physics limitation to MAKE_JOBS 1 Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/ignition.dsl | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index f1652ef27..391d24d92 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -419,7 +419,7 @@ gz_software.each { gz_sw -> label Globals.nontest_label("large-memory") // on ARM native nodes in buildfarm we need to restrict to 1 the // compilation threads to avoid OOM killer - extra_str += '\nif [ $(uname -m) = "aarch64" ]; then export MAKE_JOBS=1; fi' + extra_str += '\nexport MAKE_JOBS=1' } steps { @@ -458,8 +458,10 @@ gz_software.each { gz_sw -> include_gpu_label_if_needed(gz_ci_any_job, software_name) gz_ci_any_job.with { - if (gz_sw == 'physics') + if (gz_sw == 'physics') { label Globals.nontest_label("large-memory") + extra_str += '\nexport MAKE_JOBS=1' + } steps { @@ -567,8 +569,10 @@ void generate_ci_job(gz_ci_job, gz_sw, branch, distro, arch, include_gpu_label_if_needed(gz_ci_job, gz_sw) gz_ci_job.with { - if (gz_sw == 'physics') + if (gz_sw == 'physics') { label Globals.nontest_label("large-memory") + extra_str += '\nexport MAKE_JOBS=1' + } if (gz_sw == 'gazebo') gz_sw = 'sim' @@ -598,7 +602,7 @@ all_debbuilders().each { debbuilder_name -> // Gazebo physics consumes huge amount of memory making arm node to FAIL // Force here to use one compilation thread if (debbuilder_name.contains("-physics")) - extra_str += '\nif [ $(uname -m) = "aarch64" ]; then export MAKE_JOBS=1; fi' + extra_str += '\nexport MAKE_JOBS=1' def build_pkg_job = job("${debbuilder_name}-debbuilder") OSRFLinuxBuildPkg.create(build_pkg_job) From 1edad1681602d6689467f4dbdf10622010a524fc Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Sat, 5 Aug 2023 13:24:42 +0200 Subject: [PATCH 413/540] Protect against the lack of pre_setup_script_hook attribute Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/gazebo_libs.dsl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jenkins-scripts/dsl/gazebo_libs.dsl b/jenkins-scripts/dsl/gazebo_libs.dsl index 4dcd8c637..82c6b0e2b 100644 --- a/jenkins-scripts/dsl/gazebo_libs.dsl +++ b/jenkins-scripts/dsl/gazebo_libs.dsl @@ -87,7 +87,7 @@ void generate_ci_job(gz_ci_job, lib_name, branch, ci_config, { def distro = ci_config.system.version def arch = ci_config.system.arch - def pre_setup_script = ci_config.pre_setup_script_hook.get(lib_name)?.join('\n') + def pre_setup_script = ci_config.pre_setup_script_hook?.get(lib_name)?.join('\n') extra_cmd = [extra_cmd, pre_setup_script].findAll({ it != null }).join() OSRFLinuxCompilation.create(gz_ci_job, is_testing_enabled(lib_name, ci_config)) @@ -143,7 +143,7 @@ configs_per_lib_index.each { lib_name, lib_configs -> def gz_job_name_prefix = lib_name.replaceAll('-','_') def gz_ci_job_name = "${gz_job_name_prefix}-ci-pr_any-${distro}-${arch}" def gz_ci_any_job = job(gz_ci_job_name) - def pre_setup_script = ci_config.pre_setup_script_hook.get(lib_name)?.join('\n') + def pre_setup_script = ci_config.pre_setup_script_hook?.get(lib_name)?.join('\n') def extra_cmd = pre_setup_script ?: "" OSRFLinuxCompilationAnyGitHub.create(gz_ci_any_job, "gazebosim/${lib_name}", From e69e2a42b268c1f58265360e3e048158aa13d855 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Crist=C3=B3bal=20Arroyo?= Date: Tue, 8 Aug 2023 13:04:25 -0500 Subject: [PATCH 414/540] Remove duplicate `large-memory` label --- docs/BUILDFARM_NODES.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/BUILDFARM_NODES.md b/docs/BUILDFARM_NODES.md index a5daf8305..0d0dc6127 100644 --- a/docs/BUILDFARM_NODES.md +++ b/docs/BUILDFARM_NODES.md @@ -12,7 +12,6 @@ used (can check this in the Jenkins UI) | docker | Node has capabilities to run Docker CI (standard Linux CI) | Linux system with docker installed | | gpu-reliable | Node has a real GPU able to run simulation for Gazebo | Nvidia card and nvidia-docker installed on Linux | | large-memory | Node has enough RAM to run really demanding RAM compilations | Hardware has no less than 16Gb of RAM and can run abichecker on ign-physics | -| large-memory | Node has enough RAM to run non trivial compilations | Hardware has no less than 16GB of RAM on Linux | | linux-arm64 | Node has capabilities to run native arm64 code (mostly used in packaging) | Bare-metal ARM machine | | linux-armhf | Node has capabilities to run native armhf code (mostly used in packaging) | Bare-metal ARM machine | | osx | Node has capabilities to run native OsX code | Apple system | From 1d4a222abe56c13dfe280b60a2eb9632efe600b5 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Mon, 14 Aug 2023 13:03:56 +0200 Subject: [PATCH 415/540] Script to generate tarballs from Gazebo libs source code Signed-off-by: Jose Luis Rivero --- .../docker/gz-source-generation.bash | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 jenkins-scripts/docker/gz-source-generation.bash diff --git a/jenkins-scripts/docker/gz-source-generation.bash b/jenkins-scripts/docker/gz-source-generation.bash new file mode 100644 index 000000000..c5bb4d36f --- /dev/null +++ b/jenkins-scripts/docker/gz-source-generation.bash @@ -0,0 +1,40 @@ +#!/bin/bash -xe +[[ -L ${0} ]] && SCRIPT_DIR=$(readlink ${0}) || SCRIPT_DIR=${0} +SCRIPT_DIR="${SCRIPT_DIR%/*}" + +echo '# BEGIN SECTION: setup the testing enviroment' +# Define the name to be used in docker +export DOCKER_JOB_NAME="source_generation_job" +. ${SCRIPT_DIR}/lib/boilerplate_prepare.sh +echo '# END SECTION' + +cat > build.sh << DELIM +#!/bin/bash +set -ex + +PKG_DIR=\$WORKSPACE/pkgs +SOURCES_DIR=\$WORKSPACE/sources +BUILD_DIR=\$SOURCES_DIR/build + +cd \${WORKSPACE} +rm -fr \$SOURCES_DIR && mkdir \$SOURCES_DIR +git clone --depth 1 --branch ${PACKAGE_NAME}_${VERSION} ${SOURCE_REPO_URI} \${SOURCES_DIR} +rm -fr \$BUILD_DIR && mkdir \$BUILD_DIR +cd \${BUILD_DIR} +cmake .. -DPACKAGE_SOURCE_ONLY:BOOL=ON +make package_source + +rm -fr \$PKG_DIR && mkdir \$PKG_DIR +find \${BUILD_DIR} -maxdepth 1 -name '*${VERSION}.tar.*' -exec mv {} \${PKG_DIR} \\; + +if [ $(ls 2>/dev/null -Ubad1 -- "\${PKG_DIR}" | wc -l) -gt 1 ]; then + echo "Found more than one file inside pkgs directory:" + ls \${PKG_DIR} + exit 1 +fi +DELIM + +export DEPENDENCY_PKGS="cmake git" + +. ${SCRIPT_DIR}/lib/docker_generate_dockerfile.bash +. ${SCRIPT_DIR}/lib/docker_run.bash From f281a021d233ccf1369c3f116e4161904b199933 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Mon, 14 Aug 2023 13:14:45 +0200 Subject: [PATCH 416/540] DSL class for the tarball generation Signed-off-by: Jose Luis Rivero --- .../dsl/_configs_/OSRFSourceCreation.groovy | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy diff --git a/jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy b/jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy new file mode 100644 index 000000000..5d24ff8da --- /dev/null +++ b/jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy @@ -0,0 +1,60 @@ +package _configs_ + +import javaposse.jobdsl.dsl.Job +import _configs_.Globals + +class OSRFSourceCreation +{ + static void addParameters(Job job, Map default_params = [:]) + { + job.with + { + parameters { + stringParam("VERSION", + default_params.find{ it.key == "VERSION"}?.value, + "Packages version to be built or nightly (enable nightly build mode)") + stringParam("RELEASE_VERSION", + default_params.find{ it.key == "RELEASE_VERSION"}?.value, + "Packages release version") + stringParam("RELEASE_REPO_BRANCH", + default_params.find{ it.key == "RELEASE_REPO_BRANCH"}?.value, + "Branch from the -release repo to be used") + stringParam("UPLOAD_TO_REPO", + default_params.find{ it.key == "UPLOAD_TO_REPO"}?.value, + "OSRF repo name to upload the package to: stable | prerelease | nightly | none (for testing proposes)") + } + } + } + + static void create(Job job, Map default_params = [:]) + { + OSRFLinuxBuildPkgBase.create(job) + GenericRemoteToken.create(job) + OSRFSourceCreation.addParameters(job, default_params) + + job.with + { + wrappers { + preBuildCleanup() + } + + properties { + priority 100 + } + + steps { + systemGroovyCommand("""\ + build.setDescription( + '' + build.buildVariableResolver.resolve('VERSION') + '-' + + build.buildVariableResolver.resolve('RELEASE_VERSION') + '' + + '
' + + 'branch: ' + build.buildVariableResolver.resolve('RELEASE_REPO_BRANCH') + ' | ' + + 'upload to: ' + build.buildVariableResolver.resolve('UPLOAD_TO_REPO') + + '
' + + 'RTOOLS_BRANCH: ' + build.buildVariableResolver.resolve('RTOOLS_BRANCH')); + """.stripIndent() + ) + } + } + } +} From 8210a70e563d28a14166b922bebe58a7263e4588 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Mon, 14 Aug 2023 13:15:01 +0200 Subject: [PATCH 417/540] Add a test job for the source generation Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/test.dsl | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/jenkins-scripts/dsl/test.dsl b/jenkins-scripts/dsl/test.dsl index 66661adb7..134f4e306 100644 --- a/jenkins-scripts/dsl/test.dsl +++ b/jenkins-scripts/dsl/test.dsl @@ -17,6 +17,30 @@ OSRFLinuxCompilationAnyGitHub.create(ignition_ci_pr_job, false, ['main']) + + +def gz_source_job = job("_test_gz_source") +OSRFSourceCreation.create(gz_source_job) +gz_source_job.with +{ + label Globals.nontest_label("docker") + + def PACKAGE_NAME="gz-cmake3" + + steps { + shell("""\ + #!/bin/bash -xe + + # Use Jammy/amd64 as base image to generate sources + export DISTRO=jammy + export ARCH=amd64 + export PACKAGE_NAME=${PACKAGE_NAME} + export SOURCE_REPO_URI=https://github.com/gazebosim/gz-cmake.git + + /bin/bash -x ./scripts/jenkins-scripts/docker/gz-source-generation.bash + """.stripIndent()) + } +} // ------------------------------------------------------------------- def outdated_job_runner = job("_test_outdated_job_runner") OSRFBase.create(outdated_job_runner) From baa460799efbf0a37990cc5924379193f56d7bd3 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Mon, 14 Aug 2023 14:11:57 +0200 Subject: [PATCH 418/540] Relocate common configurations in DSL Class Signed-off-by: Jose Luis Rivero --- .../dsl/_configs_/OSRFSourceCreation.groovy | 16 ++++++++++++ jenkins-scripts/dsl/test.dsl | 25 +++---------------- 2 files changed, 19 insertions(+), 22 deletions(-) diff --git a/jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy b/jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy index 5d24ff8da..cb01aa186 100644 --- a/jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy +++ b/jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy @@ -10,6 +10,12 @@ class OSRFSourceCreation job.with { parameters { + stringParam("PACKAGE_NAME", + default_params.find{ it.key == "PACKAGE_NAME"}?.value, + "Software name (i.e gz-cmake3)") + stringParam("SOURCE_REPO_URI", + default_params.find{ it.key == "SOURCE_REPO_URI"}?.value, + "GitHub URI to release the sources from (i.e: https://github.com/gazebosim/gz-cmake.git)") stringParam("VERSION", default_params.find{ it.key == "VERSION"}?.value, "Packages version to be built or nightly (enable nightly build mode)") @@ -34,6 +40,8 @@ class OSRFSourceCreation job.with { + label Globals.nontest_label("docker") + wrappers { preBuildCleanup() } @@ -54,6 +62,14 @@ class OSRFSourceCreation 'RTOOLS_BRANCH: ' + build.buildVariableResolver.resolve('RTOOLS_BRANCH')); """.stripIndent() ) + + shell("""\ + #!/bin/bash -xe + export DISTRO=jammy + export ARCH=amd64 + + /bin/bash -x ./scripts/jenkins-scripts/docker/gz-source-generation.bash + """.stripIndent()) } } } diff --git a/jenkins-scripts/dsl/test.dsl b/jenkins-scripts/dsl/test.dsl index 134f4e306..6da586a25 100644 --- a/jenkins-scripts/dsl/test.dsl +++ b/jenkins-scripts/dsl/test.dsl @@ -17,30 +17,11 @@ OSRFLinuxCompilationAnyGitHub.create(ignition_ci_pr_job, false, ['main']) - - def gz_source_job = job("_test_gz_source") -OSRFSourceCreation.create(gz_source_job) -gz_source_job.with -{ - label Globals.nontest_label("docker") - - def PACKAGE_NAME="gz-cmake3" +OSRFSourceCreation.create(gz_source_job, [ + PACKAGE_NAME: "gz-cmake3", + SOURCE_REPO_URI: "https://github.com/gazebosim/gz-cmake.git"]) - steps { - shell("""\ - #!/bin/bash -xe - - # Use Jammy/amd64 as base image to generate sources - export DISTRO=jammy - export ARCH=amd64 - export PACKAGE_NAME=${PACKAGE_NAME} - export SOURCE_REPO_URI=https://github.com/gazebosim/gz-cmake.git - - /bin/bash -x ./scripts/jenkins-scripts/docker/gz-source-generation.bash - """.stripIndent()) - } -} // ------------------------------------------------------------------- def outdated_job_runner = job("_test_outdated_job_runner") OSRFBase.create(outdated_job_runner) From 5a658b18dd9a5b82e1c0c90832c0c6232c8bf0e4 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Mon, 14 Aug 2023 14:23:13 +0200 Subject: [PATCH 419/540] Add OSRFReleasepy DSL class (#988) * Add OSRFReleasepy DSL class Wrapper to call release.py using Jenkins parameters as release.py arguments. Note that the script is not fully functional for standard releases since it currently requires a local software checkout to be run. This can work with nightly builds or future calls to release.py that don't need to generate the tarballs. The test inject DRY_RUN to avoid any real run. * Update test.dsl --- .../dsl/_configs_/OSRFReleasepy.groovy | 84 +++++++++++++++++++ jenkins-scripts/dsl/test.dsl | 6 +- 2 files changed, 89 insertions(+), 1 deletion(-) create mode 100644 jenkins-scripts/dsl/_configs_/OSRFReleasepy.groovy diff --git a/jenkins-scripts/dsl/_configs_/OSRFReleasepy.groovy b/jenkins-scripts/dsl/_configs_/OSRFReleasepy.groovy new file mode 100644 index 000000000..d701641f7 --- /dev/null +++ b/jenkins-scripts/dsl/_configs_/OSRFReleasepy.groovy @@ -0,0 +1,84 @@ +package _configs_ + +import javaposse.jobdsl.dsl.Job +import _configs_.Globals + +class OSRFReleasepy +{ + static void create(Job job, Map default_params = [:]) + { + // Base class for the job + OSRFUNIXBase.create(job) + + job.with + { + label Globals.nontest_label("master") + + parameters + { + stringParam("PACKAGE", + default_params.find{ it.key == "PACKAGE"}?.value, + "Package name to be built") + stringParam("VERSION", + default_params.find{ it.key == "VERSION"}?.value, + "Packages version to be built or nightly (enable nightly build mode)") + stringParam("RELEASE_VERSION", + default_params.find{ it.key == "RELEASE_VERSION"}?.value, + "Packages release version") + stringParam("SOURCE_TARBALL_URI", + default_params.find{ it.key == "SOURCE_TARBALL_URI"}?.value, + "URL to the tarball containing the package sources") + stringParam("RELEASE_REPO_BRANCH", + default_params.find{ it.key == "RELEASE_REPO_BRANCH"}?.value, + "Branch from the -release repo to be used") + stringParam("UPLOAD_TO_REPO", + default_params.find{ it.key == "UPLOAD_TO_REPO"}?.value, + "OSRF repo name to upload the package to: stable | prerelease | nightly | none (for testing proposes)") + stringParam("EXTRA_OSRF_REPO", + default_params.find{ it.key == "OSRF_REPOS_TO_USE"}?.value, + "OSRF repos name to use when building the package") + + booleanParam('DRY_RUN', + default_params.find{ it.key == "DRY_RUN"}?.value, + 'run a testing run with no effects') + } + + steps { + systemGroovyCommand("""\ + build.setDescription( + '' + build.buildVariableResolver.resolve('PACKAGE') + '/' + + '' + build.buildVariableResolver.resolve('VERSION') + '-' + + build.buildVariableResolver.resolve('RELEASE_VERSION') + '' + + '
' + + 'branch: ' + build.buildVariableResolver.resolve('RELEASE_REPO_BRANCH') + ' | ' + + 'upload to: ' + build.buildVariableResolver.resolve('UPLOAD_TO_REPO') + + '
' + + 'RTOOLS_BRANCH: ' + build.buildVariableResolver.resolve('RTOOLS_BRANCH')); + """.stripIndent() + ) + + shell("""\ + #!/bin/bash -xe + set +x # keep password secret + PASS=\$(cat \$HOME/build_pass) + + dry_run_str="" + if \$DRY_RUN; then + dry_run_str="--dry-run" + fi + + extra_osrf_repo="" + if [[ -n \${EXTRA_OSRF_REPO} ]]; then + extra_osrf_repo="--extra-osrf-repo \${EXTRA_OSRF_REPO}" + fi + + echo "releasing \${n} (from branch \${src_branch})" + python3 ./scripts/release.py \${dry_run_str} "\${PACKAGE}" "\${VERSION}" "\${PASS}" \${extra_osrf_repo} \ + --release-repo-branch \${RELEASE_REPO_BRANCH} \ + --upload-to-repo \${UPLOAD_TO_REPO} > log || echo "MARK_AS_UNSTABLE" + echo " - done" + """.stripIndent()) + } + } + } +} diff --git a/jenkins-scripts/dsl/test.dsl b/jenkins-scripts/dsl/test.dsl index 6da586a25..c8f542e8d 100644 --- a/jenkins-scripts/dsl/test.dsl +++ b/jenkins-scripts/dsl/test.dsl @@ -3,7 +3,7 @@ import javaposse.jobdsl.dsl.Job Globals.default_emails = "jrivero@osrfoundation.org" -// Jobs +// Usual CI jobs def ignition_ci_job = job("_test_job_from_dsl") OSRFLinuxBase.create(ignition_ci_job) def ignition_ci_job_mac = job("_test_job_from_dsl_mac") @@ -17,6 +17,10 @@ OSRFLinuxCompilationAnyGitHub.create(ignition_ci_pr_job, false, ['main']) +// releasing testing job +def releasepy_job = job("_test_releasepy") +OSRFReleasepy.create(releasepy_job, [DRY_RUN: true]) +// gz source testing job def gz_source_job = job("_test_gz_source") OSRFSourceCreation.create(gz_source_job, [ PACKAGE_NAME: "gz-cmake3", From a7fe2780af3b1f85fe66c13055ebe51858343c70 Mon Sep 17 00:00:00 2001 From: claraberendsen Date: Tue, 22 Aug 2023 16:51:28 -0400 Subject: [PATCH 420/540] remove reference to custom default binary cache --- jenkins-scripts/lib/windows_library.bat | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/jenkins-scripts/lib/windows_library.bat b/jenkins-scripts/lib/windows_library.bat index 995bb29a6..7f4e62493 100644 --- a/jenkins-scripts/lib/windows_library.bat +++ b/jenkins-scripts/lib/windows_library.bat @@ -304,8 +304,7 @@ if [%1] == [] ( echo "_install_and_upgrade_vcpkg_package called with no argument" goto :error ) -:: workaround on permissions problems for default VCPKG_DEFAULT_BINARY_CACHE -set VCPKG_DEFAULT_BINARY_CACHE=C:\Windows\Temp\vcpkg + if not exist %VCPKG_DEFAULT_BINARY_CACHE% mkdir %VCPKG_DEFAULT_BINARY_CACHE% %VCPKG_CMD% install --recurse "%1" --overlay-ports="%VCPKG_OSRF_DIR%" :: vcpkg does not upgrade installed packages using the install command From afe7ad04544664f6a22e957fe82812c7c1688090 Mon Sep 17 00:00:00 2001 From: claraberendsen Date: Tue, 22 Aug 2023 17:05:17 -0400 Subject: [PATCH 421/540] remove line --- jenkins-scripts/lib/windows_library.bat | 1 - 1 file changed, 1 deletion(-) diff --git a/jenkins-scripts/lib/windows_library.bat b/jenkins-scripts/lib/windows_library.bat index 7f4e62493..7489c7f50 100644 --- a/jenkins-scripts/lib/windows_library.bat +++ b/jenkins-scripts/lib/windows_library.bat @@ -304,7 +304,6 @@ if [%1] == [] ( echo "_install_and_upgrade_vcpkg_package called with no argument" goto :error ) - if not exist %VCPKG_DEFAULT_BINARY_CACHE% mkdir %VCPKG_DEFAULT_BINARY_CACHE% %VCPKG_CMD% install --recurse "%1" --overlay-ports="%VCPKG_OSRF_DIR%" :: vcpkg does not upgrade installed packages using the install command From 395803ac1e7f20d5ad3518aa9e5cf718920c80fa Mon Sep 17 00:00:00 2001 From: claraberendsen Date: Tue, 22 Aug 2023 17:11:17 -0400 Subject: [PATCH 422/540] remove all reference to cache --- jenkins-scripts/lib/windows_library.bat | 1 - 1 file changed, 1 deletion(-) diff --git a/jenkins-scripts/lib/windows_library.bat b/jenkins-scripts/lib/windows_library.bat index 7489c7f50..82c615bdd 100644 --- a/jenkins-scripts/lib/windows_library.bat +++ b/jenkins-scripts/lib/windows_library.bat @@ -304,7 +304,6 @@ if [%1] == [] ( echo "_install_and_upgrade_vcpkg_package called with no argument" goto :error ) -if not exist %VCPKG_DEFAULT_BINARY_CACHE% mkdir %VCPKG_DEFAULT_BINARY_CACHE% %VCPKG_CMD% install --recurse "%1" --overlay-ports="%VCPKG_OSRF_DIR%" :: vcpkg does not upgrade installed packages using the install command :: since most of the packages are coming from a frozen snapshot, it is From 9013975991dac9f797ed114d42bbad10dddfde1b Mon Sep 17 00:00:00 2001 From: claraberendsen Date: Tue, 22 Aug 2023 22:39:42 -0400 Subject: [PATCH 423/540] change default binary cache path to vcpkg default --- jenkins-scripts/lib/windows_library.bat | 3 +++ 1 file changed, 3 insertions(+) diff --git a/jenkins-scripts/lib/windows_library.bat b/jenkins-scripts/lib/windows_library.bat index 82c615bdd..6701f26a7 100644 --- a/jenkins-scripts/lib/windows_library.bat +++ b/jenkins-scripts/lib/windows_library.bat @@ -304,6 +304,9 @@ if [%1] == [] ( echo "_install_and_upgrade_vcpkg_package called with no argument" goto :error ) +:: workaround on permissions problems for default VCPKG_DEFAULT_BINARY_CACHE +set VCPKG_DEFAULT_BINARY_CACHE=C:\Users\Administrator\AppData\Local\vcpkg\archives +if not exist %VCPKG_DEFAULT_BINARY_CACHE% mkdir %VCPKG_DEFAULT_BINARY_CACHE% %VCPKG_CMD% install --recurse "%1" --overlay-ports="%VCPKG_OSRF_DIR%" :: vcpkg does not upgrade installed packages using the install command :: since most of the packages are coming from a frozen snapshot, it is From 78fb65b6e1eeee593bd4d22b4554fea625451913 Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Fri, 25 Aug 2023 14:51:46 -0700 Subject: [PATCH 424/540] brew_release.dsl: build ventura bottles (#992) Signed-off-by: Steve Peters --- jenkins-scripts/dsl/brew_release.dsl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/brew_release.dsl b/jenkins-scripts/dsl/brew_release.dsl index 5adb36bd0..2964076c8 100644 --- a/jenkins-scripts/dsl/brew_release.dsl +++ b/jenkins-scripts/dsl/brew_release.dsl @@ -4,7 +4,7 @@ import javaposse.jobdsl.dsl.Job Globals.default_emails = "jrivero@osrfoundation.org, scpeters@osrfoundation.org" // first distro in list is used as touchstone -brew_supported_distros = [ "bigsur", "monterey" ] +brew_supported_distros = [ "bigsur", "monterey", "ventura" ] bottle_hash_updater_job_name = 'generic-release-homebrew_pr_bottle_hash_updater' bottle_builder_job_name = 'generic-release-homebrew_triggered_bottle_builder' directory_for_bottles = 'pkgs' From d60ee56a8ce2dfc34197611eaa8393a3a2c606f4 Mon Sep 17 00:00:00 2001 From: "Addisu Z. Taddese" Date: Tue, 29 Aug 2023 11:16:20 -0500 Subject: [PATCH 425/540] Add stable CI job for gz-msgs10 (#993) Signed-off-by: Addisu Z. Taddese --- jenkins-scripts/dsl/gz-collections.yaml | 2 +- jenkins-scripts/dsl/ignition.dsl | 3 +-- jenkins-scripts/dsl/ignition_collection.dsl | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/jenkins-scripts/dsl/gz-collections.yaml b/jenkins-scripts/dsl/gz-collections.yaml index 82ff05115..c5276524b 100644 --- a/jenkins-scripts/dsl/gz-collections.yaml +++ b/jenkins-scripts/dsl/gz-collections.yaml @@ -73,7 +73,7 @@ collections: - name: gz-msgs major_version: 10 repo: - current_branch: main + current_branch: gz-msgs10 - name: gz-rendering major_version: 8 repo: diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index 391d24d92..0e33e6cc3 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -67,7 +67,7 @@ gz_branches = [ 'cmake' : [ '3' ], 'gui' : [ '7' ], 'launch' : [ '6' ], 'math' : [ '7' ], - 'msgs' : [ '9' ], + 'msgs' : [ '9', '10' ], 'physics' : [ '6' ], 'plugin' : [ '2' ], 'rendering' : [ '7' ], @@ -86,7 +86,6 @@ gz_prerelease_branches = [] gz_extra_debbuild = [ 'gui8', 'fuel-tools9', 'launch7', - 'msgs10', 'physics7', 'rendering8', 'sensors8', diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index a0b2e26da..eff285377 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -296,7 +296,7 @@ gz_collection_jobs = 'gz_gui-ci-main-jammy-amd64', 'gz_launch-ci-main-jammy-amd64', 'gz_math-ci-gz-math7-jammy-amd64', - 'gz_msgs-ci-main-jammy-amd64', + 'gz_msgs-ci-gz-msgs10-jammy-amd64', 'gz_physics-ci-gz-physics6-jammy-amd64', 'gz_plugin-ci-gz-plugin2-jammy-amd64', 'gz_rendering-ci-main-jammy-amd64', From 3e6c506ebf0b06b7a075b744bf4816c6f9d1e3eb Mon Sep 17 00:00:00 2001 From: "Addisu Z. Taddese" Date: Tue, 29 Aug 2023 13:49:25 -0500 Subject: [PATCH 426/540] Use stable branches for gz-rendering and sdformat Signed-off-by: Addisu Z. Taddese --- jenkins-scripts/dsl/gz-collections.yaml | 4 ++-- jenkins-scripts/dsl/ignition.dsl | 5 ++--- jenkins-scripts/dsl/ignition_collection.dsl | 12 ++++++------ source-repo-scripts/merge_forward_pull_request.bash | 8 ++++++-- source-repo-scripts/release_pull_request.bash | 8 ++++++-- 5 files changed, 22 insertions(+), 15 deletions(-) diff --git a/jenkins-scripts/dsl/gz-collections.yaml b/jenkins-scripts/dsl/gz-collections.yaml index c5276524b..cc8e38517 100644 --- a/jenkins-scripts/dsl/gz-collections.yaml +++ b/jenkins-scripts/dsl/gz-collections.yaml @@ -77,11 +77,11 @@ collections: - name: gz-rendering major_version: 8 repo: - current_branch: main + current_branch: gz-rendering8 - name: sdformat major_version: 14 repo: - current_branch: main + current_branch: sdf14 - name: gz-fuel-tools major_version: 9 repo: diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index 0e33e6cc3..1e19a2537 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -67,10 +67,10 @@ gz_branches = [ 'cmake' : [ '3' ], 'gui' : [ '7' ], 'launch' : [ '6' ], 'math' : [ '7' ], - 'msgs' : [ '9', '10' ], + 'msgs' : [ '9', '10'], 'physics' : [ '6' ], 'plugin' : [ '2' ], - 'rendering' : [ '7' ], + 'rendering' : [ '7' , '8'], 'sensors' : [ '7' ], 'sim' : [ '7' ], 'tools' : [ '2' ], @@ -87,7 +87,6 @@ gz_extra_debbuild = [ 'gui8', 'fuel-tools9', 'launch7', 'physics7', - 'rendering8', 'sensors8', 'sim8', 'transport13', diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index eff285377..b82ee6c8c 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -285,7 +285,7 @@ gz_collection_jobs = 'ign_msgs-gz-10-win', 'ign_physics-gz-6-win', 'ign_plugin-gz-2-win', - 'ign_rendering-ci-win', + 'ign_rendering-gz-8-win', 'ign_sensors-ci-win', 'ign_tools-gz-2-win', 'ign_transport-gz-13-win', @@ -299,7 +299,7 @@ gz_collection_jobs = 'gz_msgs-ci-gz-msgs10-jammy-amd64', 'gz_physics-ci-gz-physics6-jammy-amd64', 'gz_plugin-ci-gz-plugin2-jammy-amd64', - 'gz_rendering-ci-main-jammy-amd64', + 'gz_rendering-ci-gz-rendering8-jammy-amd64', 'gz_sensors-ci-main-jammy-amd64', 'gz_sim-ci-main-jammy-amd64', 'gz_tools-ci-gz-tools2-jammy-amd64', @@ -316,14 +316,14 @@ gz_collection_jobs = 'ignition_msgs-ci-gz-msgs10-homebrew-amd64', 'ignition_physics-ci-gz-physics6-homebrew-amd64', 'ignition_plugin-ci-gz-plugin2-homebrew-amd64', - 'ignition_rendering-ci-main-homebrew-amd64', + 'ignition_rendering-ci-gz-rendering8-homebrew-amd64', 'ignition_sensors-ci-main-homebrew-amd64', 'ignition_tools-ci-gz-tools2-homebrew-amd64', 'ignition_transport-ci-gz-transport13-homebrew-amd64', 'ignition_utils-ci-gz-utils2-homebrew-amd64', - 'sdformat-ci-main-jammy-amd64', - 'sdformat-ci-main-homebrew-amd64', - 'sdformat-sdf-main-win' + 'sdformat-ci-sdformat14-jammy-amd64', + 'sdformat-ci-sdformat14-homebrew-amd64', + 'sdformat-sdf-14-win' ], ] diff --git a/source-repo-scripts/merge_forward_pull_request.bash b/source-repo-scripts/merge_forward_pull_request.bash index 7b5ff7091..a6d3cc696 100755 --- a/source-repo-scripts/merge_forward_pull_request.bash +++ b/source-repo-scripts/merge_forward_pull_request.bash @@ -38,7 +38,10 @@ fi set -e -CURRENT_BRANCH=$(git branch | sed -n -e 's/^\* \(.*\)/\1/p') +LOCAL_BRANCH=$(git rev-parse --abbrev-ref HEAD) +REMOTE_BRANCH=$(git rev-parse --abbrev-ref HEAD@{upstream}) +REMOTE=${REMOTE_BRANCH/\/$LOCAL_BRANCH/} +CURRENT_BRANCH="${REMOTE}:${LOCAL_BRANCH}" ORIGIN_URL=$(git remote get-url origin) ORIGIN_ORG_REPO=$(echo ${ORIGIN_URL} | sed -e 's@.*github\.com.@@' | sed -e 's/\.git//g') @@ -58,4 +61,5 @@ gh pr create \ --repo "$ORIGIN_ORG_REPO" \ --base "$TO_BRANCH" \ --body "$BODY" \ - --head "$CURRENT_BRANCH" + --head "$CURRENT_BRANCH" \ + --web diff --git a/source-repo-scripts/release_pull_request.bash b/source-repo-scripts/release_pull_request.bash index f3800c8fd..2f700ace4 100755 --- a/source-repo-scripts/release_pull_request.bash +++ b/source-repo-scripts/release_pull_request.bash @@ -36,7 +36,10 @@ fi set -e -CURRENT_BRANCH=$(git branch | sed -n -e 's/^\* \(.*\)/\1/p') +LOCAL_BRANCH=$(git rev-parse --abbrev-ref HEAD) +REMOTE_BRANCH=$(git rev-parse --abbrev-ref HEAD@{upstream}) +REMOTE=${REMOTE_BRANCH/\/$LOCAL_BRANCH/} +CURRENT_BRANCH="${REMOTE}:${LOCAL_BRANCH}" ORIGIN_URL=$(git remote get-url origin) ORIGIN_ORG_REPO=$(echo ${ORIGIN_URL} | sed -e 's@.*github\.com.@@' | sed -e 's/\.git//g') @@ -73,4 +76,5 @@ gh pr create \ --repo "$ORIGIN_ORG_REPO" \ --base "$TO_BRANCH" \ --body "$BODY" \ - --head "$CURRENT_BRANCH" + --head "$CURRENT_BRANCH" \ + --web From b2ec1ae16c047eec0017a099db5afeda89ae482a Mon Sep 17 00:00:00 2001 From: Ian Chen Date: Wed, 30 Aug 2023 00:06:18 +0000 Subject: [PATCH 427/540] Use stable branch for gz-fuel-tools in harmonic Signed-off-by: Ian Chen --- jenkins-scripts/dsl/gz-collections.yaml | 4 ++-- jenkins-scripts/dsl/ignition.dsl | 11 +++++------ jenkins-scripts/dsl/ignition_collection.dsl | 2 +- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/jenkins-scripts/dsl/gz-collections.yaml b/jenkins-scripts/dsl/gz-collections.yaml index cc8e38517..11ae1889e 100644 --- a/jenkins-scripts/dsl/gz-collections.yaml +++ b/jenkins-scripts/dsl/gz-collections.yaml @@ -10,7 +10,7 @@ # ci: # - linux: # pr_distro: -# abi_distro: +# abi_distro: # install_distros: # - distro1 # - distro2 @@ -85,7 +85,7 @@ collections: - name: gz-fuel-tools major_version: 9 repo: - current_branch: main + current_branch: gz-fuel-tools9 - name: gz-transport major_version: 13 repo: diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index 1e19a2537..ccf842619 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -63,7 +63,7 @@ ignition_branches = [ 'cmake' : [ '2' ], gz_branches = [ 'cmake' : [ '3' ], 'common' : [ '5' ], - 'fuel-tools' : [ '8' ], + 'fuel-tools' : [ '8', '9' ], 'gui' : [ '7' ], 'launch' : [ '6' ], 'math' : [ '7' ], @@ -84,7 +84,6 @@ gz_prerelease_branches = [] // don't appear in gz_branches (like nightly builders or 0-debbuild // jobs for the special cases of foo0 packages) gz_extra_debbuild = [ 'gui8', - 'fuel-tools9', 'launch7', 'physics7', 'sensors8', @@ -380,11 +379,11 @@ void generate_install_job(prefix, gz_sw, major_version, distro, arch) // Need to be before the ci-pr_any so the abi job name is defined gz_software.each { gz_sw -> supported_arches.each { arch -> - // 1 Per library and per linux arch + // 1 Per library and per linux arch // 1.1 Per abi_distro // 1.1.1 [job] ABI checker for main branches // 1.2 Per ci_str_distro - // 1.2.1 [job] Main PR jobs (-ci-pr_any-) + // 1.2.1 [job] Main PR jobs (-ci-pr_any-) // 1.3 Per all supported_distros // 1.3.1 Per all supported branches on each library // 1.3.1.1 [job] Branch jobs -ci-$branch- @@ -499,7 +498,7 @@ gz_software.each { gz_sw -> scm('@daily') } } - // 1.3.1.2 Branch ASAN jobs -ci_asan-$branch- + // 1.3.1.2 Branch ASAN jobs -ci_asan-$branch- // -------------------------------------------------------------- def gz_ci_asan_job = job("ignition_${software_name}-ci_asan-${branch}-${distro}-${arch}") generate_asan_ci_job(gz_ci_asan_job, software_name, branch, distro, arch) @@ -551,7 +550,7 @@ void generate_asan_ci_job(gz_ci_job, gz_sw, branch, distro, arch) { generate_ci_job(gz_ci_job, gz_sw, branch, distro, arch, '-DGZ_SANITIZER=Address', - Globals.MAKETEST_SKIP_GZ, + Globals.MAKETEST_SKIP_GZ, 'export ASAN_OPTIONS=check_initialization_order=true:strict_init_order=true') } diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index b82ee6c8c..52cd8a14f 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -292,7 +292,7 @@ gz_collection_jobs = 'ign_utils-gz-2-win', 'gz_cmake-ci-gz-cmake3-jammy-amd64', 'gz_common-ci-gz-common5-jammy-amd64', - 'gz_fuel_tools-ci-main-jammy-amd64', + 'gz_fuel_tools-ci-gz-fuel-tools9-jammy-amd64', 'gz_gui-ci-main-jammy-amd64', 'gz_launch-ci-main-jammy-amd64', 'gz_math-ci-gz-math7-jammy-amd64', From 0bc31361d918d02f1afdeb1d9d8828ed022e13c7 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 30 Aug 2023 16:42:06 +0200 Subject: [PATCH 428/540] Update terminal dashboard to support Harmonic (#998) Signed-off-by: Jose Luis Rivero --- terminal-dashboard/_dashboard_lib.sh | 20 +++++++++++++++++++- terminal-dashboard/table.bash | 2 ++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/terminal-dashboard/_dashboard_lib.sh b/terminal-dashboard/_dashboard_lib.sh index 92d393246..0a6fbc676 100644 --- a/terminal-dashboard/_dashboard_lib.sh +++ b/terminal-dashboard/_dashboard_lib.sh @@ -62,9 +62,27 @@ get_libraries_by_collection() gz-sim7 gz-launch6 gz-garden" + elif [ "$COLLECTION" = "harmonic" ]; then + LIBS=" + gz-cmake3 + gz-math7 + gz-utils2 + gz-tools2 + gz-common5 + gz-msgs10 + gz-transport13 + gz-fuel-tools9 + gz-plugin2 + gz-rendering8 + sdformat14 + gz-physics7 + gz-sensors8 + gz-gui8 + gz-sim8 + gz-launch7 + gz-harmonic" else return 1 fi - echo "$LIBS" } diff --git a/terminal-dashboard/table.bash b/terminal-dashboard/table.bash index 2fe6c14eb..637386f81 100755 --- a/terminal-dashboard/table.bash +++ b/terminal-dashboard/table.bash @@ -67,6 +67,8 @@ for LIB in $(get_libraries_by_collection "${COLLECTION}" ); do VERS=( "bionic" "focal" "jammy" ) elif [[ $COLLECTION == "garden" ]]; then VERS=( "focal" "jammy" ) + elif [[ $COLLECTION == "harmonic" ]]; then + VERS=( "jammy" ) fi fi From 5a99a86248b51f06ef135a6a2f5f161b8c77df85 Mon Sep 17 00:00:00 2001 From: Ian Chen Date: Wed, 30 Aug 2023 11:22:34 -0700 Subject: [PATCH 429/540] Use stable branches for gz-sensors and gz-transport in harmonic (#997) * Use stable branches for gz-sensors and gz-transport inharmonic --------- Signed-off-by: Ian Chen --- jenkins-scripts/dsl/gz-collections.yaml | 4 ++-- jenkins-scripts/dsl/ignition.dsl | 8 +++----- jenkins-scripts/dsl/ignition_collection.dsl | 8 ++++---- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/jenkins-scripts/dsl/gz-collections.yaml b/jenkins-scripts/dsl/gz-collections.yaml index 11ae1889e..915c95b64 100644 --- a/jenkins-scripts/dsl/gz-collections.yaml +++ b/jenkins-scripts/dsl/gz-collections.yaml @@ -89,7 +89,7 @@ collections: - name: gz-transport major_version: 13 repo: - current_branch: main + current_branch: gz-transport13 - name: gz-gui major_version: 8 repo: @@ -97,7 +97,7 @@ collections: - name: gz-sensors major_version: 8 repo: - current_branch: main + current_branch: gz-sensors8 - name: gz-physics major_version: 7 repo: diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index ccf842619..02ef1fd6b 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -70,11 +70,11 @@ gz_branches = [ 'cmake' : [ '3' ], 'msgs' : [ '9', '10'], 'physics' : [ '6' ], 'plugin' : [ '2' ], - 'rendering' : [ '7' , '8'], - 'sensors' : [ '7' ], + 'rendering' : [ '7', '8' ], + 'sensors' : [ '7', '8' ], 'sim' : [ '7' ], 'tools' : [ '2' ], - 'transport' : [ '12' ], + 'transport' : [ '12', '13' ], 'utils' : [ '2' ]] // DESC: prerelease branches are managed as any other supported branches for // special cases different to major branches: get compilation CI on the branch @@ -86,9 +86,7 @@ gz_prerelease_branches = [] gz_extra_debbuild = [ 'gui8', 'launch7', 'physics7', - 'sensors8', 'sim8', - 'transport13', 'utils1' // see comment https://github.com/gazebo-tooling/release-tools/pull/431#issuecomment-815099918 ] // DESC: exclude ignition from generate any install testing job diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index 52cd8a14f..0474d1693 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -286,7 +286,7 @@ gz_collection_jobs = 'ign_physics-gz-6-win', 'ign_plugin-gz-2-win', 'ign_rendering-gz-8-win', - 'ign_sensors-ci-win', + 'ign_sensors-gz-8-win', 'ign_tools-gz-2-win', 'ign_transport-gz-13-win', 'ign_utils-gz-2-win', @@ -300,10 +300,10 @@ gz_collection_jobs = 'gz_physics-ci-gz-physics6-jammy-amd64', 'gz_plugin-ci-gz-plugin2-jammy-amd64', 'gz_rendering-ci-gz-rendering8-jammy-amd64', - 'gz_sensors-ci-main-jammy-amd64', + 'gz_sensors-ci-gz-sensors8-jammy-amd64', 'gz_sim-ci-main-jammy-amd64', 'gz_tools-ci-gz-tools2-jammy-amd64', - 'gz_transport-ci-main-jammy-amd64', + 'gz_transport-ci-gz-transport13-jammy-amd64', 'gz_utils-ci-gz-utils2-jammy-amd64', 'ignition_cmake-ci-gz-cmake3-homebrew-amd64', 'ignition_common-ci-gz-common5-homebrew-amd64', @@ -317,7 +317,7 @@ gz_collection_jobs = 'ignition_physics-ci-gz-physics6-homebrew-amd64', 'ignition_plugin-ci-gz-plugin2-homebrew-amd64', 'ignition_rendering-ci-gz-rendering8-homebrew-amd64', - 'ignition_sensors-ci-main-homebrew-amd64', + 'ignition_sensors-ci-gz-sensors8-homebrew-amd64', 'ignition_tools-ci-gz-tools2-homebrew-amd64', 'ignition_transport-ci-gz-transport13-homebrew-amd64', 'ignition_utils-ci-gz-utils2-homebrew-amd64', From 0bfea542e0f1cc38cf8a93d5d1a89f6b13804a53 Mon Sep 17 00:00:00 2001 From: Ian Chen Date: Wed, 30 Aug 2023 15:49:01 -0700 Subject: [PATCH 430/540] User stable branches for gz-physics and gz-gui in harmonic (#1000) Signed-off-by: Ian Chen --- jenkins-scripts/dsl/gz-collections.yaml | 4 ++-- jenkins-scripts/dsl/ignition.dsl | 8 +++----- jenkins-scripts/dsl/ignition_collection.dsl | 12 ++++++------ 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/jenkins-scripts/dsl/gz-collections.yaml b/jenkins-scripts/dsl/gz-collections.yaml index 915c95b64..d91fd3c70 100644 --- a/jenkins-scripts/dsl/gz-collections.yaml +++ b/jenkins-scripts/dsl/gz-collections.yaml @@ -93,7 +93,7 @@ collections: - name: gz-gui major_version: 8 repo: - current_branch: main + current_branch: gz-gui8 - name: gz-sensors major_version: 8 repo: @@ -101,7 +101,7 @@ collections: - name: gz-physics major_version: 7 repo: - current_branch: main + current_branch: gz-physics7 - name: gz-sim major_version: 8 repo: diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index 02ef1fd6b..b78a5d6f0 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -64,11 +64,11 @@ ignition_branches = [ 'cmake' : [ '2' ], gz_branches = [ 'cmake' : [ '3' ], 'common' : [ '5' ], 'fuel-tools' : [ '8', '9' ], - 'gui' : [ '7' ], + 'gui' : [ '7', '8' ], 'launch' : [ '6' ], 'math' : [ '7' ], 'msgs' : [ '9', '10'], - 'physics' : [ '6' ], + 'physics' : [ '6', '7' ], 'plugin' : [ '2' ], 'rendering' : [ '7', '8' ], 'sensors' : [ '7', '8' ], @@ -83,9 +83,7 @@ gz_prerelease_branches = [] // DESC: versioned names to generate debbuild jobs for special cases that // don't appear in gz_branches (like nightly builders or 0-debbuild // jobs for the special cases of foo0 packages) -gz_extra_debbuild = [ 'gui8', - 'launch7', - 'physics7', +gz_extra_debbuild = [ 'launch7', 'sim8', 'utils1' // see comment https://github.com/gazebo-tooling/release-tools/pull/431#issuecomment-815099918 ] diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index 0474d1693..5370b4530 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -279,11 +279,11 @@ gz_collection_jobs = 'ign_common-gz-5-win', 'ign_fuel-tools-gz-9-win', 'ign_gazebo-ci-win', - 'ign_gui-ci-win', + 'ign_gui-gz-8-win', 'ign_launch-ci-win', 'ign_math-gz-7-win', 'ign_msgs-gz-10-win', - 'ign_physics-gz-6-win', + 'ign_physics-gz-7-win', 'ign_plugin-gz-2-win', 'ign_rendering-gz-8-win', 'ign_sensors-gz-8-win', @@ -293,11 +293,11 @@ gz_collection_jobs = 'gz_cmake-ci-gz-cmake3-jammy-amd64', 'gz_common-ci-gz-common5-jammy-amd64', 'gz_fuel_tools-ci-gz-fuel-tools9-jammy-amd64', - 'gz_gui-ci-main-jammy-amd64', + 'gz_gui-ci-gz-gui8-jammy-amd64', 'gz_launch-ci-main-jammy-amd64', 'gz_math-ci-gz-math7-jammy-amd64', 'gz_msgs-ci-gz-msgs10-jammy-amd64', - 'gz_physics-ci-gz-physics6-jammy-amd64', + 'gz_physics-ci-gz-physics7-jammy-amd64', 'gz_plugin-ci-gz-plugin2-jammy-amd64', 'gz_rendering-ci-gz-rendering8-jammy-amd64', 'gz_sensors-ci-gz-sensors8-jammy-amd64', @@ -310,11 +310,11 @@ gz_collection_jobs = 'ignition_fuel-tools-ci-gz-fuel-tools9-homebrew-amd64', 'ignition_harmonic-ci-main-homebrew-amd64', 'ignition_gazebo-ci-main-homebrew-amd64', - 'ignition_gui-ci-main-homebrew-amd64', + 'ignition_gui-ci-gz-gui8-homebrew-amd64', 'ignition_launch-ci-main-homebrew-amd64', 'ignition_math-ci-gz-math7-homebrew-amd64', 'ignition_msgs-ci-gz-msgs10-homebrew-amd64', - 'ignition_physics-ci-gz-physics6-homebrew-amd64', + 'ignition_physics-ci-gz-physics7-homebrew-amd64', 'ignition_plugin-ci-gz-plugin2-homebrew-amd64', 'ignition_rendering-ci-gz-rendering8-homebrew-amd64', 'ignition_sensors-ci-gz-sensors8-homebrew-amd64', From aacfb4ec040b578d6797df8b2d3f56f32c9e7521 Mon Sep 17 00:00:00 2001 From: Ian Chen Date: Wed, 30 Aug 2023 23:07:42 -0700 Subject: [PATCH 431/540] Use stable branches for gz-sim and gz-launch in harmonic (#1001) Signed-off-by: Ian Chen --- jenkins-scripts/dsl/gz-collections.yaml | 4 ++-- jenkins-scripts/dsl/ignition.dsl | 8 +++----- jenkins-scripts/dsl/ignition_collection.dsl | 12 ++++++------ 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/jenkins-scripts/dsl/gz-collections.yaml b/jenkins-scripts/dsl/gz-collections.yaml index d91fd3c70..d06eab493 100644 --- a/jenkins-scripts/dsl/gz-collections.yaml +++ b/jenkins-scripts/dsl/gz-collections.yaml @@ -105,11 +105,11 @@ collections: - name: gz-sim major_version: 8 repo: - current_branch: main + current_branch: gz-sim8 - name: gz-launch major_version: 7 repo: - current_branch: main + current_branch: gz-launch7 - name: gz-harmonic major_version: 1 repo: diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index b78a5d6f0..566a601f4 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -65,14 +65,14 @@ gz_branches = [ 'cmake' : [ '3' ], 'common' : [ '5' ], 'fuel-tools' : [ '8', '9' ], 'gui' : [ '7', '8' ], - 'launch' : [ '6' ], + 'launch' : [ '6', '7' ], 'math' : [ '7' ], 'msgs' : [ '9', '10'], 'physics' : [ '6', '7' ], 'plugin' : [ '2' ], 'rendering' : [ '7', '8' ], 'sensors' : [ '7', '8' ], - 'sim' : [ '7' ], + 'sim' : [ '7', '8' ], 'tools' : [ '2' ], 'transport' : [ '12', '13' ], 'utils' : [ '2' ]] @@ -83,9 +83,7 @@ gz_prerelease_branches = [] // DESC: versioned names to generate debbuild jobs for special cases that // don't appear in gz_branches (like nightly builders or 0-debbuild // jobs for the special cases of foo0 packages) -gz_extra_debbuild = [ 'launch7', - 'sim8', - 'utils1' // see comment https://github.com/gazebo-tooling/release-tools/pull/431#issuecomment-815099918 +gz_extra_debbuild = [ 'utils1' // see comment https://github.com/gazebo-tooling/release-tools/pull/431#issuecomment-815099918 ] // DESC: exclude ignition from generate any install testing job gz_no_pkg_yet = [ ] diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index 5370b4530..db6a08f97 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -278,9 +278,9 @@ gz_collection_jobs = 'ign_cmake-gz-3-win', 'ign_common-gz-5-win', 'ign_fuel-tools-gz-9-win', - 'ign_gazebo-ci-win', + 'ign_gazebo-gz-8-win', 'ign_gui-gz-8-win', - 'ign_launch-ci-win', + 'ign_launch-gz-7-win', 'ign_math-gz-7-win', 'ign_msgs-gz-10-win', 'ign_physics-gz-7-win', @@ -294,14 +294,14 @@ gz_collection_jobs = 'gz_common-ci-gz-common5-jammy-amd64', 'gz_fuel_tools-ci-gz-fuel-tools9-jammy-amd64', 'gz_gui-ci-gz-gui8-jammy-amd64', - 'gz_launch-ci-main-jammy-amd64', + 'gz_launch-ci-gz-launch7-jammy-amd64', 'gz_math-ci-gz-math7-jammy-amd64', 'gz_msgs-ci-gz-msgs10-jammy-amd64', 'gz_physics-ci-gz-physics7-jammy-amd64', 'gz_plugin-ci-gz-plugin2-jammy-amd64', 'gz_rendering-ci-gz-rendering8-jammy-amd64', 'gz_sensors-ci-gz-sensors8-jammy-amd64', - 'gz_sim-ci-main-jammy-amd64', + 'gz_sim-ci-gz-sim8-jammy-amd64', 'gz_tools-ci-gz-tools2-jammy-amd64', 'gz_transport-ci-gz-transport13-jammy-amd64', 'gz_utils-ci-gz-utils2-jammy-amd64', @@ -309,9 +309,9 @@ gz_collection_jobs = 'ignition_common-ci-gz-common5-homebrew-amd64', 'ignition_fuel-tools-ci-gz-fuel-tools9-homebrew-amd64', 'ignition_harmonic-ci-main-homebrew-amd64', - 'ignition_gazebo-ci-main-homebrew-amd64', + 'ignition_gazebo-ci-gz-sim8-homebrew-amd64', 'ignition_gui-ci-gz-gui8-homebrew-amd64', - 'ignition_launch-ci-main-homebrew-amd64', + 'ignition_launch-gz-launch7-homebrew-amd64', 'ignition_math-ci-gz-math7-homebrew-amd64', 'ignition_msgs-ci-gz-msgs10-homebrew-amd64', 'ignition_physics-ci-gz-physics7-homebrew-amd64', From f691cbd6dcee969fdb3e2234f197fb4a8671ad74 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 31 Aug 2023 16:32:03 +0200 Subject: [PATCH 432/540] Generalize ros_gz scripts to support other releases than Garden (#999) * Support for GAZEBO_DISTRO in ros_gz bloom --------- Signed-off-by: Jose Luis Rivero --- bloom/ros_gz/Dockerfile | 16 +++++++----- bloom/ros_gz/README.md | 17 +++++++++--- bloom/ros_gz/_check_metadata.bash | 16 +++++------- bloom/ros_gz/bloom_from_special_env.bash | 33 +++++++++++++++++------- 4 files changed, 55 insertions(+), 27 deletions(-) diff --git a/bloom/ros_gz/Dockerfile b/bloom/ros_gz/Dockerfile index 2d98e3a60..620cd9931 100644 --- a/bloom/ros_gz/Dockerfile +++ b/bloom/ros_gz/Dockerfile @@ -1,6 +1,10 @@ ARG ROS_DISTRO=humble +ARG GAZEBO_DISTRO=garden +ARG URL_OSRF_ROSDEP_REPLACE= FROM ros:$ROS_DISTRO-ros-base +ARG GAZEBO_DISTRO +ARG URL_OSRF_ROSDEP_REPLACE ENV LANG C ENV LC_ALL C @@ -13,17 +17,17 @@ RUN apt-get update && apt-get install -y \ wget \ && rm -rf /var/lib/apt/lists/* -# Install osrf-rosdep for Garden +# Install osrf-rosdep for Gazebo distributions not in ROS RUN sudo bash -c \ 'wget https://raw.githubusercontent.com/osrf/osrf-rosdep/master/gz/00-gazebo.list -O /etc/ros/rosdep/sources.list.d/00-gazebo.list' -# Replace Fortress with Garden to make bloom to work -# since it does not support arbitrary environemnt variables +# Replace url +# since it does not support arbitrary environment variables RUN sudo bash -c \ - 'wget https://raw.githubusercontent.com/osrf/osrf-rosdep/master/gz/replace_fortress_with_garden/00-replace-gz-fortress-with-garden.list -O /etc/ros/rosdep/sources.list.d/00-replace-gz-fortress-with-garden.list' + "wget $URL_OSRF_ROSDEP_REPLACE -O /etc/ros/rosdep/sources.list.d/00-replace-gz.list" # Not for using bloom but useful for rosdep calls -ENV GZ_VERSION=garden -ENV IGNITION_VERSION=garden +ENV GZ_VERSION=$GAZEBO_DISTRO +ENV IGNITION_VERSION=$GAZEBO_DISTRO WORKDIR /tmp COPY _check_metadata.bash . diff --git a/bloom/ros_gz/README.md b/bloom/ros_gz/README.md index 393aa4c2d..466b17005 100644 --- a/bloom/ros_gz/README.md +++ b/bloom/ros_gz/README.md @@ -7,7 +7,7 @@ * Create a custom track in tracks.yaml 3. Run a new release * Prerequisites - * Bloom a new release + * Bloom a new release ## 1. Background @@ -17,6 +17,7 @@ found in the [REP-3](http://www.ros.org/reps/rep-2000.html). Some examples: * ROS 2 Foxy: Citadel * ROS 2 Humble: Fortress + * ROS 2 Iron: Fortress Some use cases require the use of alternative combinations of ROS and Gazebo versions. The `ros_gz` code is usually prepared to be compatible with @@ -34,7 +35,7 @@ version the version of `ros_gz` released will be the latest one existing in the official `gbp -release repository`. The version would be the same but the release number will start on 1000. -## 2. Initial setup +## 2. Initial setup To release a modified version of `ros_gz` which supports a different major version of gazebo, before running bloom some actions need to be taken: @@ -46,6 +47,10 @@ For a new official wrappers the notation used below correspond to `ros_ign-relea 1. Fork (manually or using gh) current gbp repository: https://github.com/ros2-gbp/ros_ign-release +Note: the same -release repository can obviously hosts multiple ROS 2 releases but +can not have multipe Gazebo releases since the bloom templates are set to just one +package name. + 1. Clone the new repo, go to the directory and run rename-gazebo-ros-pkgs.bash - Usage: `$ rename-ros_gz-pkgs.bash ` - Example: `$ rename-ros_gz-pkgs.bash garden humble` @@ -85,7 +90,13 @@ this, there is a `Dockerfile` that provides the needed modifications and a scrip that encapsulates the bloom arguments to be passed and the use of this enviroment. ``` -./bloom_from_special_env.bash +./bloom_from_special_env.bash + +i.e: release ros_gz on ROS 2 Humble replacing Gazebo Fortress by Gazebo Garden +./bloom_from_special_env.bash \ + humble \ + garden \ + https://raw.githubusercontent.com/osrf/osrf-rosdep/master/gz/replace_fortress_with_garden/00-replace-gz-fortress-with-garden.list ``` The script will create the docker enviroment with the rosdep modifications needed diff --git a/bloom/ros_gz/_check_metadata.bash b/bloom/ros_gz/_check_metadata.bash index e4bc4eb09..da06ad3f9 100755 --- a/bloom/ros_gz/_check_metadata.bash +++ b/bloom/ros_gz/_check_metadata.bash @@ -1,17 +1,15 @@ -#!/bin/bash - -set -e - +#!/bin/bash -e if [[ $# -lt 2 ]]; then - echo "Usage: $0 " + echo "Usage: $0 " exit 1 fi RELEASE_REPO_URL=${1} ROS_DISTRO=${2} +GAZEBO_DISTRO=${3} -echo " * Checking that ${RELEASE_REPO_URL} has a correct gzgarden metadata for ROS ${ROS_DISTRO}" +echo " * Checking that ${RELEASE_REPO_URL} has a correct gz${GAZEBO_DISTRO} metadata for ROS ${ROS_DISTRO}" TMP_DIR=$(mktemp -d) git clone -q "${RELEASE_REPO_URL}" "${TMP_DIR}" @@ -20,8 +18,8 @@ DEBIAN_LATEST_TAG=$(git for-each-ref --sort=creatordate --format '%(refname)' re grep "debian/ros-${ROS_DISTRO}" | \ tail -1 | sed 's:refs/tags/::') git checkout -q "${DEBIAN_LATEST_TAG}" -if ! dpkg-parsechangelog -SSource 2> /dev/null | grep -q 'gzgarden'; then - echo " !! Latest changelog entry does not have gzgarden name" +if ! dpkg-parsechangelog -SSource 2> /dev/null | grep -q "gz${GAZEBO_DISTRO}"; then + echo " !! Latest changelog entry does not have gz${GAZEBO_DISTRO} name" echo " !! in the repo tag '${DEBIAN_LATEST_TAG}' for repo '${RELEASE_REPO_URL}'" echo -n " !! Latest changelog is:" dpkg-parsechangelog -SSource 2> /dev/null || exit @@ -30,4 +28,4 @@ if ! dpkg-parsechangelog -SSource 2> /dev/null | grep -q 'gzgarden'; then fi popd >/dev/null || exit -echo " + Changelog has correct gzgarden metadata for ${DEBIAN_LATEST_TAG}" +echo " + Changelog has correct gz${GAZEBO_DISTRO} metadata for ${DEBIAN_LATEST_TAG}" diff --git a/bloom/ros_gz/bloom_from_special_env.bash b/bloom/ros_gz/bloom_from_special_env.bash index d60281a9d..63be8c767 100755 --- a/bloom/ros_gz/bloom_from_special_env.bash +++ b/bloom/ros_gz/bloom_from_special_env.bash @@ -1,14 +1,18 @@ -#!/bin/bash +#!/bin/bash -ex -if [[ $# -lt 1 ]]; then - echo "Usage: $0 " +if [[ $# -lt 3 ]]; then + echo "Usage: $0 " + echo "Example:" + echo " $0 humble garden https://raw.githubusercontent.com/osrf/osrf-rosdep/master/gz/replace_fortress_with_garden/00-replace-gz-fortress-with-garden.list" exit 1 fi ROS_DISTRO=${1} +GAZEBO_DISTRO=${2} +URL_OSRF_ROSDEP_REPLACE=${3} + # Might be used with testing proposes to change gbp-release repo -GBP_ORG=${GBP_ORG:-gazebo-release} -RELEASE_REPO_URL=https://github.com/${GBP_ORG}/ros_ign-release +RELEASE_REPO_URL=${RELEASE_REPO_URL:-https://github.com/gazebo-release/ros_ign-release} if ! command rocker --version &> /dev/null then @@ -16,21 +20,32 @@ then exit 1 fi +RELEASE_REPO_TRACK=${ROS_DISTRO}_gz${GAZEBO_DISTRO} +RAW_RELEASE_REPO=${RELEASE_REPO_URL/github.com/raw.githubusercontent.com} + +if ! curl --silent "${RAW_RELEASE_REPO}/master/tracks.yaml" | grep "${RELEASE_REPO_TRACK}" > /dev/null 2> /dev/null; then + echo "Did not found ${RELEASE_REPO_TRACK} track in ${RAW_RELEASE_REPO}" + exit 1 +fi + # TODO: update URL for the release repo BLOOM_CMD="/usr/bin/bloom-release --no-pull-request \ --rosdistro ${ROS_DISTRO}\ - --track ${ROS_DISTRO}_gzgarden \ + --track ${RELEASE_REPO_TRACK} \ --override-release-repository-url \ ${RELEASE_REPO_URL} ros_gz" -TAG_NAME=${TAG_NAME:-ros_gz-${ROS_DISTRO}-fortress-garden-release} +TAG_NAME=${TAG_NAME:-ros_gz-${ROS_DISTRO}-${GAZEBO_DISTRO}-release} echo " * Build the docker release environment" -docker build . -t "${TAG_NAME}" --build-arg ROS_DISTRO="${ROS_DISTRO}" +docker build . -t "${TAG_NAME}" \ + --build-arg ROS_DISTRO="${ROS_DISTRO}" \ + --build-arg GAZEBO_DISTRO="${GAZEBO_DISTRO}" \ + --build-arg URL_OSRF_ROSDEP_REPLACE="${URL_OSRF_ROSDEP_REPLACE}" echo " * Using rocker to enter in the release environment" rocker --home --user "${TAG_NAME}" "${BLOOM_CMD}" echo " * Exit the docker release environment" echo " * Restoring the rosdep cache in the user" rosdep update echo " * Running safety check in generated metadata" -docker run "${TAG_NAME}" /tmp/_check_metadata.bash "${RELEASE_REPO_URL}" "${ROS_DISTRO}" +docker run "${TAG_NAME}" /tmp/_check_metadata.bash "${RELEASE_REPO_URL}" "${ROS_DISTRO}" "${GAZEBO_DISTRO}" From 78cfa9ed24ba16519f18c535b605ed8da6838f74 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 1 Sep 2023 15:35:46 +0200 Subject: [PATCH 433/540] Default timeout value in _gazebo_utils (#1002) Signed-off-by: Jose Luis Rivero --- jenkins-scripts/docker/lib/_gazebo_utils.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/jenkins-scripts/docker/lib/_gazebo_utils.sh b/jenkins-scripts/docker/lib/_gazebo_utils.sh index 2f0bf0168..c1cea83f8 100644 --- a/jenkins-scripts/docker/lib/_gazebo_utils.sh +++ b/jenkins-scripts/docker/lib/_gazebo_utils.sh @@ -84,12 +84,14 @@ cd build-deps # nightlies uploads. update_done=false seconds_waiting=0 +timeout=\${timeout:-0} while (! \$update_done); do sudo DEBIAN_FRONTEND=noninteractive mk-build-deps \ -r -i ../debian/control \ --tool 'apt-get --yes -o Debug::pkgProblemResolver=yes -o Debug::BuildDeps=yes' \ && break sleep 60 && seconds_waiting=\$((seconds_waiting+60)) + apt-get update [ \$seconds_waiting -ge \$timeout ] && exit 1 done cd .. From e8628d70299d6401745c251f39aedb0697fa9489 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Mon, 11 Sep 2023 18:23:34 +0200 Subject: [PATCH 434/540] Make repository_uploader to support S3_FILES_TO_UPLOAD (#1006) Signed-off-by: Jose Luis Rivero --- jenkins-scripts/lib/repository_uploader.bash | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/jenkins-scripts/lib/repository_uploader.bash b/jenkins-scripts/lib/repository_uploader.bash index 2b15375b5..79e01c224 100644 --- a/jenkins-scripts/lib/repository_uploader.bash +++ b/jenkins-scripts/lib/repository_uploader.bash @@ -131,8 +131,12 @@ case ${UPLOAD_TO_REPO} in ;; esac -# .zip | (mostly) windows packages -for pkg in `ls $pkgs_path/*.zip`; do + +# S3_FILES_TO_UPLOAD can contain a list of filenames relative to $pkgs_path that will be uploaded to S3 +# together with any .zip (mostly old windows packages) if they exists +S3_FILES_TO_UPLOAD="${S3_FILES_TO_UPLOAD} $(find "$pkgs_path" -type f -name '*.zip' -printf '%f\n' || true)" + +for pkg in ${S3_FILES_TO_UPLOAD}; do # S3_UPLOAD_PATH should be send by the upstream job if [[ -z ${S3_UPLOAD_PATH} ]]; then echo "S3_UPLOAD_PATH was not defined. Not uploading" @@ -140,7 +144,7 @@ for pkg in `ls $pkgs_path/*.zip`; do fi # Seems important to upload the path with a final slash - S3_upload ${pkg} "${S3_UPLOAD_PATH}" + S3_upload "${pkgs_path}/${pkg}" "${S3_UPLOAD_PATH}" done # .bottle | brew binaries From 791800677f670e5dbd41671313599c21405b29fe Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 12 Sep 2023 16:09:03 +0200 Subject: [PATCH 435/540] Make DSL generation to store jobs generated in a log file (#985) * Generate a log of the jobs generated * Run logging on github actions --- Signed-off-by: Jose Luis Rivero --- .github/workflows/ci.yaml | 19 +++++++++++----- jenkins-scripts/dsl/gazebo_libs.dsl | 35 ++++++++++++++++++++++++----- 2 files changed, 43 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 6f41679b5..41c07b533 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -50,25 +50,28 @@ jobs: sudo mkdir -p /var/lib/jenkins/ && sudo touch /var/lib/jenkins/remote_token sudo chown -R ${USER} /var/lib/jenkins cd jenkins-scripts/dsl - java -jar tools/jobdsl.jar *.dsl + WRITE_JOB_LOG=1 java -jar tools/jobdsl.jar *.dsl # export files for later diff - mkdir /tmp/pr_xml_configuration + mkdir /tmp/pr_xml_configuration && mkdir /tmp/pr_log_generated/ mv *.xml /tmp/pr_xml_configuration/ - - name: Geneate master DSL files + mv *.txt /tmp/pr_log_generated/ + - name: Generate master DSL files if: steps.dsl_check.outputs.run_job == 'true' run: | git clean -f -e jobdsl.jar git checkout master cd jenkins-scripts/dsl - java -jar tools/jobdsl.jar *.dsl - mkdir /tmp/current_xml_configuration + WRITE_JOB_LOG=1 java -jar tools/jobdsl.jar *.dsl + mkdir /tmp/current_xml_configuration && mkdir /tmp/current_log_generated/ mv *.xml /tmp/current_xml_configuration/ + mv *.txt /tmp/current_log_generated/ || true - name: Generating diffs if: steps.dsl_check.outputs.run_job == 'true' run: | # somehow the Jenkins views changed the portlet_ id on every run. diff -qr -I '.*dashboard_portlet_.*.*' /tmp/current_xml_configuration /tmp/pr_xml_configuration > /tmp/xml_config_files_changed.diff || true diff -ur -I '.*dashboard_portlet_.*.*' /tmp/current_xml_configuration /tmp/pr_xml_configuration > /tmp/xml_config_content_changed.diff || true + diff -ur /tmp/current_log_generated /tmp/pr_log_generated > /tmp/log_content_changed.diff || true - name: Archive files changes if: steps.dsl_check.outputs.run_job == 'true' uses: actions/upload-artifact@v3 @@ -81,3 +84,9 @@ jobs: with: name: xml_config_content_changed path: /tmp/xml_config_content_changed.diff + - name: Archive log changes + if: steps.dsl_check.outputs.run_job == 'true' + uses: actions/upload-artifact@v3 + with: + name: log_content_changed + path: /tmp/log_content_changed.diff diff --git a/jenkins-scripts/dsl/gazebo_libs.dsl b/jenkins-scripts/dsl/gazebo_libs.dsl index 82c6b0e2b..ce6181ab4 100644 --- a/jenkins-scripts/dsl/gazebo_libs.dsl +++ b/jenkins-scripts/dsl/gazebo_libs.dsl @@ -9,6 +9,10 @@ GLOBAL_SHELL_CMD='' // GZ COLLECTIONS ENABLE_CPPCHECK = true +def WRITE_JOB_LOG = System.getenv('WRITE_JOB_LOG') ?: false +logging_list = [:] +logging_list['branch_ci'] = [] + // Jenkins needs the relative path to work and locally the simulation is done // using a symlink file = readFileFromWorkspace("scripts/jenkins-scripts/dsl/gz-collections.yaml") @@ -57,12 +61,16 @@ boolean is_testing_enabled(lib_name, ci_config) * avoiding to parse them several times. * * ci_configs_by_lib index structure: - * lib_name : [ ci_config_name : [ branches ] ] + * lib_name : [ ci_config_name : [ .branch .collection ] ] * * The index main keys are the lib names (i.e: gz-cmake) and associated them * another map of CI configuration names supported as keys (i.e: jammy) with the - * list of associated branches for that CI configuration (i.e [gz-cmake3, gz-cmake4]) - * as values. + * list of associated items composed by a map: branch (and collection) that CI configuration + * (i.e [[branch:gz-cmake3, collection: harmonic], [branch: gz-cmake3, collection: garden]) + * as values. In a graphic; + * + * index[gz-cmake][jammy] -> [ branch: gz-cmake3, collection: garden , + * branch: gz-cmake3, collection: harmonic] */ void generate_ciconfigs_by_lib(config, configs_per_lib_index) { @@ -76,7 +84,7 @@ void generate_ciconfigs_by_lib(config, configs_per_lib_index) def branch = lib.repo.current_branch collection.ci.configs.each { config_name -> configs_per_lib_index["$libName"]["${config_name}"] = configs_per_lib_index["$libName"]["${config_name}"]?: [] - configs_per_lib_index["$libName"]["${config_name}"].contains(branch) ?: configs_per_lib_index["$libName"]["${config_name}"] << branch + configs_per_lib_index["$libName"]["${config_name}"].contains(branch) ?: configs_per_lib_index["$libName"]["${config_name}"] << [branch: branch, collection: collection.name] } } } @@ -126,7 +134,8 @@ configs_per_lib_index.each { lib_name, lib_configs -> lib_configs.each { ci_configs -> def config_name = ci_configs.getKey() def ci_config = gz_collections_yaml.ci_configs.find{ it.name == config_name } - def branch_names = ci_configs.getValue() + def branches_with_collections = ci_configs.getValue() + def branch_names = branches_with_collections.collect { it.branch }.unique() if (ci_config.exclude.contains(lib_name)) return assert(lib_name) @@ -171,7 +180,8 @@ configs_per_lib_index.each { lib_name, lib_configs -> } // end of ci_any_job // CI branch jobs (-ci-$branch-) (pulling check every 5 minutes) - branch_names.each { branch_name -> + branches_with_collections.each { branch_and_collection -> + branch_name = branch_and_collection.branch def gz_ci_job = job("${gz_job_name_prefix}-ci-${branch_name}-${distro}-${arch}") generate_ci_job(gz_ci_job, lib_name, branch_name, ci_config) gz_ci_job.with @@ -180,7 +190,20 @@ configs_per_lib_index.each { lib_name, lib_configs -> scm('@daily') } } + + logging_list['branch_ci'].add( + [collection: branch_and_collection.collection, + job_name: gz_ci_job.name]) } // end_of_branch } //en of lib_configs } // end of lib + +if (WRITE_JOB_LOG) { + File log_file = new File("jobs.txt") + log_file.withWriter{ file_writer -> + logging_list.each { log_type, items -> + items.each {file_writer.println "${log_type} ${it.collection} ${it.job_name}"} + } + } +} From 8ca053b830b16ebd678f07bb20a722f9c2d1d565 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 15 Sep 2023 17:15:49 +0200 Subject: [PATCH 436/540] Move sdformat14 to stable in sdformat.dsl (#1011) Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/sdformat.dsl | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/jenkins-scripts/dsl/sdformat.dsl b/jenkins-scripts/dsl/sdformat.dsl index 30009b06b..27fa20683 100644 --- a/jenkins-scripts/dsl/sdformat.dsl +++ b/jenkins-scripts/dsl/sdformat.dsl @@ -1,10 +1,10 @@ import _configs_.* import javaposse.jobdsl.dsl.Job -def sdformat_supported_versions = [ 'sdformat6' , 'sdformat9', 'sdformat12', 'sdformat13' ] +def sdformat_supported_versions = [ 'sdformat6' , 'sdformat9', 'sdformat12', 'sdformat13', 'sdformat14' ] def sdformat_gz11_versions = [ 'sdformat9', 'sdformat12', 'sdformat13', 'main' ] // nightly and prereleases -def extra_sdformat_debbuilder = ['sdformat14'] +def extra_sdformat_debbuilder = [] // Main platform using for quick CI def ci_distro = Globals.get_ci_distro() @@ -89,7 +89,6 @@ ci_distro.each { distro -> // -------------------------------------------------------------- // 2. Create the any job String sdf_repo = "gazebosim/sdformat" - def sdformat_ci_any_job = job(ci_build_any_job_name_linux) OSRFLinuxCompilationAnyGitHub.create(sdformat_ci_any_job, sdf_repo) sdformat_ci_any_job.with From 2581ee00e7fdd805d325eeb3bf4a925d74ef4db6 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 15 Sep 2023 17:27:46 +0200 Subject: [PATCH 437/540] Do use ign instead of gz for testing ROS imports on Fortress (#1009) Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/ignition_collection.dsl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index db6a08f97..e140b089d 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -432,6 +432,9 @@ gz_collections_yaml.collections.each { collection -> export ROS_BOOTSTRAP=true # needed for arm64 machines and other arch tests export ENABLE_GZ_SIM_RUNTIME_TEST=false + if [[ ${gz_collection_name} == 'citadel' || ${gz_collection_name} == 'fortress' ]]; then + export GZ_SIM_RUNTIME_TEST_USE_IGN=true + fi if [[ \${JENKINS_NODE_TAG} == 'gpu-reliable' ]]; then export ENABLE_GZ_SIM_RUNTIME_TEST=true fi From 04ad161d7a42f70706f9f388d0e731dd39af42d8 Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Mon, 18 Sep 2023 03:17:06 -0700 Subject: [PATCH 438/540] homebrew_formula_pullrequest.bash: sanitize ~ (#1007) Sanitize ~ characters from the brew version string. Signed-off-by: Steve Peters --- .../lib/homebrew_formula_pullrequest.bash | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/jenkins-scripts/lib/homebrew_formula_pullrequest.bash b/jenkins-scripts/lib/homebrew_formula_pullrequest.bash index e353f1f4a..0256f8eed 100644 --- a/jenkins-scripts/lib/homebrew_formula_pullrequest.bash +++ b/jenkins-scripts/lib/homebrew_formula_pullrequest.bash @@ -26,6 +26,10 @@ if [ -z "${SOURCE_TARBALL_SHA}" ]; then fi echo '# END SECTION' +# brew does not support ~ in VERSION string, so convert it to - +VERSION_SANITIZED=$(echo "${VERSION}" | tr '~' '-') +echo Sanitizing version string ${VERSION} to ${VERSION_SANITIZED} + PULL_REQUEST_HEAD_REPO=git@github.com:osrfbuild/homebrew-simulation.git . ${SCRIPT_LIBDIR}/_homebrew_github_setup.bash @@ -48,7 +52,7 @@ echo VERSION_LINE=$(awk \ "/^ version ['\"]/ {print FNR}" ${FORMULA_PATH} | head -1) # check if version can be correctly auto-detected from url -if ${BREW} ruby -e "exit Version.parse(\"${SOURCE_TARBALL_URI}\").to_s == \"${VERSION}\"" +if ${BREW} ruby -e "exit Version.parse(\"${SOURCE_TARBALL_URI}\").to_s == \"${VERSION_SANITIZED}\"" then echo Version can be correctly auto-detected from URL if [ -n "${VERSION_LINE}" ]; then @@ -59,12 +63,12 @@ else if [ -z "${VERSION_LINE}" ]; then # Need to insert explicit version tag after url echo Adding explicit version tag after URL - sed -i -e "${URI_LINE}a\ version \"${VERSION}\"" ${FORMULA_PATH} + sed -i -e "${URI_LINE}a\ version \"${VERSION_SANITIZED}\"" ${FORMULA_PATH} else echo Changing version to - echo ${VERSION} + echo ${VERSION_SANITIZED} echo on line number ${VERSION_LINE} - sed -i -e "${VERSION_LINE}c\ version \"${VERSION}\"" ${FORMULA_PATH} + sed -i -e "${VERSION_LINE}c\ version \"${VERSION_SANITIZED}\"" ${FORMULA_PATH} fi fi @@ -104,6 +108,6 @@ if [ "$FORMULA_REVISION" -gt 0 ]; then fi # create branch with name and sanitized version string -PULL_REQUEST_BRANCH="${PACKAGE_ALIAS}_`echo ${VERSION} | tr ' ~:^?*[' '_'`" +PULL_REQUEST_BRANCH="${PACKAGE_ALIAS}_`echo ${VERSION_SANITIZED} | tr ' ~:^?*[' '_'`" . ${SCRIPT_LIBDIR}/_homebrew_github_commit.bash From 87481352f1df1c65c3946f19b5edcb7b42c255f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Crist=C3=B3bal=20Arroyo?= Date: Tue, 19 Sep 2023 08:26:53 -0500 Subject: [PATCH 439/540] Update _outdated_job_runner jobs (#981) * Update jobs Signed-off-by: Crola1702 * Update pseudo-documentation Signed-off-by: Crola1702 * Update tracked jobs Signed-off-by: Crola1702 * Fix sdf13 pin Signed-off-by: Crola1702 --------- Signed-off-by: Crola1702 --- .../tools/outdated-job-runner.groovy | 68 ++++++++++--------- 1 file changed, 37 insertions(+), 31 deletions(-) diff --git a/jenkins-scripts/tools/outdated-job-runner.groovy b/jenkins-scripts/tools/outdated-job-runner.groovy index bf030d162..bf9fc5e16 100644 --- a/jenkins-scripts/tools/outdated-job-runner.groovy +++ b/jenkins-scripts/tools/outdated-job-runner.groovy @@ -64,61 +64,75 @@ def runJobsInAvailableNodes(LinkedHashMap outdatedJobs) { } // TODO (Cristobal): This is a hardcoded list of jobs that we're tracking. We should get this list dynamically -// Currently is the result of https://github.com/osrf/buildfarmer/blob/main/common.py#L51 minus debbuilders +// Currently is the result of https://github.com/osrf/buildfarmer/blob/main/scripts/gazebo_non_nightly_jobs.py def trackedJobsList = [ + 'gz_cmake-ci-gz-cmake3-jammy-amd64', + 'gz_common-ci-gz-common5-jammy-amd64', + 'gz_fuel_tools-ci-gz-fuel-tools9-jammy-amd64', + 'gz_gui-ci-gz-gui8-jammy-amd64', + 'gz_launch-ci-gz-launch7-jammy-amd64', + 'gz_math-ci-gz-math7-jammy-amd64', + 'gz_msgs-ci-gz-msgs10-jammy-amd64', + 'gz_physics-ci-gz-physics7-jammy-amd64', + 'gz_plugin-ci-gz-plugin2-jammy-amd64', + 'gz_rendering-ci-gz-rendering8-jammy-amd64', + 'gz_sensors-ci-gz-sensors8-jammy-amd64', + 'gz_sim-ci-gz-sim8-jammy-amd64', + 'gz_tools-ci-gz-tools2-jammy-amd64', + 'gz_transport-ci-gz-transport13-jammy-amd64', + 'gz_utils-ci-gz-utils2-jammy-amd64', 'ign_cmake-gz-3-win', 'ign_cmake-ign-2-win', 'ign_common-gz-5-win', 'ign_common-ign-3-win', 'ign_common-ign-4-win', - 'ign_fuel-tools-ci-win', 'ign_fuel-tools-gz-8-win', + 'ign_fuel-tools-gz-9-win', 'ign_fuel-tools-ign-4-win', 'ign_fuel-tools-ign-7-win', - 'ign_gazebo-ci-win', 'ign_gazebo-gz-7-win', + 'ign_gazebo-gz-8-win', 'ign_gazebo-ign-3-win', 'ign_gazebo-ign-6-win', - 'ign_gui-ci-win', 'ign_gui-gz-7-win', + 'ign_gui-gz-8-win', 'ign_gui-ign-3-win', 'ign_gui-ign-6-win', - 'ign_launch-ci-win', 'ign_launch-gz-6-win', + 'ign_launch-gz-7-win', 'ign_launch-ign-2-win', 'ign_launch-ign-5-win', 'ign_math-gz-7-win', 'ign_math-ign-6-win', - 'ign_msgs-ci-win', + 'ign_msgs-gz-10-win', 'ign_msgs-gz-9-win', 'ign_msgs-ign-5-win', 'ign_msgs-ign-8-win', 'ign_physics-gz-6-win', + 'ign_physics-gz-7-win', 'ign_physics-ign-2-win', 'ign_physics-ign-5-win', 'ign_plugin-gz-2-win', 'ign_plugin-ign-1-win', - 'ign_rendering-ci-win', 'ign_rendering-gz-7-win', + 'ign_rendering-gz-8-win', 'ign_rendering-ign-3-win', 'ign_rendering-ign-6-win', - 'ign_sensors-ci-win', 'ign_sensors-gz-7-win', + 'ign_sensors-gz-8-win', 'ign_sensors-ign-3-win', 'ign_sensors-ign-6-win', 'ign_tools-gz-2-win', 'ign_tools-ign-1-win', - 'ign_transport-ci-win', 'ign_transport-gz-12-win', + 'ign_transport-gz-13-win', 'ign_transport-ign-11-win', 'ign_transport-ign-8-win', 'ign_utils-gz-2-win', 'ign_utils-ign-1-win', - 'ignition_cmake-ci-gz-cmake3-focal-amd64', 'ignition_cmake-ci-gz-cmake3-homebrew-amd64', 'ignition_cmake-ci-ign-cmake2-focal-amd64', 'ignition_cmake-ci-ign-cmake2-homebrew-amd64', - 'ignition_common-ci-gz-common5-focal-amd64', 'ignition_common-ci-gz-common5-homebrew-amd64', 'ignition_common-ci-ign-common3-focal-amd64', 'ignition_common-ci-ign-common3-homebrew-amd64', @@ -126,98 +140,90 @@ def trackedJobsList = [ 'ignition_common-ci-ign-common4-homebrew-amd64', 'ignition_fuel-tools-ci-gz-fuel-tools8-focal-amd64', 'ignition_fuel-tools-ci-gz-fuel-tools8-homebrew-amd64', + 'ignition_fuel-tools-ci-gz-fuel-tools9-homebrew-amd64', 'ignition_fuel-tools-ci-ign-fuel-tools4-focal-amd64', 'ignition_fuel-tools-ci-ign-fuel-tools4-homebrew-amd64', 'ignition_fuel-tools-ci-ign-fuel-tools7-focal-amd64', 'ignition_fuel-tools-ci-ign-fuel-tools7-homebrew-amd64', - 'ignition_fuel-tools-ci-main-focal-amd64', - 'ignition_fuel-tools-ci-main-homebrew-amd64', 'ignition_gazebo-ci-gz-sim7-focal-amd64', 'ignition_gazebo-ci-gz-sim7-homebrew-amd64', + 'ignition_gazebo-ci-gz-sim8-homebrew-amd64', 'ignition_gazebo-ci-ign-gazebo3-focal-amd64', 'ignition_gazebo-ci-ign-gazebo3-homebrew-amd64', 'ignition_gazebo-ci-ign-gazebo6-focal-amd64', 'ignition_gazebo-ci-ign-gazebo6-homebrew-amd64', - 'ignition_gazebo-ci-main-focal-amd64', - 'ignition_gazebo-ci-main-homebrew-amd64', 'ignition_gui-ci-gz-gui7-focal-amd64', 'ignition_gui-ci-gz-gui7-homebrew-amd64', + 'ignition_gui-ci-gz-gui8-homebrew-amd64', 'ignition_gui-ci-ign-gui3-focal-amd64', 'ignition_gui-ci-ign-gui3-homebrew-amd64', 'ignition_gui-ci-ign-gui6-focal-amd64', 'ignition_gui-ci-ign-gui6-homebrew-amd64', - 'ignition_gui-ci-main-focal-amd64', - 'ignition_gui-ci-main-homebrew-amd64', 'ignition_launch-ci-gz-launch6-focal-amd64', 'ignition_launch-ci-gz-launch6-homebrew-amd64', + 'ignition_launch-ci-gz-launch7-homebrew-amd64', 'ignition_launch-ci-ign-launch2-focal-amd64', 'ignition_launch-ci-ign-launch2-homebrew-amd64', 'ignition_launch-ci-ign-launch5-focal-amd64', 'ignition_launch-ci-ign-launch5-homebrew-amd64', - 'ignition_launch-ci-main-focal-amd64', - 'ignition_launch-ci-main-homebrew-amd64', - 'ignition_math-ci-gz-math7-focal-amd64', 'ignition_math-ci-gz-math7-homebrew-amd64', 'ignition_math-ci-ign-math6-focal-amd64', 'ignition_math-ci-ign-math6-homebrew-amd64', + 'ignition_msgs-ci-gz-msgs10-homebrew-amd64', 'ignition_msgs-ci-gz-msgs9-focal-amd64', 'ignition_msgs-ci-gz-msgs9-homebrew-amd64', 'ignition_msgs-ci-ign-msgs5-focal-amd64', 'ignition_msgs-ci-ign-msgs5-homebrew-amd64', 'ignition_msgs-ci-ign-msgs8-focal-amd64', 'ignition_msgs-ci-ign-msgs8-homebrew-amd64', - 'ignition_msgs-ci-main-focal-amd64', - 'ignition_msgs-ci-main-homebrew-amd64', 'ignition_physics-ci-gz-physics6-focal-amd64', 'ignition_physics-ci-gz-physics6-homebrew-amd64', + 'ignition_physics-ci-gz-physics7-homebrew-amd64', 'ignition_physics-ci-ign-physics2-focal-amd64', 'ignition_physics-ci-ign-physics2-homebrew-amd64', 'ignition_physics-ci-ign-physics5-focal-amd64', 'ignition_physics-ci-ign-physics5-homebrew-amd64', - 'ignition_plugin-ci-gz-plugin2-focal-amd64', 'ignition_plugin-ci-gz-plugin2-homebrew-amd64', 'ignition_plugin-ci-ign-plugin1-focal-amd64', 'ignition_plugin-ci-ign-plugin1-homebrew-amd64', 'ignition_rendering-ci-gz-rendering7-focal-amd64', 'ignition_rendering-ci-gz-rendering7-homebrew-amd64', + 'ignition_rendering-ci-gz-rendering8-homebrew-amd64', 'ignition_rendering-ci-ign-rendering3-focal-amd64', 'ignition_rendering-ci-ign-rendering3-homebrew-amd64', 'ignition_rendering-ci-ign-rendering6-focal-amd64', 'ignition_rendering-ci-ign-rendering6-homebrew-amd64', - 'ignition_rendering-ci-main-focal-amd64', - 'ignition_rendering-ci-main-homebrew-amd64', 'ignition_sensors-ci-gz-sensors7-focal-amd64', 'ignition_sensors-ci-gz-sensors7-homebrew-amd64', + 'ignition_sensors-ci-gz-sensors8-homebrew-amd64', 'ignition_sensors-ci-ign-sensors3-focal-amd64', 'ignition_sensors-ci-ign-sensors3-homebrew-amd64', 'ignition_sensors-ci-ign-sensors6-focal-amd64', 'ignition_sensors-ci-ign-sensors6-homebrew-amd64', - 'ignition_sensors-ci-main-focal-amd64', - 'ignition_sensors-ci-main-homebrew-amd64', - 'ignition_tools-ci-gz-tools2-focal-amd64', 'ignition_tools-ci-gz-tools2-homebrew-amd64', 'ignition_tools-ci-ign-tools1-focal-amd64', 'ignition_tools-ci-ign-tools1-homebrew-amd64', 'ignition_transport-ci-gz-transport12-focal-amd64', 'ignition_transport-ci-gz-transport12-homebrew-amd64', + 'ignition_transport-ci-gz-transport13-homebrew-amd64', 'ignition_transport-ci-ign-transport11-focal-amd64', 'ignition_transport-ci-ign-transport11-homebrew-amd64', 'ignition_transport-ci-ign-transport8-focal-amd64', 'ignition_transport-ci-ign-transport8-homebrew-amd64', - 'ignition_transport-ci-main-focal-amd64', - 'ignition_transport-ci-main-homebrew-amd64', - 'ignition_utils-ci-gz-utils2-focal-amd64', 'ignition_utils-ci-gz-utils2-homebrew-amd64', 'ignition_utils-ci-ign-utils1-focal-amd64', 'ignition_utils-ci-ign-utils1-homebrew-amd64', + 'sdformat-ci-sdf14-jammy-amd64', 'sdformat-ci-sdformat12-focal-amd64', 'sdformat-ci-sdformat12-homebrew-amd64', 'sdformat-ci-sdformat13-focal-amd64', 'sdformat-ci-sdformat13-homebrew-amd64', + 'sdformat-ci-sdformat14-homebrew-amd64', 'sdformat-ci-sdformat9-focal-amd64', 'sdformat-ci-sdformat9-homebrew-amd64', 'sdformat-sdf-12-win', 'sdformat-sdf-13-win', + 'sdformat-sdf-14-win', 'sdformat-sdf-9-win', ]; def trackedJobs = new LinkedHashSet(trackedJobsList); From 8d891d89d3dcabd24d89e6d4e218b3e53a1455d5 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 20 Sep 2023 16:02:58 +0200 Subject: [PATCH 440/540] Fixes for reprepro-importer (#1015) * Do not use sudo * Do not hardcode GNUPGHOME Signed-off-by: Jose Luis Rivero --- jenkins-scripts/docker/reprepro_updater.bash | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/jenkins-scripts/docker/reprepro_updater.bash b/jenkins-scripts/docker/reprepro_updater.bash index 580070b9a..3246350fd 100644 --- a/jenkins-scripts/docker/reprepro_updater.bash +++ b/jenkins-scripts/docker/reprepro_updater.bash @@ -40,13 +40,12 @@ git clone "${REPREPRO_GIT_REPO}" -b "${REPREPRO_GIT_BRANCH}" "${REPREPRO_REPO_PA echo '# END SECTION' export PYTHONPATH="${REPREPRO_REPO_PATH}/src" -export GNUPGHOME=/var/lib/jenkins/.gnupg echo '# BEGIN SECTION: run reprepro' cd "${REPREPRO_REPO_PATH}/scripts" -sudo -E bash -c "PYTHONPATH=${REPREPRO_REPO_PATH}/src python3 import_upstream.py ${REPREPRO_PARAMS} \ - ${UPLOAD_TO_REPO:-:_} \ - ${REPREPRO_REPO_PATH}/config/packages.osrfoundation.org/${REPREPRO_IMPORT_YAML_FILE}" +PYTHONPATH=${REPREPRO_REPO_PATH}/src python3 import_upstream.py ${REPREPRO_PARAMS} \ + "${UPLOAD_TO_REPO:-:_}" \ + "${REPREPRO_REPO_PATH}/config/packages.osrfoundation.org/${REPREPRO_IMPORT_YAML_FILE}" echo '# END SECTION' echo '#BEGIN: exit the venv' From 59d8bbc85af731ce6120e277e288d200d139c6ed Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 20 Sep 2023 19:45:46 +0200 Subject: [PATCH 441/540] Disable S3 uploads for testing (#1016) Signed-off-by: Jose Luis Rivero --- jenkins-scripts/lib/repository_uploader.bash | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jenkins-scripts/lib/repository_uploader.bash b/jenkins-scripts/lib/repository_uploader.bash index 79e01c224..f45ee35b8 100644 --- a/jenkins-scripts/lib/repository_uploader.bash +++ b/jenkins-scripts/lib/repository_uploader.bash @@ -112,8 +112,8 @@ case ${UPLOAD_TO_REPO} in exit 1 fi ;; - "nightly") - # No uploads for nightly packages test runs + "nightly" | "testing") + # No uploads for nightly or testing packages test runs ENABLE_S3_UPLOAD=false ;; "none") From 76e716f48d11ffc98a7298ad082c2e5578ba5f4d Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 22 Sep 2023 17:07:51 +0200 Subject: [PATCH 442/540] Improve OSRFSourceCreation (#1008) * Make static parameters to look like that Use one option parameter to block jobs from changing names and repos and use class members to support static members. * Add an step to export TARBALL_NAME * Support for adding downstream: releasepy + repo_uploader * Added Global support to handle S3 paths * Added _test_repository_uploader job --------- Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/_configs_/Globals.groovy | 24 ++++ .../dsl/_configs_/OSRFSourceCreation.groovy | 106 +++++++++++++++--- jenkins-scripts/dsl/test.dsl | 54 ++++++++- 3 files changed, 166 insertions(+), 18 deletions(-) diff --git a/jenkins-scripts/dsl/_configs_/Globals.groovy b/jenkins-scripts/dsl/_configs_/Globals.groovy index 1386da8b6..2adc32511 100644 --- a/jenkins-scripts/dsl/_configs_/Globals.groovy +++ b/jenkins-scripts/dsl/_configs_/Globals.groovy @@ -131,4 +131,28 @@ class Globals static String nontest_label(String original_label) { return "(${original_label}) && !test-instance" } + + static String get_canonical_package_name(String package_name) { + return package_name.replaceAll('\\d*$', '') + } + + static String _s3_releases_dir(String package_name) { + return get_canonical_package_name(package_name) + '/releases' + } + + static String _s3_build_tarball_name(String package_name, String version) { + // canonical_name + version + return package_name.replaceAll('\\d*$', '') + '-' + version + } + + static String s3_upload_tarball_path(String package_name) { + return 's3://osrf-distributions/' + _s3_releases_dir(package_name) + } + + // Not yet in use. Requires changing release.py + static String s3_download_uri(String package_name, String version) { + return 'https://osrf-distributions.s3.amazonaws.com/' + \ + _s3_releases_dir(package_name) + \ + _s3_build_tarball_name(package_name, version) + } } diff --git a/jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy b/jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy index cb01aa186..a7eed1933 100644 --- a/jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy +++ b/jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy @@ -5,43 +5,48 @@ import _configs_.Globals class OSRFSourceCreation { + static String properties_file = "package_name.prop" + static String package_name = "" + static void addParameters(Job job, Map default_params = [:]) { + package_name = default_params.find{ it.key == "PACKAGE"}?.value + job.with { parameters { - stringParam("PACKAGE_NAME", - default_params.find{ it.key == "PACKAGE_NAME"}?.value, - "Software name (i.e gz-cmake3)") - stringParam("SOURCE_REPO_URI", - default_params.find{ it.key == "SOURCE_REPO_URI"}?.value, - "GitHub URI to release the sources from (i.e: https://github.com/gazebosim/gz-cmake.git)") + choiceParam('PACKAGE', + [default_params.find{ it.key == "PACKAGE"}?.value], + "Package name (can not be modified)") + choiceParam('SOURCE_REPO_URI', + [default_params.find{ it.key == "SOURCE_REPO_URI"}?.value], + "Software repository URL (can not be modified)") stringParam("VERSION", default_params.find{ it.key == "VERSION"}?.value, "Packages version to be built or nightly (enable nightly build mode)") stringParam("RELEASE_VERSION", default_params.find{ it.key == "RELEASE_VERSION"}?.value, - "Packages release version") + "For downstream jobs: Packages release version") stringParam("RELEASE_REPO_BRANCH", default_params.find{ it.key == "RELEASE_REPO_BRANCH"}?.value, - "Branch from the -release repo to be used") + "For downstream jobs: Branch from the -release repo to be used") stringParam("UPLOAD_TO_REPO", default_params.find{ it.key == "UPLOAD_TO_REPO"}?.value, - "OSRF repo name to upload the package to: stable | prerelease | nightly | none (for testing proposes)") + "For downstream jobs: OSRF repo name to upload the package to: stable | prerelease | nightly | none (for testing proposes)") } } } - static void create(Job job, Map default_params = [:]) + static void create(Job job, Map default_params = [:], Map default_hidden_params = [:]) { OSRFLinuxBuildPkgBase.create(job) GenericRemoteToken.create(job) OSRFSourceCreation.addParameters(job, default_params) + def pkg_sources_dir="pkgs" + job.with { - label Globals.nontest_label("docker") - wrappers { preBuildCleanup() } @@ -50,6 +55,9 @@ class OSRFSourceCreation priority 100 } + def canonical_package_name = Globals.get_canonical_package_name( + default_params.find{ it.key == "PACKAGE"}.value) + steps { systemGroovyCommand("""\ build.setDescription( @@ -62,14 +70,78 @@ class OSRFSourceCreation 'RTOOLS_BRANCH: ' + build.buildVariableResolver.resolve('RTOOLS_BRANCH')); """.stripIndent() ) + shell("""\ + #!/bin/bash -xe + + # Use Jammy/amd64 as base image to generate sources + export DISTRO=jammy + export ARCH=amd64 + /bin/bash -x ./scripts/jenkins-scripts/docker/gz-source-generation.bash + """.stripIndent() + ) shell("""\ - #!/bin/bash -xe - export DISTRO=jammy - export ARCH=amd64 + #!/bin/bash -xe + + # Export information from the build in properties_files. The tarball extraction helps to + # deal with changes in the compression of the tarballs. + tarball=\$(find \${WORKSPACE}/${pkg_sources_dir} \ + -type f \ + -name ${canonical_package_name}-\${VERSION}.tar.* \ + -printf "%f\\n") + if [[ -z \${tarball} ]] || [[ \$(wc -w <<< \${tarball}) != 1 ]]; then + echo "Tarball name extraction returned \${tarball} which is not a one word string" + exit 1 + fi - /bin/bash -x ./scripts/jenkins-scripts/docker/gz-source-generation.bash - """.stripIndent()) + echo "TARBALL_NAME=\${tarball}" >> ${properties_file} + """.stripIndent() + ) + } + } + } + + // Useful to inject testing jobs + static void call_uploader_and_releasepy(Job job, + String repository_uploader_jobname, + String releasepy_jobname) + { + job.with + { + publishers { + postBuildScripts { + steps { + conditionalSteps { + condition { + not { + expression('none|None|^$','${ENV,var="UPLOAD_TO_REPO"}') + } + } + steps { + // Invoke repository_uploader + downstreamParameterized { + trigger(repository_uploader_jobname) { + parameters { + currentBuild() + predefinedProps([PROJECT_NAME_TO_COPY_ARTIFACTS: '${JOB_NAME}', + S3_UPLOAD_PATH: Globals.s3_upload_tarball_path(package_name)]) + propertiesFile(properties_file) // TARBALL_NAME + } + } + } + downstreamParameterized { + trigger(releasepy_jobname) { + parameters { + currentBuild() + predefinedProps([PROJECT_NAME_TO_COPY_ARTIFACTS: "\${JOB_NAME}"]) + propertiesFile(properties_file) // TARBALL_NAME + } + } + } + } + } + } + } } } } diff --git a/jenkins-scripts/dsl/test.dsl b/jenkins-scripts/dsl/test.dsl index c8f542e8d..08c9a991a 100644 --- a/jenkins-scripts/dsl/test.dsl +++ b/jenkins-scripts/dsl/test.dsl @@ -20,11 +20,63 @@ OSRFLinuxCompilationAnyGitHub.create(ignition_ci_pr_job, // releasing testing job def releasepy_job = job("_test_releasepy") OSRFReleasepy.create(releasepy_job, [DRY_RUN: true]) +releasepy_job.with { + blockOn("_test_repository_uploader") { + blockLevel('GLOBAL') + scanQueueFor('ALL') + } +} // gz source testing job def gz_source_job = job("_test_gz_source") OSRFSourceCreation.create(gz_source_job, [ - PACKAGE_NAME: "gz-cmake3", + PACKAGE: "gz-cmake3" , SOURCE_REPO_URI: "https://github.com/gazebosim/gz-cmake.git"]) +OSRFSourceCreation.call_uploader_and_releasepy(gz_source_job, + '_test_repository_uploader', + '_test_releasepy') +// repository_uploader fake test job + +def pkg_sources_dir = 'pkgs' +def repo_uploader = job("_test_repository_uploader") +OSRFBase.create(repo_uploader) +repo_uploader.with +{ + label Globals.nontest_label("docker") + + wrappers { + preBuildCleanup() + } + + parameters + { + stringParam('PACKAGE','','Package name') + stringParam('TARBALL_NAME', '', 'Tarball name to upload') + stringParam('S3_UPLOAD_PATH','', 'S3 path to upload') + stringParam('UPLOAD_TO_REPO','none','repo to upload') + } + + steps + { + copyArtifacts('_test_gz_source') + { + includePatterns("${pkg_sources_dir}/\${TARBALL_NAME}") + buildSelector { + upstreamBuild() + } + } + + shell("""\ + #!/bin/bash -xe + + # check that the tarball name actually exist + + ls -R \${WORKSPACE} + test -f \${WORKSPACE}/${pkg_sources_dir}/\${TARBALL_NAME} + + echo "Fake upload of \${TARBALL_NAME} to \${S3_UPLOAD_PATH}" + """.stripIndent()) + } +} // ------------------------------------------------------------------- def outdated_job_runner = job("_test_outdated_job_runner") From b4994d1f9109091a51467ea95da02fee7da63c48 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 26 Sep 2023 20:55:19 +0200 Subject: [PATCH 443/540] Include all gazebo releases in gz-collection.yaml (#1017) * Update gz-collections with all supported distros --------- Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/gz-collections.yaml | 253 ++++++++++++++++++++ jenkins-scripts/dsl/ignition_collection.dsl | 229 +++++++++--------- 2 files changed, 373 insertions(+), 109 deletions(-) diff --git a/jenkins-scripts/dsl/gz-collections.yaml b/jenkins-scripts/dsl/gz-collections.yaml index d06eab493..0eec17016 100644 --- a/jenkins-scripts/dsl/gz-collections.yaml +++ b/jenkins-scripts/dsl/gz-collections.yaml @@ -28,22 +28,234 @@ spec_version: 0 collections: - name: 'citadel' libs: + - name: ign-cmake + major_version: 2 + repo: + current_branch: ign-cmake2 + - name: ign-tools + major_version: 1 + repo: + current_branch: ign-tools1 + - name: ign-math + major_version: 6 + repo: + current_branch: ign-math6 + - name: ign-plugin + major_version: 1 + repo: + current_branch: ign-plugin1 + - name: ign-common + major_version: 3 + repo: + current_branch: ign-common3 + - name: ign-msgs + major_version: 5 + repo: + current_branch: ign-msgs5 + - name: ign-rendering + major_version: 3 + repo: + current_branch: ign-rendering3 + - name: sdformat + major_version: 9 + repo: + current_branch: sdf9 + - name: ign-fuel-tools + major_version: 4 + repo: + current_branch: ign-fuel-tools4 + - name: ign-transport + major_version: 8 + repo: + current_branch: ign-transport8 + - name: ign-gui + major_version: 3 + repo: + current_branch: ign-gui3 + - name: ign-sensors + major_version: 3 + repo: + current_branch: ign-sensors3 + - name: ign-physics + major_version: 2 + repo: + current_branch: ign-physics2 + - name: ign-gazebo + major_version: 7 + repo: + current_branch: ign-gazebo3 + - name: ign-launch + major_version: 2 + repo: + current_branch: ign-launch2 + - name: ign-citadel + major_version: 1 + repo: + current_branch: main ci: configs: - bionic packaging: + linux: + package_name: + ignore_major_version: + - ign-citadel - name: 'fortress' libs: + - name: gz-cmake + major_version: 3 + repo: + current_branch: gz-cmake2 + - name: gz-tools + major_version: 1 + repo: + current_branch: gz-tools1 + - name: gz-utils + major_version: 1 + repo: + current_branch: gz-utils1 + - name: gz-math + major_version: 6 + repo: + current_branch: gz-math6 + - name: gz-plugin + major_version: 1 + repo: + current_branch: gz-plugin1 + - name: gz-common + major_version: 4 + repo: + current_branch: gz-common4 + - name: gz-msgs + major_version: 8 + repo: + current_branch: gz-msgs8 + - name: gz-rendering + major_version: 6 + repo: + current_branch: gz-rendering6 + - name: sdformat + major_version: 12 + repo: + current_branch: sdf12 + - name: gz-fuel-tools + major_version: 7 + repo: + current_branch: gz-fuel-tools7 + - name: gz-transport + major_version: 11 + repo: + current_branch: gz-transport11 + - name: gz-gui + major_version: 6 + repo: + current_branch: gz-gui6 + - name: gz-sensors + major_version: 7 + repo: + current_branch: gz-sensors7 + - name: gz-physics + major_version: 6 + repo: + current_branch: gz-physics6 + - name: gz-sim + major_version: 6 + repo: + current_branch: gz-sim6 + - name: gz-launch + major_version: 5 + repo: + current_branch: gz-launch5 + - name: gz-fortress + major_version: 1 + repo: + current_branch: main ci: configs: - focal packaging: + linux: + package_name: + ignore_major_version: + - gz-fortress - name: 'garden' libs: + - name: gz-cmake + major_version: 3 + repo: + current_branch: gz-cmake3 + - name: gz-tools + major_version: 2 + repo: + current_branch: gz-tools2 + - name: gz-utils + major_version: 2 + repo: + current_branch: gz-utils2 + - name: gz-math + major_version: 7 + repo: + current_branch: gz-math7 + - name: gz-plugin + major_version: 2 + repo: + current_branch: gz-plugin2 + - name: gz-common + major_version: 5 + repo: + current_branch: gz-common5 + - name: gz-msgs + major_version: 9 + repo: + current_branch: gz-msgs9 + - name: gz-rendering + major_version: 7 + repo: + current_branch: gz-rendering7 + - name: sdformat + major_version: 13 + repo: + current_branch: sdf13 + - name: gz-fuel-tools + major_version: 8 + repo: + current_branch: gz-fuel-tools8 + - name: gz-transport + major_version: 12 + repo: + current_branch: gz-transport12 + - name: gz-gui + major_version: 7 + repo: + current_branch: gz-gui7 + - name: gz-sensors + major_version: 7 + repo: + current_branch: gz-sensors7 + - name: gz-physics + major_version: 6 + repo: + current_branch: gz-physics6 + - name: gz-sim + major_version: 7 + repo: + current_branch: gz-sim7 + - name: gz-launch + major_version: 6 + repo: + current_branch: gz-launch6 + - name: gz-garden + major_version: 1 + repo: + current_branch: main ci: configs: - focal packaging: + linux: + package_name: + ignore_major_version: + - gz-garden - name: 'harmonic' libs: - name: gz-cmake @@ -125,6 +337,46 @@ collections: nightly: distros: - jammy + - name: '__upcoming__' + libs: + - name: gz-cmake + major_version: 4 + repo: + current_branch: main + - name: gz-tools + major_version: 3 + repo: + current_branch: main + - name: gz-utils + major_version: 3 + repo: + current_branch: main + - name: gz-math + major_version: 8 + repo: + current_branch: main + - name: gz-plugin + major_version: 3 + repo: + current_branch: main + - name: gz-common + major_version: 6 + repo: + current_branch: main + - name: gz-msgs + major_version: 11 + repo: + current_branch: main + - name: gz-physics + major_version: 7 + repo: + current_branch: main + ci: + configs: + - jammy + packaging: + exclude: + - __upcoming__ ci_configs: - name: bionic system: @@ -183,6 +435,7 @@ ci_configs: - gz-sensors exclude: - gz-harmonic + - __upcoming__ pre_setup_script_hook: gz-physics: - "export MAKE_JOBS=1" diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index e140b089d..e30fcaff5 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -366,26 +366,83 @@ void generate_install_job(prefix, gz_collection_name, distro, arch) gz_collections_yaml.collections.each { collection -> gz_collection_name = collection.name - // COLCON - Windows - def gz_win_ci_job = job("ign_${gz_collection_name}-ci-win") - Globals.gazebodistro_branch = true - OSRFWinCompilation.create(gz_win_ci_job, false) - gz_win_ci_job.with + dashboard_name = "ign-${gz_collection_name}" + if (gz_collection_name == "__upcoming__") + dashboard_name = gz_collection_name + + // Gazebo dashboards + // -------------------------------------------------------------- + dashboardView(dashboard_name) { + jobs { + gz_collection_jobs["${gz_collection_name}"].each { jobname -> + name(jobname) + } + if (collection.packaging?.linux?.nightly) { + collection.libs.each { lib -> + name(get_debbuilder_name(lib, collection.packaging)) + } + } + } + + columns { + status() + weather() + name() + testResult(0) + lastSuccess() + lastFailure() + lastDuration() + buildButton() + + } + + bottomPortlets { + jenkinsJobsList { + displayName('Jenkins jobs list') + } + } + + configure { view -> + view / columns << "hudson.plugins.warnings.WarningsColumn" (plugin: 'warnings@5.0.1') + + def topPortlets = view / NodeBuilder.newInstance().topPortlets {} + + topPortlets << 'hudson.plugins.view.dashboard.core.UnstableJobsPortlet' { + id createPortletId() + name 'Failing jobs' + showOnlyFailedJobs 'true' + recurse 'false' + } + } + } + + if (! collection.packaging.exclude?.contains(gz_collection_name)) { + // DEBBUILD: linux package builder + // -------------------------------------------------------------- + def build_pkg_job = job("gz-${gz_collection_name}-debbuilder") + OSRFLinuxBuildPkg.create(build_pkg_job) + build_pkg_job.with + { steps { - batchFile("""\ - set IGNITION_COLLECTION=${gz_collection_name} - call "./scripts/jenkins-scripts/lib/ign_collection-base.bat" + shell("""\ + #!/bin/bash -xe + + /bin/bash -x ./scripts/jenkins-scripts/docker/multidistribution-ignition-debbuild.bash """.stripIndent()) } + } } - Globals.gazebodistro_branch = false + collection.ci.configs.each { ci_config_name -> ci_config = gz_collections_yaml.ci_configs.find { it.name == ci_config_name } distro = ci_config.system.version arch = ci_config.system.arch + if (ci_config.exclude.contains(gz_collection_name)) + return + // INSTALL JOBS: // -------------------------------------------------------------- if ((gz_collection_name == "citadel") || (gz_collection_name == "fortress")) { @@ -442,121 +499,75 @@ gz_collections_yaml.collections.each { collection -> """.stripIndent()) } } - } - // MAC Brew CI job - // -------------------------------------------------------------- - def gz_brew_ci_job = job("ignition_${gz_collection_name}-ci-main-homebrew-amd64") - OSRFBrewCompilation.create(gz_brew_ci_job, DISABLE_TESTS) - OSRFGitHub.create(gz_brew_ci_job, - "gazebosim/gz-${gz_collection_name}", - "main", - "ign-${gz_collection_name}") - gz_brew_ci_job.with - { - steps { - shell("""\ - #!/bin/bash -xe - - /bin/bash -xe - "./scripts/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash" "gz-${gz_collection_name}" - """.stripIndent()) - } - } - - // MAC Brew bottle install job - // -------------------------------------------------------------- - def gz_brew_install_bottle_job = job("ignition_${gz_collection_name}-install_bottle-homebrew-amd64") - OSRFBrewInstall.create(gz_brew_install_bottle_job) - - gz_brew_install_bottle_job.with - { - triggers { - cron('@daily') - } - - def bottle_name = "ignition-${gz_collection_name}" - - steps { - shell("""\ - #!/bin/bash -xe - - /bin/bash -x ./scripts/jenkins-scripts/lib/project-install-homebrew.bash ${bottle_name} - """.stripIndent()) - } - - publishers + // COLCON - Windows + Globals.gazebodistro_branch = true + def gz_win_ci_job = job("ign_${gz_collection_name}-ci-win") + OSRFWinCompilation.create(gz_win_ci_job, false) + gz_win_ci_job.with { - configure { project -> - project / publishers << 'hudson.plugins.logparser.LogParserPublisher' { - unstableOnWarning true - failBuildOnError false - parsingRulesPath('/var/lib/jenkins/logparser_warn_on_mark_unstable') - } - } + steps { + batchFile("""\ + set IGNITION_COLLECTION=${gz_collection_name} + call "./scripts/jenkins-scripts/lib/ign_collection-base.bat" + """.stripIndent()) + } } - } - - // DEBBUILD: linux package builder - // -------------------------------------------------------------- - def build_pkg_job = job("gz-${gz_collection_name}-debbuilder") - OSRFLinuxBuildPkg.create(build_pkg_job) - build_pkg_job.with - { - steps { - shell("""\ - #!/bin/bash -xe + Globals.gazebodistro_branch = false - /bin/bash -x ./scripts/jenkins-scripts/docker/multidistribution-ignition-debbuild.bash - """.stripIndent()) - } - } + // MAC Brew CI job + // -------------------------------------------------------------- + def gz_brew_ci_job = job("ignition_${gz_collection_name}-ci-main-homebrew-amd64") + OSRFBrewCompilation.create(gz_brew_ci_job, DISABLE_TESTS) + OSRFGitHub.create(gz_brew_ci_job, + "gazebosim/gz-${gz_collection_name}", + "main", + "ign-${gz_collection_name}") + gz_brew_ci_job.with + { + steps { + shell("""\ + #!/bin/bash -xe - // Gazebo dashboards - dashboardView("ign-${gz_collection_name}") - { - jobs { - gz_collection_jobs["${gz_collection_name}"].each { jobname -> - name(jobname) + /bin/bash -xe + "./scripts/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash" "gz-${gz_collection_name}" + """.stripIndent()) } - if (collection.packaging?.linux?.nightly) { - collection.libs.each { lib -> - name(get_debbuilder_name(lib, collection.packaging)) - } - } - } + } - columns { - status() - weather() - name() - testResult(0) - lastSuccess() - lastFailure() - lastDuration() - buildButton() + // MAC Brew bottle install job + // -------------------------------------------------------------- + def gz_brew_install_bottle_job = job("ignition_${gz_collection_name}-install_bottle-homebrew-amd64") + OSRFBrewInstall.create(gz_brew_install_bottle_job) + gz_brew_install_bottle_job.with + { + triggers { + cron('@daily') } - bottomPortlets { - jenkinsJobsList { - displayName('Jenkins jobs list') - } - } + def bottle_name = "ignition-${gz_collection_name}" - configure { view -> - view / columns << "hudson.plugins.warnings.WarningsColumn" (plugin: 'warnings@5.0.1') + steps { + shell("""\ + #!/bin/bash -xe - def topPortlets = view / NodeBuilder.newInstance().topPortlets {} + /bin/bash -x ./scripts/jenkins-scripts/lib/project-install-homebrew.bash ${bottle_name} + """.stripIndent()) + } - topPortlets << 'hudson.plugins.view.dashboard.core.UnstableJobsPortlet' { - id createPortletId() - name 'Failing jobs' - showOnlyFailedJobs 'true' - recurse 'false' + publishers + { + configure { project -> + project / publishers << 'hudson.plugins.logparser.LogParserPublisher' { + unstableOnWarning true + failBuildOnError false + parsingRulesPath('/var/lib/jenkins/logparser_warn_on_mark_unstable') + } } } - } + } + } } // NIGHTLY GENERATION From d0e2db33a99977e31bab3b03f974a4786f3af6ef Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Tue, 26 Sep 2023 15:06:04 -0700 Subject: [PATCH 444/540] brew_release.dsl: disable bigsur (#1018) homebrew-core has stopped building bigsur bottles, so we will as well. Signed-off-by: Steve Peters --- jenkins-scripts/dsl/brew_release.dsl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/brew_release.dsl b/jenkins-scripts/dsl/brew_release.dsl index 2964076c8..e3e22a916 100644 --- a/jenkins-scripts/dsl/brew_release.dsl +++ b/jenkins-scripts/dsl/brew_release.dsl @@ -4,7 +4,7 @@ import javaposse.jobdsl.dsl.Job Globals.default_emails = "jrivero@osrfoundation.org, scpeters@osrfoundation.org" // first distro in list is used as touchstone -brew_supported_distros = [ "bigsur", "monterey", "ventura" ] +brew_supported_distros = [ "monterey", "ventura" ] bottle_hash_updater_job_name = 'generic-release-homebrew_pr_bottle_hash_updater' bottle_builder_job_name = 'generic-release-homebrew_triggered_bottle_builder' directory_for_bottles = 'pkgs' From 0f925822c5212f957fcc7931113fa136737e3417 Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Mon, 2 Oct 2023 03:02:25 -0700 Subject: [PATCH 445/540] Fix CI for pull requests from forks (#1021) Signed-off-by: Steve Peters --- .github/workflows/ci.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 41c07b533..f0ee0cdd8 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -15,8 +15,9 @@ jobs: - name: Idenfify files changed in this PR id: files run: | - git diff --name-only origin/${{ github.base_ref }}...origin/${{ github.head_ref }} - echo "changed-files=$(git diff --name-only origin/${{ github.base_ref }}...origin/${{ github.head_ref }}| tr '\n' ' ')" >> $GITHUB_OUTPUT + git fetch origin ${{ github.ref }} + git diff --name-only origin/${{ github.base_ref }}...FETCH_HEAD + echo "changed-files=$(git diff --name-only origin/${{ github.base_ref }}...FETCH_HEAD| tr '\n' ' ')" >> $GITHUB_OUTPUT - name: Run testing on changed config files id: dsl_check run: | From b7e7475918fe232acee73200ca0d1ae690859d69 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Mon, 2 Oct 2023 12:02:43 +0200 Subject: [PATCH 446/540] Fix the check for existing github repository (#1019) Signed-off-by: Jose Luis Rivero --- release.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/release.py b/release.py index 521289c3d..6b42a21bd 100755 --- a/release.py +++ b/release.py @@ -183,12 +183,8 @@ def parse_args(argv): return args def get_release_repository_info(package): - # Do not use git@github method since it fails in non existant repositories - # asking for stdin user/pass. Same happen if no user/pass is provided - # using the fake foo:foo here seems to work - github_test_url = "https://foo:foo@github.com/gazebo-release/" + package + "-release" - if (github_repo_exists(github_test_url)): - github_url = "https://github.com/gazebo-release/" + package + "-release" + github_url = "https://github.com/gazebo-release/" + package + "-release" + if (github_repo_exists(github_url)): return 'git', github_url error("release repository not found in github.com/gazebo-release") From 948fbf16db1e6aee6f82a9e9285ee427d25c1008 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Mon, 2 Oct 2023 18:44:30 +0200 Subject: [PATCH 447/540] Support for ROS 2 Iron CI in gzgarden wrappers (#960) Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/ros_gz_bridge.dsl | 33 ++++++++++++++++----------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/jenkins-scripts/dsl/ros_gz_bridge.dsl b/jenkins-scripts/dsl/ros_gz_bridge.dsl index 245c5c671..80492d865 100644 --- a/jenkins-scripts/dsl/ros_gz_bridge.dsl +++ b/jenkins-scripts/dsl/ros_gz_bridge.dsl @@ -10,6 +10,11 @@ def bridge_packages = [ 'ros_gz_sim_demos' ] +def gzgarden_ros_distros_ci = [ + 'humble', + 'iron' +] + // BLOOM PACKAGE BUILDER JOBS bridge_packages.each { pkg -> pkg_dashed = pkg.replaceAll("_", "-") @@ -40,7 +45,7 @@ bridge_packages.each { pkg -> stringParam("LINUX_DISTRO", 'ubuntu', "Linux distribution to build packages for") stringParam("DISTRO", "jammy", "Linux release inside LINUX_DISTRO to build packages for") stringParam("ARCH", "amd64", "Architecture to build packages for") - stringParam('ROS_DISTRO', 'humble','ROS DISTRO to build pakcages for') + stringParam('ROS_DISTRO', null,'ROS DISTRO to build pakcages for') stringParam("UPLOAD_TO_REPO", 'stable', "OSRF repo name to upload the package to") stringParam('UPSTREAM_RELEASE_REPO', 'https://github.com/j-rivero/ros_ign-release', 'Release repository url') } @@ -191,18 +196,20 @@ manual_install_test_job.with } } -def periodic_install_test_job = job("ros_gzgarden_bridge-install-pkg_humble-ci-jammy-amd64") -generate_install_test_job(periodic_install_test_job) -periodic_install_test_job.with -{ - parameters { - stringParam("DISTRO", 'jammy', "Linux release inside LINUX_DISTRO to build packages for") - stringParam("ARCH", 'amd64', "Architecture to build packages for") - stringParam("ROS_DISTRO", 'humble', "ROS distribution") - stringParam("GZ_VERSION", 'garden', "Gazebo version") - } +gzgarden_ros_distros_ci.each { ros_distro -> + def periodic_install_test_job = job("ros_gzgarden_bridge-install-pkg_${ros_distro}-ci-jammy-amd64") + generate_install_test_job(periodic_install_test_job) + periodic_install_test_job.with + { + parameters { + stringParam("DISTRO", 'jammy', "Linux release inside LINUX_DISTRO to build packages for") + stringParam("ARCH", 'amd64', "Architecture to build packages for") + stringParam("ROS_DISTRO", ros_distro, "ROS distribution") + stringParam("GZ_VERSION", 'garden', "Gazebo version") + } - triggers { - scm('@daily') + triggers { + scm('@daily') + } } } From bea701aa436bfb9031da16d2e9280369233c849a Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Mon, 2 Oct 2023 20:19:45 +0200 Subject: [PATCH 448/540] Patches and doc for GzHarmonic ros_gz creation (#1003) * Complete README with custom RELEASE_REPO_URL * Fix problems with renaming different Gazebo distros * Add the list of active releases --------- Signed-off-by: Jose Luis Rivero --- bloom/ros_gz/Dockerfile | 8 ++++---- bloom/ros_gz/README.md | 22 ++++++++++++++++++++++ bloom/ros_gz/rename-ros_gz-pkgs.bash | 4 +++- 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/bloom/ros_gz/Dockerfile b/bloom/ros_gz/Dockerfile index 620cd9931..628e39865 100644 --- a/bloom/ros_gz/Dockerfile +++ b/bloom/ros_gz/Dockerfile @@ -17,11 +17,11 @@ RUN apt-get update && apt-get install -y \ wget \ && rm -rf /var/lib/apt/lists/* -# Install osrf-rosdep for Gazebo distributions not in ROS +# Install osrf-rosdep for Gazebo distributions not in ROS. +# Note that 01 is important to give priority to the replace list below RUN sudo bash -c \ - 'wget https://raw.githubusercontent.com/osrf/osrf-rosdep/master/gz/00-gazebo.list -O /etc/ros/rosdep/sources.list.d/00-gazebo.list' -# Replace url -# since it does not support arbitrary environment variables + 'wget https://raw.githubusercontent.com/osrf/osrf-rosdep/master/gz/00-gazebo.list -O /etc/ros/rosdep/sources.list.d/01-gazebo.list' +# Bloom does not support environment vars, use the trick or replacing one rosdep keys by others RUN sudo bash -c \ "wget $URL_OSRF_ROSDEP_REPLACE -O /etc/ros/rosdep/sources.list.d/00-replace-gz.list" diff --git a/bloom/ros_gz/README.md b/bloom/ros_gz/README.md index 466b17005..f0840eee8 100644 --- a/bloom/ros_gz/README.md +++ b/bloom/ros_gz/README.md @@ -27,6 +27,14 @@ Although using the officially supported version is the recommended way specially for non experienced users, some use cases might need to use a newer version of Gazebo than the one selected in REP-2000. +### List of active relases + +| Gazebo Release | ROS / ROS 2 Release | status | ros_gz branch | -release repository | +| ---------------|---------------------|------------|---------------|---------------------| +| Garden | Humble | stable | humble | https://github.com/gazebo-release/ros_ign-release | +| Garden | Iron | prerelease | iron | https://github.com/gazebo-release/ros_ign-release | +| Harmonic | Iron | prerelease | ros2 | https://github.com/j-rivero/ros_ign-gzharmonic-release | + ### Upstream versions released using this tutorial The `gbp -release repository` hosts the latest version released by the @@ -35,6 +43,7 @@ version the version of `ros_gz` released will be the latest one existing in the official `gbp -release repository`. The version would be the same but the release number will start on 1000. + ## 2. Initial setup To release a modified version of `ros_gz` which supports a different major @@ -55,6 +64,19 @@ package name. - Usage: `$ rename-ros_gz-pkgs.bash ` - Example: `$ rename-ros_gz-pkgs.bash garden humble` +The script supports to inject a custom RELEASE_REPO_URL that points to a bloom gbp +repository different than https://github.com/gazebo-release/ros_ign-release. + +``` +i.e use a https://github.com/gazebo-testing/ros_ign-gzharmonic-release as gbp testing repository + +RELEASE_REPO_URL=https://github.com/gazebo-testing/ros_ign-gzharmonic-release \ + ./bloom_from_special_env.bash \ + humble \ + garden \ + https://raw.githubusercontent.com/osrf/osrf-rosdep/master/gz/replace_fortress_with_garden/00-replace-gz-fortress-with-garden.list +``` + ### 2.2 Create a custom track in tracks.yml diff --git a/bloom/ros_gz/rename-ros_gz-pkgs.bash b/bloom/ros_gz/rename-ros_gz-pkgs.bash index 66b3a7953..6f02fe75c 100755 --- a/bloom/ros_gz/rename-ros_gz-pkgs.bash +++ b/bloom/ros_gz/rename-ros_gz-pkgs.bash @@ -47,7 +47,9 @@ for pkg in ${PKGS}; do git commit debian/control.em debian/changelog.em -m "Patch name to release ${GZ_RELEASE} version" # Include conflict with initial package name in ROS sed -i -e '/^Depends/a\Conflicts: \@(Package)' debian/control.em - + if [[ -n ${GZ_RELEASE_TO_CONFLICT} ]]; then + sed -i -e "s/Conflicts: @(Package)/Conflicts: @(Package.replace('-gz','-gz${GZ_RELEASE_TO_CONFLICT}'))/" debian/control.em + fi git commit debian/control.em -m "Set up a conflict with official ROS packages" git push origin "debian/$distro/$pkg" done From 983e8481577fa3c1ec3489f441c5c4004bdea6f6 Mon Sep 17 00:00:00 2001 From: "Addisu Z. Taddese" Date: Tue, 3 Oct 2023 06:07:52 -0500 Subject: [PATCH 449/540] Make prevous version optional in source_changelog.bash (#1012) The latest tagged version is used as the default. Signed-off-by: Addisu Z. Taddese Co-authored-by: Jose Luis Rivero --- source-repo-scripts/source_changelog.bash | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/source-repo-scripts/source_changelog.bash b/source-repo-scripts/source_changelog.bash index eb733dbb4..71a8aea23 100755 --- a/source-repo-scripts/source_changelog.bash +++ b/source-repo-scripts/source_changelog.bash @@ -1,10 +1,18 @@ #!/bin/bash +# Generates a list of changes since the last tagged version. +# bash source_changelog.bash +# +# Optionally, the previous version can be provided # bash source_changelog.bash - -PREV_VER=$1 +# +# E.g. +# bash source_changelog.bash 3.0.0 git fetch --tags +PREV_VER=${1:-$(git describe --tags --abbrev=0 | sed 's/.*_//')} +echo "Changes since $PREV_VER" + ORIGIN_URL=$(git remote get-url origin) REPO=$(basename ${ORIGIN_URL%.git}) @@ -14,7 +22,7 @@ PREV_TAG=$(git tag | grep "_${PREV_VER}$") # Compare current branch to PREV_TAG BRANCH=$(git rev-parse --abbrev-ref HEAD) -COMMITS=$(git log ${BRANCH}...${PREV_TAG} --pretty=format:"%h") +COMMITS=$(git log ${BRANCH}...${PREV_TAG} --no-merges --pretty=format:"%h") for COMMIT in $COMMITS do From 61642d77a5aad9999ef809e731490b7977d38d2f Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 3 Oct 2023 16:19:45 +0200 Subject: [PATCH 450/540] Changes to complete the README on the creation of unofficial wrappers --- bloom/ros_gz/README.md | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/bloom/ros_gz/README.md b/bloom/ros_gz/README.md index f0840eee8..eba4bf4e9 100644 --- a/bloom/ros_gz/README.md +++ b/bloom/ros_gz/README.md @@ -43,7 +43,6 @@ version the version of `ros_gz` released will be the latest one existing in the official `gbp -release repository`. The version would be the same but the release number will start on 1000. - ## 2. Initial setup To release a modified version of `ros_gz` which supports a different major @@ -53,31 +52,25 @@ version of gazebo, before running bloom some actions need to be taken: For a new official wrappers the notation used below correspond to `ros_ign-release`: - 1. Fork (manually or using gh) current gbp repository: + 1. Fork (or create a new repo with all the tags and branches) the current gbp repository: https://github.com/ros2-gbp/ros_ign-release Note: the same -release repository can obviously hosts multiple ROS 2 releases but can not have multipe Gazebo releases since the bloom templates are set to just one package name. +On the fork, the `rename-ros_gz-pkgs.bash` script will change the bloom templates to modify package names and inject the `GZ_VERSION` needed: + 1. Clone the new repo, go to the directory and run rename-gazebo-ros-pkgs.bash - Usage: `$ rename-ros_gz-pkgs.bash ` - Example: `$ rename-ros_gz-pkgs.bash garden humble` -The script supports to inject a custom RELEASE_REPO_URL that points to a bloom gbp -repository different than https://github.com/gazebo-release/ros_ign-release. +If several unofficial wrappers using different Gazebo releases are going to exist for the same ROS 2 distribution (i.e: Garden and Harmonic for ROS 2 Iron), the rename script supports to declare a conflict using the `GZ_RELEASE_TO_CONFLICT` variable: ``` -i.e use a https://github.com/gazebo-testing/ros_ign-gzharmonic-release as gbp testing repository - -RELEASE_REPO_URL=https://github.com/gazebo-testing/ros_ign-gzharmonic-release \ - ./bloom_from_special_env.bash \ - humble \ - garden \ - https://raw.githubusercontent.com/osrf/osrf-rosdep/master/gz/replace_fortress_with_garden/00-replace-gz-fortress-with-garden.list +GZ_RELEASE_TO_CONFLICT=garden ./rename-ros_gz-pkgs.bash harmonic humble ``` - ### 2.2 Create a custom track in tracks.yml Copy the existing ROS 2 yaml track information and rename it to `${ros2}_gz${version}`. @@ -126,6 +119,19 @@ and invoke rocker with `--home` and `--user` flags to pass the credentials and customatizations needed for the bloom call. It will run the `bloom-release` command with the arguments required for the ros_gz wrappers. +The script supports to inject a custom RELEASE_REPO_URL that points to a bloom gbp +repository different than https://github.com/gazebo-release/ros_ign-release. + +``` +i.e use a https://github.com/gazebo-testing/ros_ign-gzharmonic-release as gbp testing repository + +RELEASE_REPO_URL=https://github.com/gazebo-testing/ros_ign-gzharmonic-release \ + ./bloom_from_special_env.bash \ + humble \ + garden \ + https://raw.githubusercontent.com/osrf/osrf-rosdep/master/gz/replace_fortress_with_garden/00-replace-gz-fortress-with-garden.list +``` + ### 3.2 Launching jobs in the osrf buildfarm The previous step generates the metadata needed to build the debians but there is From cf72fde46135c3d507d5ca0f5e33ab4989aae47b Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 3 Oct 2023 16:45:04 +0200 Subject: [PATCH 451/540] Avoid to check old ros-ign tags Signed-off-by: Jose Luis Rivero --- bloom/ros_gz/_check_metadata.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bloom/ros_gz/_check_metadata.bash b/bloom/ros_gz/_check_metadata.bash index da06ad3f9..1bf1db53f 100755 --- a/bloom/ros_gz/_check_metadata.bash +++ b/bloom/ros_gz/_check_metadata.bash @@ -15,7 +15,7 @@ TMP_DIR=$(mktemp -d) git clone -q "${RELEASE_REPO_URL}" "${TMP_DIR}" pushd "${TMP_DIR}" 2> /dev/null > /dev/null || exit DEBIAN_LATEST_TAG=$(git for-each-ref --sort=creatordate --format '%(refname)' refs/tags | \ - grep "debian/ros-${ROS_DISTRO}" | \ + grep -v 'ros-ign-' | grep "debian/ros-${ROS_DISTRO}" \ tail -1 | sed 's:refs/tags/::') git checkout -q "${DEBIAN_LATEST_TAG}" if ! dpkg-parsechangelog -SSource 2> /dev/null | grep -q "gz${GAZEBO_DISTRO}"; then From e2687f1d036c835b19df649a272204d0840c0b52 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 5 Oct 2023 17:00:20 +0200 Subject: [PATCH 452/540] Big clean up of release.py script (#1014) * Remove -no-ignition-auto option * Remove unused variables * Fix spacing and blank lines * For create_tarball_name remove the option of package_alias * Remove sanity_check_gazebo_versions (outdated) * Use byte stream for empty return type in check_call * Change method not to end up in exit * Remove upstream option * Remove i386 from release.py * Fix tarball name and tag name --------- Signed-off-by: Jose Luis Rivero --- release.py | 184 +++++++++++++++++++++++++---------------------------- 1 file changed, 87 insertions(+), 97 deletions(-) diff --git a/release.py b/release.py index 6b42a21bd..f76ba66c6 100755 --- a/release.py +++ b/release.py @@ -17,13 +17,12 @@ except KeyError: JENKINS_URL = 'http://build.osrfoundation.org' JOB_NAME_PATTERN = '%s-debbuilder' -JOB_NAME_UPSTREAM_PATTERN = 'upstream-%s-debbuilder' -GENERIC_BREW_PULLREQUEST_JOB='generic-release-homebrew_pull_request_updater' +GENERIC_BREW_PULLREQUEST_JOB = 'generic-release-homebrew_pull_request_updater' UPLOAD_DEST_PATTERN = 's3://osrf-distributions/%s/releases/' DOWNLOAD_URI_PATTERN = 'https://osrf-distributions.s3.amazonaws.com/%s/releases/' -LINUX_DISTROS = [ 'ubuntu', 'debian' ] -SUPPORTED_ARCHS = ['amd64', 'i386', 'armhf', 'arm64'] +LINUX_DISTROS = ['ubuntu', 'debian'] +SUPPORTED_ARCHS = ['amd64', 'armhf', 'arm64'] RELEASEPY_NO_ARCH_PREFIX = '.releasepy_NO_ARCH_' # Ubuntu distributions are automatically taken from the top directory of @@ -36,7 +35,6 @@ DRY_RUN = False NIGHTLY = False PRERELEASE = False -UPSTREAM = False NO_SRC_FILE = False IGNORE_DRY_RUN = True @@ -57,43 +55,53 @@ 'ign-transport12', 'ign-utils2'] + class ErrorNoPermsRepo(Exception): pass + class ErrorNoUsernameSupplied(Exception): pass + class ErrorURLNotFound404(Exception): pass + class ErrorNoOutput(Exception): pass + def error(msg): print("\n !! " + msg + "\n") sys.exit(1) + def print_success(msg): print(" + OK " + msg) + # Remove the last character if it is a number. # That should leave just the package name instead of packageVersion # I.E gazebo5 -> gazebo def get_canonical_package_name(pkg_name): - return pkg_name.rstrip('1234567890') + return pkg_name.rstrip('1234567890') + def is_catkin_package(): return os.path.isfile("package.xml") + def github_repo_exists(url): try: check_call(['git', 'ls-remote', '-q', '--exit-code', url], IGNORE_DRY_RUN) - except (ErrorURLNotFound404, ErrorNoOutput) as e: + except (ErrorURLNotFound404, ErrorNoOutput): return False except Exception as e: error("Unexpected problem checking for git repo: " + str(e)) return True + def generate_package_source(srcdir, builddir): cmake_cmd = ["cmake"] @@ -112,14 +120,14 @@ def exists_main_branch(github_url): try: if (check_call(check_main_cmd, IGNORE_DRY_RUN)): return True - except Exception as e: + except Exception: return False + def parse_args(argv): global DRY_RUN global NIGHTLY global PRERELEASE - global UPSTREAM global NO_SRC_FILE parser = argparse.ArgumentParser(description='Make releases.') @@ -128,8 +136,6 @@ def parse_args(argv): parser.add_argument('jenkins_token', help='secret token to allow access to Jenkins to start builds') parser.add_argument('--dry-run', dest='dry_run', action='store_true', default=False, help='dry-run; i.e., do actually run any of the commands') - parser.add_argument('-u', '--upstream', dest='upstream', action='store_true', default=False, - help='release non OSRF software (do not generate and upload source tar.bz)') parser.add_argument('-a', '--package-alias', dest='package_alias', default=None, help='different name that we are releasing under') @@ -143,8 +149,6 @@ def parse_args(argv): help='no-sanity-checks; i.e. skip sanity checks commands') parser.add_argument('--no-generate-source-file', dest='no_source_file', action='store_true', default=False, help='Do not generate source file when building') - parser.add_argument('--no-ignition-auto', dest='no_ignition_auto', action='store_true', default=False, - help='Use package name to create --package-alias if package name starts with ign-') parser.add_argument('--upload-to-repo', dest='upload_to_repository', default="stable", help='OSRF repo to upload: stable | prerelease | nightly') parser.add_argument('--extra-osrf-repo', dest='extra_repo', default="", @@ -159,35 +163,29 @@ def parse_args(argv): if args.package in GARDEN_IGN_PACKAGES: print(f"Garden packages start with gz- prefix, changing {args.package} to {args.package.replace('ign-','gz-')}",) - args.package = args.package.replace('ign-','gz-') + args.package = args.package.replace('ign-', 'gz-') args.package_alias = args.package - # If ignition auto is enabled, replace ign- with ignition- at the beginning - if not args.no_ignition_auto and args.package.startswith('ign-'): - args.package_alias = args.package.replace('ign-', 'ignition-') DRY_RUN = args.dry_run - UPSTREAM = args.upstream NO_SRC_FILE = args.no_source_file - UPLOAD_REPO = args.upload_to_repository if args.upload_to_repository == 'nightly': NIGHTLY = True - NIGHTLY_BRANCH = args.nightly_branch if args.upload_to_repository == 'prerelease': PRERELEASE = True - # Upstream and nightly do not generate a tar.bz2 file - if args.upstream or NIGHTLY: + # Nightly do not generate a tar.bz2 file + if NIGHTLY: NO_SRC_FILE = True args.no_source_file = True return args + def get_release_repository_info(package): github_url = "https://github.com/gazebo-release/" + package + "-release" if (github_repo_exists(github_url)): return 'git', github_url - error("release repository not found in github.com/gazebo-release") def download_release_repository(package, release_branch): vcs, url = get_release_repository_info(package) @@ -206,16 +204,18 @@ def download_release_repository(package, release_branch): check_call(cmd, IGNORE_DRY_RUN) return release_tmp_dir, release_branch + def sanity_package_name_underscore(package, package_alias): # Alias is never empty. It hosts a exect copy of package if not provided if '_' in package_alias and package_alias != package: - error("Found an underscore in package_alias. It will conflict with debian package names. May be fixed changing the underscore for a dash.") + error("Found an underscore in package_alias. It will conflict with debian package names. May be fixed changing the underscore for a dash.") if '_' in package and package_alias == package: - error("Found an underscore in package name without providing a package alias (-a ). You probably want to match the package name in the debian changelog") + error("Found an underscore in package name without providing a package alias (-a ). You probably want to match the package name in the debian changelog") print_success("No underscore in package name") + def sanity_package_name(repo_dir, package, package_alias): expected_name = package @@ -223,11 +223,11 @@ def sanity_package_name(repo_dir, package, package_alias): expected_name = package_alias # Use igntiion for Citadel and Fortress, gz for Garden and beyond - gz_name = expected_name.replace("ignition", "gz"); - gz_name = gz_name.replace("gazebo", "sim"); + gz_name = expected_name.replace("ignition", "gz") + gz_name = gz_name.replace("gazebo", "sim") - cmd = ["find", repo_dir, "-name", "changelog","-exec","head","-n","1","{}",";"] - out, err = check_call(cmd, IGNORE_DRY_RUN) + cmd = ["find", repo_dir, "-name", "changelog", "-exec", "head", "-n", "1", "{}", ";"] + out, _ = check_call(cmd, IGNORE_DRY_RUN) for line in out.decode().split('\n'): if not line: continue @@ -235,8 +235,8 @@ def sanity_package_name(repo_dir, package, package_alias): if line.partition(' ')[0] != expected_name and line.partition(' ')[0] != gz_name: error("Error in changelog package name or alias: " + line) - cmd = ["find", repo_dir, "-name", "control","-exec","grep","-H","Source:","{}",";"] - out, err = check_call(cmd, IGNORE_DRY_RUN) + cmd = ["find", repo_dir, "-name", "control", "-exec", "grep", "-H", "Source:", "{}", ";"] + out, _ = check_call(cmd, IGNORE_DRY_RUN) for line in out.decode().split('\n'): if not line: continue @@ -246,17 +246,18 @@ def sanity_package_name(repo_dir, package, package_alias): print_success("Package names in changelog and control") + def sanity_package_version(repo_dir, version, release_version): - cmd = ["find", repo_dir, "-name", "changelog","-exec","head","-n","1","{}",";"] - out, err = check_call(cmd, IGNORE_DRY_RUN) + cmd = ["find", repo_dir, "-name", "changelog", "-exec", "head", "-n", "1", "{}", ";"] + out, _ = check_call(cmd, IGNORE_DRY_RUN) for line in out.decode().split('\n'): if not line: continue # return full version in brackets - full_version=line.split(' ')[1] + full_version = line.split(' ')[1] # get only version (not release) in brackets - c_version=full_version[full_version.find("(")+1:full_version.find("-")] - c_revision=full_version[full_version.find("-")+1:full_version.rfind("~")] + c_version = full_version[full_version.find("(")+1:full_version.find("-")] + c_revision = full_version[full_version.find("-")+1:full_version.rfind("~")] if c_version != version: error("Error in package version. Repo version: " + c_version + " Provided version: " + version) @@ -267,17 +268,6 @@ def sanity_package_version(repo_dir, version, release_version): print_success("Package versions in changelog") print_success("Package release versions in changelog") -def sanity_check_gazebo_versions(package, version): - if package == 'gazebo': - if int(version[0]) > 1: - error("Error in gazebo version. Please use 'gazebo-current' package for gazebo 2") - elif package == 'gazebo-current': - if int(version[0]) < 2: - error("Error in gazebo-current version. Please use 'gazebo' package for gazebo 1.x") - else: - return - - print_success("Gazebo version in proper gazebo package") def sanity_check_sdformat_versions(package, version): if package == 'sdformat': @@ -288,12 +278,14 @@ def sanity_check_sdformat_versions(package, version): print_success("sdformat version in proper sdformat package") + def sanity_check_repo_name(repo_name): if repo_name in OSRF_REPOS_SUPPORTED: return error("Upload repo value: " + repo_name + " is not valid. stable | prerelease | nightly") + def sanity_project_package_in_stable(version, repo_name): if repo_name != 'stable': return @@ -306,17 +298,19 @@ def sanity_project_package_in_stable(version, repo_name): return + def sanity_use_prerelease_branch(release_branch): if release_branch == 'prerelease': error("The use of prerelease branch is now deprecated. Please check internal wiki instructions") return + def check_s3cmd_configuration(): # Need to check if s3cmd is installed try: subprocess.call(["s3cmd", "--version"]) - except OSError as e: + except OSError: error("s3cmd command for uploading is not available. Install it using: apt-get install s3cmd") # Need to check if configuration for s3 exists @@ -326,6 +320,7 @@ def check_s3cmd_configuration(): return True + def sanity_checks(args, repo_dir): sanity_package_name_underscore(args.package, args.package_alias) sanity_package_name(repo_dir, args.package, args.package_alias) @@ -335,7 +330,6 @@ def sanity_checks(args, repo_dir): if not NIGHTLY: check_s3cmd_configuration() sanity_package_version(repo_dir, args.version, str(args.release_version)) - sanity_check_gazebo_versions(args.package, args.version) sanity_check_sdformat_versions(args.package, args.version) sanity_project_package_in_stable(args.version, args.upload_to_repository) @@ -351,6 +345,7 @@ def get_exclusion_arches(files): return r + def discover_distros(repo_dir): if not os.path.isdir(repo_dir): return None @@ -358,12 +353,16 @@ def discover_distros(repo_dir): _, subdirs, files = os.walk(repo_dir).__next__() repo_arch_exclusion = get_exclusion_arches(files) - if '.git' in subdirs: subdirs.remove('.git') - # remove ubuntu (common stuff) and debian (new supported distro at top level) - if 'ubuntu' in subdirs: subdirs.remove('ubuntu') - if 'debian' in subdirs: subdirs.remove('debian') + if '.git' in subdirs: + subdirs.remove('.git') + # remove ubuntu (common stuff) and debian (supported distro at top level) + if 'ubuntu' in subdirs: + subdirs.remove('ubuntu') + if 'debian' in subdirs: + subdirs.remove('debian') # Some releasing methods use patches/ in root - if 'patches' in subdirs: subdirs.remove('patches') + if 'patches' in subdirs: + subdirs.remove('patches') if not subdirs: error('Can not find distributions directories in the -release repo') @@ -378,18 +377,19 @@ def discover_distros(repo_dir): print('Releasing for distributions: ') for k in distro_arch_list: - print( "- " + k + " (" + ', '.join(distro_arch_list[k]) +")") + print("- " + k + " (" + ', '.join(distro_arch_list[k]) + ")") return distro_arch_list -def check_call(cmd, ignore_dry_run = False): + +def check_call(cmd, ignore_dry_run=False): if ignore_dry_run: # Commands that do not change anything in repo level - print('Dry-run running:\n %s\n'%(' '.join(cmd))) + print('Dry-run running:\n %s\n' % (' '.join(cmd))) else: - print('Running:\n %s'%(' '.join(cmd))) + print('Running:\n %s' % (' '.join(cmd))) if DRY_RUN and not ignore_dry_run: - return '', '' + return b'', b'' else: po = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = po.communicate() @@ -406,30 +406,28 @@ def check_call(cmd, ignore_dry_run = False): raise ErrorNoOutput() # Unkown exception - print('Error running command (%s).'%(' '.join(cmd))) - print('stdout: %s'%(out.decode())) - print('stderr: %s'%(err.decode())) + print('Error running command (%s).' % (' '.join(cmd))) + print('stdout: %s' % (out.decode())) + print('stderr: %s' % (err.decode())) raise Exception('subprocess call failed') return out, err # Returns tarball name: package name/alias without versions def create_tarball_name(args): - # For ignition, we use the package_alias instead of package - return re.sub(r'[0-9]+$', '', - args.package if not IGN_REPO else args.package_alias) + return re.sub(r'[0-9]+$', '', args.package) # Returns: sha, tarball file name, tarball full path def create_tarball_path(tarball_name, version, builddir, dry_run): - tarball_fname = '%s-%s.tar.bz2'%(tarball_name, version) + tarball_fname = '%s-%s.tar.bz2' % (tarball_name, version) # Try using the tarball_name as it is tarball_path = os.path.join(builddir, tarball_fname) if not os.path.isfile(tarball_path): # Try looking for special project names using underscores - alt_tarball_name = "_".join(tarball_name.rsplit("-",1)) - alt_tarball_fname = '%s-%s.tar.bz2'%(alt_tarball_name, version) + alt_tarball_name = "_".join(tarball_name.rsplit("-", 1)) + alt_tarball_fname = '%s-%s.tar.bz2' % (alt_tarball_name, version) alt_tarball_path = os.path.join(builddir, alt_tarball_fname) if (not dry_run): if not os.path.isfile(alt_tarball_path): @@ -446,6 +444,7 @@ def create_tarball_path(tarball_name, version, builddir, dry_run): return out.split(' ')[0], tarball_fname, tarball_path + def generate_upload_tarball(args): ################################################### # Platform-agnostic stuff. @@ -489,7 +488,7 @@ def generate_upload_tarball(args): # If we're releasing under a different name, then rename the tarball (the # package itself doesn't know anything about this). if args.package != args.package_alias: - tarball_fname = '%s-%s.tar.bz2'%(args.package_alias, args.version) + tarball_fname = '%s-%s.tar.bz2' % (args.package_alias, args.version) if (not args.dry_run): dest_file = os.path.join(builddir, tarball_fname) # Do not copy if files are the same @@ -515,19 +514,19 @@ def generate_upload_tarball(args): try: # tilde is not a valid character in git - tag = '%s_%s' % (args.package_alias, args.version.replace('~','-')) + tag = '%s_%s' % (args.package_alias, args.version.replace('~', '-')) check_call(['git', 'tag', '-f', tag]) check_call(['git', 'push', '--tags']) - except ErrorNoPermsRepo as e: + except ErrorNoPermsRepo: print('The Git server reports problems with permissions') print('The branch could be blocked by configuration if you do not have') print('rights to push code in default branch.') sys.exit(1) - except ErrorNoUsernameSupplied as e: + except ErrorNoUsernameSupplied: print('git tag could not be committed because you have not configured') print('your username. Use "git config --username" to set your username.') sys.exit(1) - except Exception as e: + except Exception: print('There was a problem with pushing tags to the git repository') print('Do you have write perms in the repository?') sys.exit(1) @@ -546,14 +545,12 @@ def go(argv): args.release_version = 1 # Sanity checks and dicover supported distributions before proceed. - # UPSTREAM repository is not known in release-tools script - if not UPSTREAM: - repo_dir, args.release_repo_branch = download_release_repository(args.package, args.release_repo_branch) - # The supported distros are the ones in the top level of -release repo - ubuntu_distros = discover_distros(repo_dir) # top level, ubuntu - debian_distros = discover_distros(repo_dir + '/debian/') # debian dir top level, Debian - if not args.no_sanity_checks: - sanity_checks(args, repo_dir) + repo_dir, args.release_repo_branch = download_release_repository(args.package, args.release_repo_branch) + # The supported distros are the ones in the top level of -release repo + ubuntu_distros = discover_distros(repo_dir) # top level, ubuntu + debian_distros = discover_distros(repo_dir + '/debian/') # debian dir top level, Debian + if not args.no_sanity_checks: + sanity_checks(args, repo_dir) source_tarball_uri = '' source_tarball_sha = '' @@ -587,17 +584,15 @@ def go(argv): # name must be modified in the future params['SOURCE_TARBALL_URI'] = args.nightly_branch - if UPSTREAM: - job_name = JOB_NAME_UPSTREAM_PATTERN%(args.package) - else: - job_name = JOB_NAME_PATTERN%(args.package) - + job_name = JOB_NAME_PATTERN % (args.package) params_query = urllib.parse.urlencode(params) # RELEASING FOR BREW - brew_url = '%s/job/%s/buildWithParameters?%s'%(JENKINS_URL, - GENERIC_BREW_PULLREQUEST_JOB, - params_query) + brew_url = '%s/job/%s/buildWithParameters?%s' % ( + JENKINS_URL, + GENERIC_BREW_PULLREQUEST_JOB, + params_query) + if not NIGHTLY and not args.bump_rev_linux_only: print('- Brew: %s' % (brew_url)) if not DRY_RUN: @@ -622,9 +617,6 @@ def go(argv): if (PRERELEASE or NIGHTLY): if (a == 'armhf' or a == 'arm64'): continue - # Only i386 for Ubuntu in Bionic - if (a == 'i386' and l != 'debian' and d != 'bionic'): - continue linux_platform_params = params.copy() linux_platform_params['ARCH'] = a @@ -643,9 +635,6 @@ def go(argv): ('gz-physics' in args.package_alias): linux_platform_params['JENKINS_NODE_TAG'] = 'large-memory' - if (NIGHTLY and a == 'i386'): - continue - # control nightly generation using a single machine to process # all distribution builds to avoid race conditions. Note: this # assumes that large-memory nodes are beind used for nightly @@ -657,11 +646,12 @@ def go(argv): linux_platform_params_query = urllib.parse.urlencode(linux_platform_params) - url = '%s/job/%s/buildWithParameters?%s'%(JENKINS_URL, job_name, linux_platform_params_query) - print('- Linux: %s'%(url)) + url = '%s/job/%s/buildWithParameters?%s' % (JENKINS_URL, job_name, linux_platform_params_query) + print('- Linux: %s' % (url)) if not DRY_RUN: urllib.request.urlopen(url) + if __name__ == '__main__': go(sys.argv) From 7e831e61a2a94e8573ce6b9736f09a782c588a1f Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 5 Oct 2023 17:02:10 +0200 Subject: [PATCH 453/540] Install gz-sim*-cli on ros_gz testing builds (#1004) * Install gz-sim*-cli on ros_gz testing builds --------- Signed-off-by: Jose Luis Rivero --- jenkins-scripts/docker/ros_gz-install-test-job.bash | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/jenkins-scripts/docker/ros_gz-install-test-job.bash b/jenkins-scripts/docker/ros_gz-install-test-job.bash index 1421d0003..274356136 100644 --- a/jenkins-scripts/docker/ros_gz-install-test-job.bash +++ b/jenkins-scripts/docker/ros_gz-install-test-job.bash @@ -30,6 +30,10 @@ fi echo '# END SECTION' echo '# BEGIN SECTION: test gz_sim via ros2 launch' +# Workaround until ros_gz defines a dependency on gz-sim*-cli. Avoid to use +# apt --install-suggests since it installs the whole universe of packages +VER=\$(dpkg -l | grep libgz-sim | grep ^ii | head -1 | awk '{ print \$3 }') +sudo apt-get install -y gz-sim\${VER:0:1}-cli TEST_START=\`date +%s\` # preserve-status did not work here timeout 180 ros2 launch ros_gz_sim gz_sim.launch.py gz_args:=shapes.sdf || true @@ -42,9 +46,6 @@ if [ \$DIFF -lt 180 ]; then fi """ -# Need bc to proper testing and parsing the time -# TODO: fix gz-sim-cli dependency on ros_gz package -# to avoid hardcoded version on gz-sim -export DEPENDENCY_PKGS="${DEPENDENCY_PKGS} wget ros-${ROS_DISTRO}-ros-base gz-sim7-cli" +export DEPENDENCY_PKGS="${DEPENDENCY_PKGS} wget ros-${ROS_DISTRO}-ros-base" . ${SCRIPT_DIR}/lib/generic-install-base.bash From 81651dd2d3ab9331a495170a6bc7111fc9c3921d Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 6 Oct 2023 18:22:25 +0200 Subject: [PATCH 454/540] Change all PR ci jobs (-ci-pr_any-) to use the yaml configuration (#1022) * Just report true on exsiting linux -ci-pr_any * Geneate -ci-pr_any- for all jobs using yaml Signed-off-by: Jose Luis Rivero --------- Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/gazebo_libs.dsl | 8 ++++---- jenkins-scripts/dsl/ignition.dsl | 28 +++++----------------------- jenkins-scripts/dsl/sdformat.dsl | 20 ++++---------------- 3 files changed, 13 insertions(+), 43 deletions(-) diff --git a/jenkins-scripts/dsl/gazebo_libs.dsl b/jenkins-scripts/dsl/gazebo_libs.dsl index ce6181ab4..cff29f441 100644 --- a/jenkins-scripts/dsl/gazebo_libs.dsl +++ b/jenkins-scripts/dsl/gazebo_libs.dsl @@ -75,10 +75,6 @@ boolean is_testing_enabled(lib_name, ci_config) void generate_ciconfigs_by_lib(config, configs_per_lib_index) { config.collections.each { collection -> - // TODO(jrivero): limit to harmonic for testing proposes - if (collection.name != 'harmonic') - return - collection.libs.each { lib -> def libName = lib.name def branch = lib.repo.current_branch @@ -181,6 +177,10 @@ configs_per_lib_index.each { lib_name, lib_configs -> // CI branch jobs (-ci-$branch-) (pulling check every 5 minutes) branches_with_collections.each { branch_and_collection -> + // TODO: remove after testing + if (branch_and_collection.collection != 'harmonic') + return + branch_name = branch_and_collection.branch def gz_ci_job = job("${gz_job_name_prefix}-ci-${branch_name}-${distro}-${arch}") generate_ci_job(gz_ci_job, lib_name, branch_name, ci_config) diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index 566a601f4..45f4bafdd 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -443,31 +443,13 @@ gz_software.each { gz_sw -> def gz_ci_job_name = "ignition_${software_name}-ci-pr_any-ubuntu_auto-${arch}" def gz_ci_any_job = job(gz_ci_job_name) def gz_checkout_dir = "ign-${software_name}" - OSRFLinuxCompilationAnyGitHub.create(gz_ci_any_job, - "gazebosim/${gz_checkout_dir}", - enable_testing(software_name)) - include_gpu_label_if_needed(gz_ci_any_job, software_name) + GenericAnyJobGitHub.create(gz_ci_any_job, + "gazebosim/${gz_checkout_dir}", + [], + true) gz_ci_any_job.with { - if (gz_sw == 'physics') { - label Globals.nontest_label("large-memory") - extra_str += '\nexport MAKE_JOBS=1' - } - - steps - { - shell("""\ - #!/bin/bash -xe - - export DISTRO=${ci_distro_str} - - ${GLOBAL_SHELL_CMD} - - export ARCH=${arch} - - /bin/bash -xe ./scripts/jenkins-scripts/docker/gz_${gz_sw.replaceAll('-','_')}-compilation.bash - """.stripIndent()) - } // end of steps + description 'Automatic generated job by DSL jenkins. Stub job for migration, not doing any check' } // end of ci_any_job // add ci-pr_any to the list for CIWorkflow diff --git a/jenkins-scripts/dsl/sdformat.dsl b/jenkins-scripts/dsl/sdformat.dsl index 27fa20683..d983d18f5 100644 --- a/jenkins-scripts/dsl/sdformat.dsl +++ b/jenkins-scripts/dsl/sdformat.dsl @@ -87,24 +87,12 @@ ci_distro.each { distro -> } // -------------------------------------------------------------- - // 2. Create the any job + // 2. Create a fake any job for migrating + // TODO: remove the job once the migration in https://github.com/gazebo-tooling/release-tools/issues/1010 + // is done. String sdf_repo = "gazebosim/sdformat" def sdformat_ci_any_job = job(ci_build_any_job_name_linux) - OSRFLinuxCompilationAnyGitHub.create(sdformat_ci_any_job, sdf_repo) - sdformat_ci_any_job.with - { - steps - { - shell("""\ - #!/bin/bash -xe - - export DISTRO=${ci_distro_str} - - export ARCH=${arch} - /bin/bash -xe ./scripts/jenkins-scripts/docker/sdformat-compilation.bash - """.stripIndent()) - } - } + GenericAnyJobGitHub.create(sdformat_ci_any_job, sdf_repo) } // end of arch } // end of distro From 6ebb3ad2494f877c1213533b5e2fb1173db5ff47 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 6 Oct 2023 16:31:10 +0200 Subject: [PATCH 455/540] Fix PACKAGE_NAME replaced by PACKAGE in previous changes Signed-off-by: Jose Luis Rivero --- jenkins-scripts/docker/gz-source-generation.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/docker/gz-source-generation.bash b/jenkins-scripts/docker/gz-source-generation.bash index c5bb4d36f..41669ed51 100644 --- a/jenkins-scripts/docker/gz-source-generation.bash +++ b/jenkins-scripts/docker/gz-source-generation.bash @@ -18,7 +18,7 @@ BUILD_DIR=\$SOURCES_DIR/build cd \${WORKSPACE} rm -fr \$SOURCES_DIR && mkdir \$SOURCES_DIR -git clone --depth 1 --branch ${PACKAGE_NAME}_${VERSION} ${SOURCE_REPO_URI} \${SOURCES_DIR} +git clone --depth 1 --branch ${PACKAGE}_${VERSION} ${SOURCE_REPO_URI} \${SOURCES_DIR} rm -fr \$BUILD_DIR && mkdir \$BUILD_DIR cd \${BUILD_DIR} cmake .. -DPACKAGE_SOURCE_ONLY:BOOL=ON From c613670ae307b9eff516dee6073539797ff1cd1e Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 6 Oct 2023 16:47:30 +0200 Subject: [PATCH 456/540] Change TARBALL_NAME bY S3_FILES_TO_UPLOAD Do not use TARBALL_NAME anymore and use S3_FILES_TO_UPLOAD instead. Improve the testing script by listing all possible files passed. Fix the use of relative path instead the absolute one. Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/_configs_/Globals.groovy | 19 +++++--------- .../dsl/_configs_/OSRFSourceCreation.groovy | 11 +++++--- jenkins-scripts/dsl/test.dsl | 26 ++++++++++++++++--- 3 files changed, 36 insertions(+), 20 deletions(-) diff --git a/jenkins-scripts/dsl/_configs_/Globals.groovy b/jenkins-scripts/dsl/_configs_/Globals.groovy index 2adc32511..3da1e8c38 100644 --- a/jenkins-scripts/dsl/_configs_/Globals.groovy +++ b/jenkins-scripts/dsl/_configs_/Globals.groovy @@ -136,23 +136,18 @@ class Globals return package_name.replaceAll('\\d*$', '') } - static String _s3_releases_dir(String package_name) { + static String s3_releases_dir(String package_name) { return get_canonical_package_name(package_name) + '/releases' } - static String _s3_build_tarball_name(String package_name, String version) { - // canonical_name + version - return package_name.replaceAll('\\d*$', '') + '-' + version - } - static String s3_upload_tarball_path(String package_name) { - return 's3://osrf-distributions/' + _s3_releases_dir(package_name) + return 's3://osrf-distributions/' + s3_releases_dir(package_name) } - // Not yet in use. Requires changing release.py - static String s3_download_uri(String package_name, String version) { - return 'https://osrf-distributions.s3.amazonaws.com/' + \ - _s3_releases_dir(package_name) + \ - _s3_build_tarball_name(package_name, version) + static String s3_download_url_basedir(String package_name) { + return 'https://osrf-distributions.s3.amazonaws.com/' + s3_releases_dir(package_name) } + + /* rest of the s3 paths need to be cumputed during job running time since + * they depend on VERSION and it is not avialble at DSL time */ } diff --git a/jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy b/jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy index a7eed1933..6f33c7389 100644 --- a/jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy +++ b/jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy @@ -57,6 +57,8 @@ class OSRFSourceCreation def canonical_package_name = Globals.get_canonical_package_name( default_params.find{ it.key == "PACKAGE"}.value) + def s3_download_url_basedir = Globals.s3_download_url_basedir( + default_params.find{ it.key == "PACKAGE"}?.value) steps { systemGroovyCommand("""\ @@ -94,7 +96,8 @@ class OSRFSourceCreation exit 1 fi - echo "TARBALL_NAME=\${tarball}" >> ${properties_file} + echo "S3_FILES_TO_UPLOAD=\${tarball}" >> ${properties_file} + echo "SOURCE_TARBALL_URI=$s3_download_url_basedir/\${tarball}" >> ${properties_file} """.stripIndent() ) } @@ -124,8 +127,8 @@ class OSRFSourceCreation parameters { currentBuild() predefinedProps([PROJECT_NAME_TO_COPY_ARTIFACTS: '${JOB_NAME}', - S3_UPLOAD_PATH: Globals.s3_upload_tarball_path(package_name)]) - propertiesFile(properties_file) // TARBALL_NAME + S3_UPLOAD_PATH: Globals.s3_releases_dir(package_name)]) // relative path + propertiesFile(properties_file) // S3_FILES_TO_UPLOAD } } } @@ -134,7 +137,7 @@ class OSRFSourceCreation parameters { currentBuild() predefinedProps([PROJECT_NAME_TO_COPY_ARTIFACTS: "\${JOB_NAME}"]) - propertiesFile(properties_file) // TARBALL_NAME + propertiesFile(properties_file) // SOURCE_TARBALL_URI } } } diff --git a/jenkins-scripts/dsl/test.dsl b/jenkins-scripts/dsl/test.dsl index 08c9a991a..fbe887c0e 100644 --- a/jenkins-scripts/dsl/test.dsl +++ b/jenkins-scripts/dsl/test.dsl @@ -50,8 +50,8 @@ repo_uploader.with parameters { stringParam('PACKAGE','','Package name') - stringParam('TARBALL_NAME', '', 'Tarball name to upload') stringParam('S3_UPLOAD_PATH','', 'S3 path to upload') + stringParam('S3_FILES_TO_UPLOAD','', 'S3 file names to upload') stringParam('UPLOAD_TO_REPO','none','repo to upload') } @@ -59,7 +59,7 @@ repo_uploader.with { copyArtifacts('_test_gz_source') { - includePatterns("${pkg_sources_dir}/\${TARBALL_NAME}") + includePatterns("${pkg_sources_dir}/*") buildSelector { upstreamBuild() } @@ -71,9 +71,27 @@ repo_uploader.with # check that the tarball name actually exist ls -R \${WORKSPACE} - test -f \${WORKSPACE}/${pkg_sources_dir}/\${TARBALL_NAME} - echo "Fake upload of \${TARBALL_NAME} to \${S3_UPLOAD_PATH}" + for pkg in \$(ls ${pkg_sources_dir}/); do + test -f \${WORKSPACE}/${pkg_sources_dir}/\${pkg} + done + + echo "Fake upload of \${S3_FILES_TO_UPLOAD} to \${S3_UPLOAD_PATH}" + # code copied from repository_uploader + pkgs_path="\$WORKSPACE/pkgs" + + for pkg in \${S3_FILES_TO_UPLOAD}; do + # S3_UPLOAD_PATH should be send by the upstream job + if [[ -z \${S3_UPLOAD_PATH} ]]; then + echo "S3_UPLOAD_PATH was not defined. Not uploading" + exit 1 + fi + + # Seems important to upload the path with a final slash + echo "WILL RUN: s3cmd put \${pkgs_path}/\${pkg} s3://osrf-distributions/\${S3_UPLOAD_PATH}" + done + + """.stripIndent()) } } From 45282b84759350ebbcb776b2bff437cd9faf9bde Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 6 Oct 2023 16:49:39 +0200 Subject: [PATCH 457/540] Use PROJECT_NAME_TO_COPY_ARTIFACTS instead of hardcoding test job Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/test.dsl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/test.dsl b/jenkins-scripts/dsl/test.dsl index fbe887c0e..66b93cde4 100644 --- a/jenkins-scripts/dsl/test.dsl +++ b/jenkins-scripts/dsl/test.dsl @@ -53,11 +53,14 @@ repo_uploader.with stringParam('S3_UPLOAD_PATH','', 'S3 path to upload') stringParam('S3_FILES_TO_UPLOAD','', 'S3 file names to upload') stringParam('UPLOAD_TO_REPO','none','repo to upload') + stringParam("PROJECT_NAME_TO_COPY_ARTIFACTS", + "", + "Internal use: parent job name passed by the job to be used in copy artifacts") } steps { - copyArtifacts('_test_gz_source') + copyArtifacts('${PROJECT_NAME_TO_COPY_ARTIFACTS}') { includePatterns("${pkg_sources_dir}/*") buildSelector { From d7c30114da138cd2b149ab5b17fd56d3ab5f9ead Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 6 Oct 2023 17:02:29 +0200 Subject: [PATCH 458/540] Support EXTRA_OSRF_REPO to be passed to OSRFReleasepy Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy | 3 +++ 1 file changed, 3 insertions(+) diff --git a/jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy b/jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy index 6f33c7389..552336e8b 100644 --- a/jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy +++ b/jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy @@ -33,6 +33,9 @@ class OSRFSourceCreation stringParam("UPLOAD_TO_REPO", default_params.find{ it.key == "UPLOAD_TO_REPO"}?.value, "For downstream jobs: OSRF repo name to upload the package to: stable | prerelease | nightly | none (for testing proposes)") + stringParam("EXTRA_OSRF_REPO", + default_params.find{ it.key == "EXTRA_OSRF_REPO"}?.value, + "For downstream jobs: OSRF extra repositories to add") } } } From e29d52bd2742c01dea0eff5a1d4e787498e6be8e Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Mon, 9 Oct 2023 18:53:48 +0200 Subject: [PATCH 459/540] Handle ~ symbols in git tags The ~ symbol used in prereleases is not supported by git tags. Needs to convert it to a -. --- jenkins-scripts/docker/gz-source-generation.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/docker/gz-source-generation.bash b/jenkins-scripts/docker/gz-source-generation.bash index 41669ed51..5aa9eba8b 100644 --- a/jenkins-scripts/docker/gz-source-generation.bash +++ b/jenkins-scripts/docker/gz-source-generation.bash @@ -18,7 +18,7 @@ BUILD_DIR=\$SOURCES_DIR/build cd \${WORKSPACE} rm -fr \$SOURCES_DIR && mkdir \$SOURCES_DIR -git clone --depth 1 --branch ${PACKAGE}_${VERSION} ${SOURCE_REPO_URI} \${SOURCES_DIR} +git clone --depth 1 --branch ${PACKAGE}_${VERSION:~:-} ${SOURCE_REPO_URI} \${SOURCES_DIR} rm -fr \$BUILD_DIR && mkdir \$BUILD_DIR cd \${BUILD_DIR} cmake .. -DPACKAGE_SOURCE_ONLY:BOOL=ON From 532446787927d7c60245822b3c85df685ff58f69 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 10 Oct 2023 17:52:15 +0200 Subject: [PATCH 460/540] Tilde need scaping in bash Signed-off-by: Jose Luis Rivero --- jenkins-scripts/docker/gz-source-generation.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/docker/gz-source-generation.bash b/jenkins-scripts/docker/gz-source-generation.bash index 5aa9eba8b..fefa7db9e 100644 --- a/jenkins-scripts/docker/gz-source-generation.bash +++ b/jenkins-scripts/docker/gz-source-generation.bash @@ -18,7 +18,7 @@ BUILD_DIR=\$SOURCES_DIR/build cd \${WORKSPACE} rm -fr \$SOURCES_DIR && mkdir \$SOURCES_DIR -git clone --depth 1 --branch ${PACKAGE}_${VERSION:~:-} ${SOURCE_REPO_URI} \${SOURCES_DIR} +git clone --depth 1 --branch ${PACKAGE}_${VERSION/\~/-} ${SOURCE_REPO_URI} \${SOURCES_DIR} rm -fr \$BUILD_DIR && mkdir \$BUILD_DIR cd \${BUILD_DIR} cmake .. -DPACKAGE_SOURCE_ONLY:BOOL=ON From b47766595e4b882c67658a6637557b0fbcbfc34d Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Tue, 10 Oct 2023 10:29:59 -0700 Subject: [PATCH 461/540] bump_dependency.bash: use - in version strings (#1028) Replace the `~` separator characters with `-` in homebrew formula version strings to satisfy osrf/homebrew-simulation#2390. Signed-off-by: Steve Peters --- source-repo-scripts/bump_dependency.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source-repo-scripts/bump_dependency.bash b/source-repo-scripts/bump_dependency.bash index 671b4f8f9..f06b29675 100755 --- a/source-repo-scripts/bump_dependency.bash +++ b/source-repo-scripts/bump_dependency.bash @@ -387,7 +387,7 @@ for ((i = 0; i < "${#LIBRARIES[@]}"; i++)); do # version PREV_VER_NONNEGATIVE=$([[ "${PREV_VER}" -lt 0 ]] && echo "0" || echo "${PREV_VER}") sed -i "/ version /d" $FORMULA - sed -i "/^ url.*/a\ version \"${PREV_VER_NONNEGATIVE}.999.999~0~`date +"%Y%m%d"`\"" $FORMULA + sed -i "/^ url.*/a\ version \"${PREV_VER_NONNEGATIVE}.999.999-0-`date +"%Y%m%d"`\"" $FORMULA # Remove extra blank lines cat -s $FORMULA | tee $FORMULA From cee5951fc0ad9bac0dfe3b23e89c8b4bb8fb232b Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 11 Oct 2023 15:59:27 +0200 Subject: [PATCH 462/540] Added missing libs in the __upcoming__ gz-collection.yaml entry Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/gz-collections.yaml | 32 +++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/jenkins-scripts/dsl/gz-collections.yaml b/jenkins-scripts/dsl/gz-collections.yaml index 0eec17016..8a021f2fd 100644 --- a/jenkins-scripts/dsl/gz-collections.yaml +++ b/jenkins-scripts/dsl/gz-collections.yaml @@ -367,10 +367,42 @@ collections: major_version: 11 repo: current_branch: main + - name: gz-rendering + major_version: 8 + repo: + current_branch: main + - name: sdformat + major_version: 15 + repo: + current_branch: main + - name: gz-fuel-tools + major_version: 10 + repo: + current_branch: main + - name: gz-transport + major_version: 14 + repo: + current_branch: main + - name: gz-gui + major_version: 9 + repo: + current_branch: main + - name: gz-sensors + major_version: 9 + repo: + current_branch: main - name: gz-physics major_version: 7 repo: current_branch: main + - name: gz-sim + major_version: 9 + repo: + current_branch: main + - name: gz-launch + major_version: 8 + repo: + current_branch: main ci: configs: - jammy From 5c978ee23d7861d6311c2ad392a0008150bd7081 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 6 Oct 2023 17:18:27 +0200 Subject: [PATCH 463/540] Add support for testing repo releasing Signed-off-by: Jose Luis Rivero --- release.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release.py b/release.py index f76ba66c6..997a826e9 100755 --- a/release.py +++ b/release.py @@ -29,7 +29,7 @@ # the release repositories, when needed. UBUNTU_DISTROS = [] -OSRF_REPOS_SUPPORTED = "stable prerelease nightly" +OSRF_REPOS_SUPPORTED = "stable prerelease nightly testing" OSRF_REPOS_SELF_CONTAINED = "" DRY_RUN = False From 23a7f77922be5a053a942096a546f4a85c52dc57 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 6 Oct 2023 17:26:10 +0200 Subject: [PATCH 464/540] Remove sanity_use_prerelease_branch, obsolete Signed-off-by: Jose Luis Rivero --- release.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/release.py b/release.py index 997a826e9..2b0973a0a 100755 --- a/release.py +++ b/release.py @@ -299,13 +299,6 @@ def sanity_project_package_in_stable(version, repo_name): return -def sanity_use_prerelease_branch(release_branch): - if release_branch == 'prerelease': - error("The use of prerelease branch is now deprecated. Please check internal wiki instructions") - - return - - def check_s3cmd_configuration(): # Need to check if s3cmd is installed try: @@ -325,7 +318,6 @@ def sanity_checks(args, repo_dir): sanity_package_name_underscore(args.package, args.package_alias) sanity_package_name(repo_dir, args.package, args.package_alias) sanity_check_repo_name(args.upload_to_repository) - sanity_use_prerelease_branch(args.release_repo_branch) if not NIGHTLY: check_s3cmd_configuration() From 7da38f3b4d45a5d8d68c21d5dc96980c8c839eef Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 6 Oct 2023 17:26:34 +0200 Subject: [PATCH 465/540] Remove the use of obsolete OSRF_REPOS_SELF_CONTAINED Signed-off-by: Jose Luis Rivero --- release.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/release.py b/release.py index 2b0973a0a..9fe69747e 100755 --- a/release.py +++ b/release.py @@ -30,7 +30,6 @@ UBUNTU_DISTROS = [] OSRF_REPOS_SUPPORTED = "stable prerelease nightly testing" -OSRF_REPOS_SELF_CONTAINED = "" DRY_RUN = False NIGHTLY = False @@ -567,9 +566,6 @@ def go(argv): if args.extra_repo: params['OSRF_REPOS_TO_USE'] += " " + args.extra_repo - if args.upload_to_repository in OSRF_REPOS_SELF_CONTAINED: - params['OSRF_REPOS_TO_USE'] = args.upload_to_repository - if NIGHTLY: params['VERSION'] = 'nightly' # reuse SOURCE_TARBALL_URI to indicate the nightly branch From 59a0e2eb0108e94647841f00882bc0c539ac1e69 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 6 Oct 2023 17:30:15 +0200 Subject: [PATCH 466/540] Move tag repo to a method --- release.py | 43 ++++++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/release.py b/release.py index 9fe69747e..a7da6ac53 100755 --- a/release.py +++ b/release.py @@ -436,6 +436,29 @@ def create_tarball_path(tarball_name, version, builddir, dry_run): return out.split(' ')[0], tarball_fname, tarball_path +def tag_repo(args): + try: + # tilde is not a valid character in git + tag = '%s_%s' % (args.package_alias, args.version.replace('~', '-')) + check_call(['git', 'tag', '-f', tag]) + check_call(['git', 'push', '--tags']) + except ErrorNoPermsRepo: + print('The Git server reports problems with permissions') + print('The branch could be blocked by configuration if you do not have') + print('rights to push code in default branch.') + sys.exit(1) + except ErrorNoUsernameSupplied: + print('git tag could not be committed because you have not configured') + print('your username. Use "git config --username" to set your username.') + sys.exit(1) + except Exception: + print('There was a problem with pushing tags to the git repository') + print('Do you have write perms in the repository?') + sys.exit(1) + + return tag + + def generate_upload_tarball(args): ################################################### # Platform-agnostic stuff. @@ -502,25 +525,7 @@ def generate_upload_tarball(args): # Tag repo os.chdir(sourcedir) - - try: - # tilde is not a valid character in git - tag = '%s_%s' % (args.package_alias, args.version.replace('~', '-')) - check_call(['git', 'tag', '-f', tag]) - check_call(['git', 'push', '--tags']) - except ErrorNoPermsRepo: - print('The Git server reports problems with permissions') - print('The branch could be blocked by configuration if you do not have') - print('rights to push code in default branch.') - sys.exit(1) - except ErrorNoUsernameSupplied: - print('git tag could not be committed because you have not configured') - print('your username. Use "git config --username" to set your username.') - sys.exit(1) - except Exception: - print('There was a problem with pushing tags to the git repository') - print('Do you have write perms in the repository?') - sys.exit(1) + _ = tag_repo(args) # TODO: Consider auto-updating the Ubuntu changelog. It requires # cloning the -release repo, making a change, and pushing it back. From 87385dc116b589ff3e338783db9b5f255cb81a04 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 6 Oct 2023 17:32:02 +0200 Subject: [PATCH 467/540] Remove obsolte NO_SRC_FILE Signed-off-by: Jose Luis Rivero --- release.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/release.py b/release.py index a7da6ac53..7be0f31c3 100755 --- a/release.py +++ b/release.py @@ -34,7 +34,6 @@ DRY_RUN = False NIGHTLY = False PRERELEASE = False -NO_SRC_FILE = False IGNORE_DRY_RUN = True @@ -127,7 +126,6 @@ def parse_args(argv): global DRY_RUN global NIGHTLY global PRERELEASE - global NO_SRC_FILE parser = argparse.ArgumentParser(description='Make releases.') parser.add_argument('package', help='which package to release') @@ -167,14 +165,12 @@ def parse_args(argv): args.package_alias = args.package DRY_RUN = args.dry_run - NO_SRC_FILE = args.no_source_file if args.upload_to_repository == 'nightly': NIGHTLY = True if args.upload_to_repository == 'prerelease': PRERELEASE = True # Nightly do not generate a tar.bz2 file if NIGHTLY: - NO_SRC_FILE = True args.no_source_file = True return args From f930d2a5318da787146f80b0514b7a2c15074bc5 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 6 Oct 2023 17:34:08 +0200 Subject: [PATCH 468/540] Fix dry-run to output real actions Signed-off-by: Jose Luis Rivero --- release.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/release.py b/release.py index 7be0f31c3..482ee00e0 100755 --- a/release.py +++ b/release.py @@ -370,12 +370,8 @@ def discover_distros(repo_dir): def check_call(cmd, ignore_dry_run=False): - if ignore_dry_run: - # Commands that do not change anything in repo level - print('Dry-run running:\n %s\n' % (' '.join(cmd))) - else: - print('Running:\n %s' % (' '.join(cmd))) if DRY_RUN and not ignore_dry_run: + print('Dry-run running:\n %s\n' % (' '.join(cmd))) return b'', b'' else: po = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) From de0a3a4f607003fabef56e3439416119f029ae16 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 6 Oct 2023 17:35:16 +0200 Subject: [PATCH 469/540] Remove is_catking_package. Obsolete Signed-off-by: Jose Luis Rivero --- release.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/release.py b/release.py index 482ee00e0..d657226c8 100755 --- a/release.py +++ b/release.py @@ -86,10 +86,6 @@ def get_canonical_package_name(pkg_name): return pkg_name.rstrip('1234567890') -def is_catkin_package(): - return os.path.isfile("package.xml") - - def github_repo_exists(url): try: check_call(['git', 'ls-remote', '-q', '--exit-code', url], IGNORE_DRY_RUN) @@ -103,9 +99,6 @@ def github_repo_exists(url): def generate_package_source(srcdir, builddir): cmake_cmd = ["cmake"] - if is_catkin_package(): - cmake_cmd = cmake_cmd + ['-DCATKIN_BUILD_BINARY_PACKAGE="1"'] - # configure and make package_source os.mkdir(builddir) os.chdir(builddir) From e3a62d9bd69a22931609c7226967445aaba66317 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 6 Oct 2023 17:52:56 +0200 Subject: [PATCH 470/540] Improve output format Signed-off-by: Jose Luis Rivero --- release.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/release.py b/release.py index d657226c8..687aee484 100755 --- a/release.py +++ b/release.py @@ -76,7 +76,7 @@ def error(msg): def print_success(msg): - print(" + OK " + msg) + print(" + OK " + msg) # Remove the last character if it is a number. @@ -185,7 +185,6 @@ def download_release_repository(package, release_branch): # If main branch exists, prefer it over master if release_branch == "master": if exists_main_branch(url): - print_success('Found main branch in repo, use it instead master') release_branch = 'main' cmd = [vcs, "clone", "-b", release_branch, url, release_tmp_dir] @@ -303,6 +302,7 @@ def check_s3cmd_configuration(): def sanity_checks(args, repo_dir): + print("Safety checks:") sanity_package_name_underscore(args.package, args.package_alias) sanity_package_name(repo_dir, args.package, args.package_alias) sanity_check_repo_name(args.upload_to_repository) @@ -367,6 +367,7 @@ def check_call(cmd, ignore_dry_run=False): print('Dry-run running:\n %s\n' % (' '.join(cmd))) return b'', b'' else: + print('Running:\n %s' % (' '.join(cmd))) po = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = po.communicate() if po.returncode != 0: From 9a6999d311e5b3d44b406a36bd76e8d57c23476f Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 6 Oct 2023 17:53:08 +0200 Subject: [PATCH 471/540] Move Jenkins call to a method Signed-off-by: Jose Luis Rivero --- release.py | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/release.py b/release.py index 687aee484..36fc0c21d 100755 --- a/release.py +++ b/release.py @@ -519,6 +519,16 @@ def generate_upload_tarball(args): return source_tarball_uri, tarball_sha +def call_jenkins_build(job_name, params, output_string): + params_query = urllib.parse.urlencode(params) + url = '%s/job/%s/buildWithParameters?%s' % (JENKINS_URL, + job_name, + params_query) + print(f"- {output_string}: {url}") + if not DRY_RUN: + urllib.request.urlopen(url) + + def go(argv): args = parse_args(argv) @@ -564,18 +574,11 @@ def go(argv): params['SOURCE_TARBALL_URI'] = args.nightly_branch job_name = JOB_NAME_PATTERN % (args.package) - params_query = urllib.parse.urlencode(params) # RELEASING FOR BREW - brew_url = '%s/job/%s/buildWithParameters?%s' % ( - JENKINS_URL, - GENERIC_BREW_PULLREQUEST_JOB, - params_query) - if not NIGHTLY and not args.bump_rev_linux_only: - print('- Brew: %s' % (brew_url)) - if not DRY_RUN: - urllib.request.urlopen(brew_url) + call_jenkins_build(GENERIC_BREW_PULLREQUEST_JOB, + params, 'Brew') # RELEASING FOR LINUX for l in LINUX_DISTROS: @@ -623,13 +626,7 @@ def go(argv): assert a == 'amd64', f'Nightly tag assumed amd64 but arch is {a}' linux_platform_params['JENKINS_NODE_TAG'] = 'linux-nightly-' + d - linux_platform_params_query = urllib.parse.urlencode(linux_platform_params) - - url = '%s/job/%s/buildWithParameters?%s' % (JENKINS_URL, job_name, linux_platform_params_query) - print('- Linux: %s' % (url)) - - if not DRY_RUN: - urllib.request.urlopen(url) + call_jenkins_build(job_name, linux_platform_params, 'Linux') if __name__ == '__main__': From 4351622ab4512b39e229725fbb9669036770c1c7 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 6 Oct 2023 18:02:24 +0200 Subject: [PATCH 472/540] Remove UBUNTU_DISTROS: non use Signed-off-by: Jose Luis Rivero --- release.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/release.py b/release.py index 36fc0c21d..2f4ad8928 100755 --- a/release.py +++ b/release.py @@ -25,10 +25,6 @@ SUPPORTED_ARCHS = ['amd64', 'armhf', 'arm64'] RELEASEPY_NO_ARCH_PREFIX = '.releasepy_NO_ARCH_' -# Ubuntu distributions are automatically taken from the top directory of -# the release repositories, when needed. -UBUNTU_DISTROS = [] - OSRF_REPOS_SUPPORTED = "stable prerelease nightly testing" DRY_RUN = False From 2ff74832c8b4e0382e0f2faa7f1d33270410eabb Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 6 Oct 2023 18:03:28 +0200 Subject: [PATCH 473/540] Remove s3cmd check to facilitate changes Still working but have little value now that new releasers are not accepted. Will facilitate diffs for new changes to come. Signed-off-by: Jose Luis Rivero --- release.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/release.py b/release.py index 2f4ad8928..b6b0e66c9 100755 --- a/release.py +++ b/release.py @@ -282,21 +282,6 @@ def sanity_project_package_in_stable(version, repo_name): return -def check_s3cmd_configuration(): - # Need to check if s3cmd is installed - try: - subprocess.call(["s3cmd", "--version"]) - except OSError: - error("s3cmd command for uploading is not available. Install it using: apt-get install s3cmd") - - # Need to check if configuration for s3 exists - s3_config = os.path.expanduser('~') + "/.s3cfg" - if not os.path.isfile(s3_config): - error(s3_config + " does not exists. Please configure s3: s3cmd --configure") - - return True - - def sanity_checks(args, repo_dir): print("Safety checks:") sanity_package_name_underscore(args.package, args.package_alias) @@ -304,7 +289,6 @@ def sanity_checks(args, repo_dir): sanity_check_repo_name(args.upload_to_repository) if not NIGHTLY: - check_s3cmd_configuration() sanity_package_version(repo_dir, args.version, str(args.release_version)) sanity_check_sdformat_versions(args.package, args.version) sanity_project_package_in_stable(args.version, args.upload_to_repository) From 984607d87695ab1ae62de5a73e6aa5bbf943486f Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Mon, 9 Oct 2023 19:17:13 +0200 Subject: [PATCH 474/540] Added error on non existing release repo --- release.py | 1 + 1 file changed, 1 insertion(+) diff --git a/release.py b/release.py index b6b0e66c9..02c787dc6 100755 --- a/release.py +++ b/release.py @@ -170,6 +170,7 @@ def get_release_repository_info(package): if (github_repo_exists(github_url)): return 'git', github_url + error("release repository not found in github.com/gazebo-release") def download_release_repository(package, release_branch): vcs, url = get_release_repository_info(package) From ade9e709881543f60b67041925dca5a8858dac3b Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Mon, 9 Oct 2023 20:46:59 +0200 Subject: [PATCH 475/540] Output platform combination. Remove JOB_NAME_PATTERN global Signed-off-by: Jose Luis Rivero --- release.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/release.py b/release.py index 02c787dc6..fe0730460 100755 --- a/release.py +++ b/release.py @@ -554,8 +554,6 @@ def go(argv): # name must be modified in the future params['SOURCE_TARBALL_URI'] = args.nightly_branch - job_name = JOB_NAME_PATTERN % (args.package) - # RELEASING FOR BREW if not NIGHTLY and not args.bump_rev_linux_only: call_jenkins_build(GENERIC_BREW_PULLREQUEST_JOB, @@ -607,7 +605,8 @@ def go(argv): assert a == 'amd64', f'Nightly tag assumed amd64 but arch is {a}' linux_platform_params['JENKINS_NODE_TAG'] = 'linux-nightly-' + d - call_jenkins_build(job_name, linux_platform_params, 'Linux') + call_jenkins_build(f"{args.package}-debbuilder", + linux_platform_params, f"{l} {d}/{a}") if __name__ == '__main__': From 636e2ab140d4281e73a7767b975c4ef9f1bb567f Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 11 Oct 2023 18:14:08 +0200 Subject: [PATCH 476/540] Need source-tarball-uri param in release.py Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/_configs_/OSRFReleasepy.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/jenkins-scripts/dsl/_configs_/OSRFReleasepy.groovy b/jenkins-scripts/dsl/_configs_/OSRFReleasepy.groovy index d701641f7..5c7ba392d 100644 --- a/jenkins-scripts/dsl/_configs_/OSRFReleasepy.groovy +++ b/jenkins-scripts/dsl/_configs_/OSRFReleasepy.groovy @@ -74,6 +74,7 @@ class OSRFReleasepy echo "releasing \${n} (from branch \${src_branch})" python3 ./scripts/release.py \${dry_run_str} "\${PACKAGE}" "\${VERSION}" "\${PASS}" \${extra_osrf_repo} \ + --source-tarball-uri \${SOURCE_TARBALL_URI} \ --release-repo-branch \${RELEASE_REPO_BRANCH} \ --upload-to-repo \${UPLOAD_TO_REPO} > log || echo "MARK_AS_UNSTABLE" echo " - done" From 42d5840047a8e88b35a57a1efcbe1e1f1cb3b627 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 11 Oct 2023 19:01:10 +0200 Subject: [PATCH 477/540] Support installation of gz-cmake to generate sources Signed-off-by: Jose Luis Rivero --- jenkins-scripts/docker/gz-source-generation.bash | 4 ++++ jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy | 9 ++++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/jenkins-scripts/docker/gz-source-generation.bash b/jenkins-scripts/docker/gz-source-generation.bash index fefa7db9e..66252259e 100644 --- a/jenkins-scripts/docker/gz-source-generation.bash +++ b/jenkins-scripts/docker/gz-source-generation.bash @@ -16,6 +16,10 @@ PKG_DIR=\$WORKSPACE/pkgs SOURCES_DIR=\$WORKSPACE/sources BUILD_DIR=\$SOURCES_DIR/build +# Need to intall all supported gz-cmake* packages in the platform +(sudo apt-get install -y *gz-cmake* || sudo apt-get install -y *ign-cmake*) || \ + (echo "Can not find any ign-cmake/gz-cmake package" && exit 1) + cd \${WORKSPACE} rm -fr \$SOURCES_DIR && mkdir \$SOURCES_DIR git clone --depth 1 --branch ${PACKAGE}_${VERSION/\~/-} ${SOURCE_REPO_URI} \${SOURCES_DIR} diff --git a/jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy b/jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy index 552336e8b..183574cf4 100644 --- a/jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy +++ b/jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy @@ -36,6 +36,12 @@ class OSRFSourceCreation stringParam("EXTRA_OSRF_REPO", default_params.find{ it.key == "EXTRA_OSRF_REPO"}?.value, "For downstream jobs: OSRF extra repositories to add") + stringParam("LINUX_DISTRO", + default_params.find{ it.key == "LINUX_DISTRO"}?.value, + "Linux distribution to use to generate sources") + stringParam("DISTRO", + default_params.find{ it.key == "DISTRO"}?.value, + "Linux release inside LINUX_DISTRO to generate sources on") } } } @@ -78,10 +84,7 @@ class OSRFSourceCreation shell("""\ #!/bin/bash -xe - # Use Jammy/amd64 as base image to generate sources - export DISTRO=jammy export ARCH=amd64 - /bin/bash -x ./scripts/jenkins-scripts/docker/gz-source-generation.bash """.stripIndent() ) From a4d7d652d14ead7297c42a300b990ebae1d2e559 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 11 Oct 2023 19:05:22 +0200 Subject: [PATCH 478/540] Do not use gz-cmake but gz-plugin in test Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/test.dsl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jenkins-scripts/dsl/test.dsl b/jenkins-scripts/dsl/test.dsl index 66b93cde4..4f3fc45d9 100644 --- a/jenkins-scripts/dsl/test.dsl +++ b/jenkins-scripts/dsl/test.dsl @@ -29,8 +29,8 @@ releasepy_job.with { // gz source testing job def gz_source_job = job("_test_gz_source") OSRFSourceCreation.create(gz_source_job, [ - PACKAGE: "gz-cmake3" , - SOURCE_REPO_URI: "https://github.com/gazebosim/gz-cmake.git"]) + PACKAGE: "gz-plugin2" , + SOURCE_REPO_URI: "https://github.com/gazebosim/gz-plugin.git"]) OSRFSourceCreation.call_uploader_and_releasepy(gz_source_job, '_test_repository_uploader', '_test_releasepy') From db34645f7f5b012e34a799520173ea26ef142952 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 11 Oct 2023 19:15:19 +0200 Subject: [PATCH 479/540] Support OSRF_REPOS_TO_USE Signed-off-by: Jose Luis Rivero --- .../dsl/_configs_/OSRFSourceCreation.groovy | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy b/jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy index 183574cf4..4e61ca7d9 100644 --- a/jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy +++ b/jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy @@ -24,6 +24,15 @@ class OSRFSourceCreation stringParam("VERSION", default_params.find{ it.key == "VERSION"}?.value, "Packages version to be built or nightly (enable nightly build mode)") + stringParam("OSRF_REPOS_TO_USE", + default_params.find{ it.key == "OSRF_REPOS_TO_USE"}?.value, + "OSRF repos name to use when building the package") + stringParam("LINUX_DISTRO", + default_params.find{ it.key == "LINUX_DISTRO"}?.value, + "Linux distribution to use to generate sources") + stringParam("DISTRO", + default_params.find{ it.key == "DISTRO"}?.value, + "Linux release inside LINUX_DISTRO to generate sources on") stringParam("RELEASE_VERSION", default_params.find{ it.key == "RELEASE_VERSION"}?.value, "For downstream jobs: Packages release version") @@ -36,12 +45,6 @@ class OSRFSourceCreation stringParam("EXTRA_OSRF_REPO", default_params.find{ it.key == "EXTRA_OSRF_REPO"}?.value, "For downstream jobs: OSRF extra repositories to add") - stringParam("LINUX_DISTRO", - default_params.find{ it.key == "LINUX_DISTRO"}?.value, - "Linux distribution to use to generate sources") - stringParam("DISTRO", - default_params.find{ it.key == "DISTRO"}?.value, - "Linux release inside LINUX_DISTRO to generate sources on") } } } From edbffdafdc0962e428903a32e514d41411df5192 Mon Sep 17 00:00:00 2001 From: Ian Chen Date: Wed, 11 Oct 2023 15:09:46 -0700 Subject: [PATCH 480/540] Update rendering and physics lib version in upcoming collection (#1031) Signed-off-by: Ian Chen --- jenkins-scripts/dsl/gz-collections.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jenkins-scripts/dsl/gz-collections.yaml b/jenkins-scripts/dsl/gz-collections.yaml index 8a021f2fd..d4c20f9c7 100644 --- a/jenkins-scripts/dsl/gz-collections.yaml +++ b/jenkins-scripts/dsl/gz-collections.yaml @@ -368,7 +368,7 @@ collections: repo: current_branch: main - name: gz-rendering - major_version: 8 + major_version: 9 repo: current_branch: main - name: sdformat @@ -392,7 +392,7 @@ collections: repo: current_branch: main - name: gz-physics - major_version: 7 + major_version: 8 repo: current_branch: main - name: gz-sim From e308f0202523eff2dc58fe46579f0007543b1e93 Mon Sep 17 00:00:00 2001 From: "Addisu Z. Taddese" Date: Thu, 12 Oct 2023 14:41:52 -0500 Subject: [PATCH 481/540] Create ign-ionic collection view and set up nightlies (#1034) Signed-off-by: Addisu Z. Taddese --- jenkins-scripts/dsl/gz-collections.yaml | 77 +++++++++++++++++++++ jenkins-scripts/dsl/ignition.dsl | 17 ++++- jenkins-scripts/dsl/ignition_collection.dsl | 53 +++++++++++++- jenkins-scripts/dsl/sdformat.dsl | 4 +- 4 files changed, 147 insertions(+), 4 deletions(-) diff --git a/jenkins-scripts/dsl/gz-collections.yaml b/jenkins-scripts/dsl/gz-collections.yaml index d4c20f9c7..23b79b306 100644 --- a/jenkins-scripts/dsl/gz-collections.yaml +++ b/jenkins-scripts/dsl/gz-collections.yaml @@ -337,6 +337,83 @@ collections: nightly: distros: - jammy + - name: 'ionic' + libs: + - name: gz-cmake + major_version: 4 + repo: + current_branch: main + - name: gz-tools + major_version: 2 + repo: + current_branch: gz-tools2 + - name: gz-utils + major_version: 3 + repo: + current_branch: main + - name: gz-math + major_version: 8 + repo: + current_branch: main + - name: gz-plugin + major_version: 3 + repo: + current_branch: main + - name: gz-common + major_version: 6 + repo: + current_branch: main + - name: gz-msgs + major_version: 11 + repo: + current_branch: main + - name: gz-rendering + major_version: 9 + repo: + current_branch: main + - name: sdformat + major_version: 15 + repo: + current_branch: main + - name: gz-fuel-tools + major_version: 10 + repo: + current_branch: main + - name: gz-transport + major_version: 14 + repo: + current_branch: main + - name: gz-gui + major_version: 9 + repo: + current_branch: main + - name: gz-sensors + major_version: 9 + repo: + current_branch: main + - name: gz-physics + major_version: 8 + repo: + current_branch: main + - name: gz-sim + major_version: 9 + repo: + current_branch: main + - name: gz-launch + major_version: 8 + repo: + current_branch: main + ci: + configs: + - jammy + packaging: + linux: + package_name: + ignore_major_version: + - gz-ionic + nightly: + distros: + - jammy - name: '__upcoming__' libs: - name: gz-cmake diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index 45f4bafdd..6fde8a44e 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -83,7 +83,22 @@ gz_prerelease_branches = [] // DESC: versioned names to generate debbuild jobs for special cases that // don't appear in gz_branches (like nightly builders or 0-debbuild // jobs for the special cases of foo0 packages) -gz_extra_debbuild = [ 'utils1' // see comment https://github.com/gazebo-tooling/release-tools/pull/431#issuecomment-815099918 +gz_extra_debbuild = [ 'cmake4', + 'common6', + 'fuel-tools10', + 'gui9', + 'launch8', + 'math8', + 'msgs11', + 'physics8', + 'plugin3', + 'rendering9', + 'sensors9', + 'sim9', + 'transport14', + 'utils3', + + 'utils1' // see comment https://github.com/gazebo-tooling/release-tools/pull/431#issuecomment-815099918 ] // DESC: exclude ignition from generate any install testing job gz_no_pkg_yet = [ ] diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index e30fcaff5..6d2ac3b7c 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -11,7 +11,7 @@ arch = 'amd64' file = readFileFromWorkspace("scripts/jenkins-scripts/dsl/gz-collections.yaml") gz_collections_yaml = new Yaml().load(file) -gz_nightly = 'harmonic' +gz_nightly = 'ionic' String get_debbuilder_name(parsed_yaml_lib, parsed_yaml_packaging) { @@ -325,6 +325,57 @@ gz_collection_jobs = 'sdformat-ci-sdformat14-homebrew-amd64', 'sdformat-sdf-14-win' ], + 'ionic' : [ + 'ign_cmake-gz-4-win', + 'ign_common-gz-6-win', + 'ign_fuel-tools-gz-10-win', + 'ign_gazebo-gz-9-win', + 'ign_gui-gz-9-win', + 'ign_launch-gz-8-win', + 'ign_math-gz-8-win', + 'ign_msgs-gz-11-win', + 'ign_physics-gz-8-win', + 'ign_plugin-gz-3-win', + 'ign_rendering-gz-9-win', + 'ign_sensors-gz-9-win', + 'ign_tools-gz-2-win', + 'ign_transport-gz-14-win', + 'ign_utils-gz-3-win', + 'gz_cmake-ci-gz-cmake4-jammy-amd64', + 'gz_common-ci-gz-common6-jammy-amd64', + 'gz_fuel_tools-ci-gz-fuel-tools10-jammy-amd64', + 'gz_gui-ci-gz-gui9-jammy-amd64', + 'gz_launch-ci-gz-launch8-jammy-amd64', + 'gz_math-ci-gz-math8-jammy-amd64', + 'gz_msgs-ci-gz-msgs11-jammy-amd64', + 'gz_physics-ci-gz-physics8-jammy-amd64', + 'gz_plugin-ci-gz-plugin3-jammy-amd64', + 'gz_rendering-ci-gz-rendering9-jammy-amd64', + 'gz_sensors-ci-gz-sensors9-jammy-amd64', + 'gz_sim-ci-gz-sim9-jammy-amd64', + 'gz_tools-ci-gz-tools2-jammy-amd64', + 'gz_transport-ci-gz-transport14-jammy-amd64', + 'gz_utils-ci-gz-utils3-jammy-amd64', + 'ignition_cmake-ci-gz-cmake4-homebrew-amd64', + 'ignition_common-ci-gz-common6-homebrew-amd64', + 'ignition_fuel-tools-ci-gz-fuel-tools10-homebrew-amd64', + 'ignition_ionic-ci-main-homebrew-amd64', + 'ignition_gazebo-ci-gz-sim9-homebrew-amd64', + 'ignition_gui-ci-gz-gui9-homebrew-amd64', + 'ignition_launch-gz-launch8-homebrew-amd64', + 'ignition_math-ci-gz-math8-homebrew-amd64', + 'ignition_msgs-ci-gz-msgs11-homebrew-amd64', + 'ignition_physics-ci-gz-physics8-homebrew-amd64', + 'ignition_plugin-ci-gz-plugin3-homebrew-amd64', + 'ignition_rendering-ci-gz-rendering9-homebrew-amd64', + 'ignition_sensors-ci-gz-sensors9-homebrew-amd64', + 'ignition_tools-ci-gz-tools2-homebrew-amd64', + 'ignition_transport-ci-gz-transport14-homebrew-amd64', + 'ignition_utils-ci-gz-utils3-homebrew-amd64', + 'sdformat-ci-sdformat15-jammy-amd64', + 'sdformat-ci-sdformat15-homebrew-amd64', + 'sdformat-sdf-15-win' + ], ] def DISABLE_TESTS = false diff --git a/jenkins-scripts/dsl/sdformat.dsl b/jenkins-scripts/dsl/sdformat.dsl index d983d18f5..f93b42396 100644 --- a/jenkins-scripts/dsl/sdformat.dsl +++ b/jenkins-scripts/dsl/sdformat.dsl @@ -2,9 +2,9 @@ import _configs_.* import javaposse.jobdsl.dsl.Job def sdformat_supported_versions = [ 'sdformat6' , 'sdformat9', 'sdformat12', 'sdformat13', 'sdformat14' ] -def sdformat_gz11_versions = [ 'sdformat9', 'sdformat12', 'sdformat13', 'main' ] +def sdformat_gz11_versions = [ 'sdformat9', 'sdformat12', 'sdformat13', 'sdformat14', 'main' ] // nightly and prereleases -def extra_sdformat_debbuilder = [] +def extra_sdformat_debbuilder = ['sdformat15'] // Main platform using for quick CI def ci_distro = Globals.get_ci_distro() From 2fa3b6585418c96be2ee003d5ecccc26ff0c7b7c Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 12 Oct 2023 21:53:44 +0200 Subject: [PATCH 482/540] Add _releasepy job Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/core.dsl | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/jenkins-scripts/dsl/core.dsl b/jenkins-scripts/dsl/core.dsl index 8c273c291..e6106298d 100644 --- a/jenkins-scripts/dsl/core.dsl +++ b/jenkins-scripts/dsl/core.dsl @@ -143,3 +143,13 @@ outdated_job_runner.with systemGroovyCommand(readFileFromWorkspace('scripts/jenkins-scripts/tools/outdated-job-runner.groovy')) } } + +// ------------------------------------------------------------------- +def releasepy_job = job("_releasepy") +OSRFReleasepy.create(releasepy_job) +releasepy_job.with { + blockOn("repository_uploader_packages") { + blockLevel('GLOBAL') + scanQueueFor('ALL') + } +} From 4a29c4ccf3e439d8769feb6ae48ff9f6b5edf9a5 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 12 Oct 2023 22:12:13 +0200 Subject: [PATCH 483/540] Need to provide a default Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/_configs_/OSRFReleasepy.groovy | 1 - jenkins-scripts/dsl/core.dsl | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/jenkins-scripts/dsl/_configs_/OSRFReleasepy.groovy b/jenkins-scripts/dsl/_configs_/OSRFReleasepy.groovy index 5c7ba392d..e799ddea1 100644 --- a/jenkins-scripts/dsl/_configs_/OSRFReleasepy.groovy +++ b/jenkins-scripts/dsl/_configs_/OSRFReleasepy.groovy @@ -37,7 +37,6 @@ class OSRFReleasepy stringParam("EXTRA_OSRF_REPO", default_params.find{ it.key == "OSRF_REPOS_TO_USE"}?.value, "OSRF repos name to use when building the package") - booleanParam('DRY_RUN', default_params.find{ it.key == "DRY_RUN"}?.value, 'run a testing run with no effects') diff --git a/jenkins-scripts/dsl/core.dsl b/jenkins-scripts/dsl/core.dsl index e6106298d..add1a9785 100644 --- a/jenkins-scripts/dsl/core.dsl +++ b/jenkins-scripts/dsl/core.dsl @@ -146,7 +146,7 @@ outdated_job_runner.with // ------------------------------------------------------------------- def releasepy_job = job("_releasepy") -OSRFReleasepy.create(releasepy_job) +OSRFReleasepy.create(releasepy_job, [DRY_RUN: false]) releasepy_job.with { blockOn("repository_uploader_packages") { blockLevel('GLOBAL') From e424350d0a048895ee208d3f4ddbe9e37050330f Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 3 Oct 2023 18:14:47 +0200 Subject: [PATCH 484/540] Do not run with bash output enabled Signed-off-by: Jose Luis Rivero --- bloom/ros_gz/bloom_from_special_env.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bloom/ros_gz/bloom_from_special_env.bash b/bloom/ros_gz/bloom_from_special_env.bash index 63be8c767..5c12dc2d3 100755 --- a/bloom/ros_gz/bloom_from_special_env.bash +++ b/bloom/ros_gz/bloom_from_special_env.bash @@ -1,4 +1,4 @@ -#!/bin/bash -ex +#!/bin/bash -e if [[ $# -lt 3 ]]; then echo "Usage: $0 " From 40963769b61d75537a062cad67a86778170f68e5 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 3 Oct 2023 18:14:59 +0200 Subject: [PATCH 485/540] Fix check command for tags Signed-off-by: Jose Luis Rivero --- bloom/ros_gz/_check_metadata.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bloom/ros_gz/_check_metadata.bash b/bloom/ros_gz/_check_metadata.bash index 1bf1db53f..0d7b025db 100755 --- a/bloom/ros_gz/_check_metadata.bash +++ b/bloom/ros_gz/_check_metadata.bash @@ -15,7 +15,7 @@ TMP_DIR=$(mktemp -d) git clone -q "${RELEASE_REPO_URL}" "${TMP_DIR}" pushd "${TMP_DIR}" 2> /dev/null > /dev/null || exit DEBIAN_LATEST_TAG=$(git for-each-ref --sort=creatordate --format '%(refname)' refs/tags | \ - grep -v 'ros-ign-' | grep "debian/ros-${ROS_DISTRO}" \ + grep -v 'ros-ign' | grep "debian/ros-${ROS_DISTRO}" | \ tail -1 | sed 's:refs/tags/::') git checkout -q "${DEBIAN_LATEST_TAG}" if ! dpkg-parsechangelog -SSource 2> /dev/null | grep -q "gz${GAZEBO_DISTRO}"; then From d6cfee28d196a1901fdbd5179ba4ea1193a42c3c Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 3 Oct 2023 18:15:21 +0200 Subject: [PATCH 486/540] Include release commands in README Signed-off-by: Jose Luis Rivero --- bloom/ros_gz/README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/bloom/ros_gz/README.md b/bloom/ros_gz/README.md index eba4bf4e9..0f44b2c69 100644 --- a/bloom/ros_gz/README.md +++ b/bloom/ros_gz/README.md @@ -29,11 +29,11 @@ newer version of Gazebo than the one selected in REP-2000. ### List of active relases -| Gazebo Release | ROS / ROS 2 Release | status | ros_gz branch | -release repository | -| ---------------|---------------------|------------|---------------|---------------------| -| Garden | Humble | stable | humble | https://github.com/gazebo-release/ros_ign-release | -| Garden | Iron | prerelease | iron | https://github.com/gazebo-release/ros_ign-release | -| Harmonic | Iron | prerelease | ros2 | https://github.com/j-rivero/ros_ign-gzharmonic-release | +| Gazebo Release | ROS / ROS 2 Release | status | ros_gz branch | -release repository | release cmd | +| ---------------|---------------------|------------|---------------|---------------------|-------------| +| Garden | Humble | stable | humble | https://github.com/gazebo-release/ros_ign-release |
```RELEASE_REPO_URL=https://github.com/gazebo-release/ros_ign-release ./bloom_from_special_env.bash humble garden https://raw.githubusercontent.com/osrf/osrf-rosdep/master/gz/replace_fortress_with_garden/00-replace-gz-fortress-with-garden.list```
| +| Garden | Iron | prerelease | iron | https://github.com/gazebo-release/ros_ign-release |
```RELEASE_REPO_URL=https://github.com/gazebo-release/ros_ign-release ./bloom_from_special_env.bash iron garden https://raw.githubusercontent.com/osrf/osrf-rosdep/master/gz/replace_fortress_with_garden/00-replace-gz-fortress-with-garden.list```
| +| Harmonic | Iron | stable | ros2 | https://github.com/gazebo-release/ros_gz-gzharmonic-release|
```RELEASE_REPO_URL=https://github.com/gazebo-release/ros_gz-gzharmonic-release ./bloom_from_special_env.bash iron harmonic https://raw.githubusercontent.com/osrf/osrf-rosdep```
| ### Upstream versions released using this tutorial From 440ec26a2c98a84d97bd483720a818d96f340c62 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 3 Oct 2023 19:24:11 +0200 Subject: [PATCH 487/540] Support ros_gz install testing for Harmonic Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/ros_gz_bridge.dsl | 50 ++++++++++++++++++++------- 1 file changed, 38 insertions(+), 12 deletions(-) diff --git a/jenkins-scripts/dsl/ros_gz_bridge.dsl b/jenkins-scripts/dsl/ros_gz_bridge.dsl index 80492d865..0d3c470d6 100644 --- a/jenkins-scripts/dsl/ros_gz_bridge.dsl +++ b/jenkins-scripts/dsl/ros_gz_bridge.dsl @@ -1,6 +1,8 @@ import _configs_.* import javaposse.jobdsl.dsl.Job +def WRITE_JOB_LOG = System.getenv('WRITE_JOB_LOG') ?: false + def bridge_packages = [ 'ros_gz', 'ros_gz_bridge', @@ -15,6 +17,14 @@ def gzgarden_ros_distros_ci = [ 'iron' ] +def unofficial_combinations = [ + 'garden' : ['humble', 'iron'], + 'harmonic' : ['iron'] +] + +logging_list = [:] +logging_list['unofficial_wrappers_install_pkg_ci'] = [] + // BLOOM PACKAGE BUILDER JOBS bridge_packages.each { pkg -> pkg_dashed = pkg.replaceAll("_", "-") @@ -196,20 +206,36 @@ manual_install_test_job.with } } -gzgarden_ros_distros_ci.each { ros_distro -> - def periodic_install_test_job = job("ros_gzgarden_bridge-install-pkg_${ros_distro}-ci-jammy-amd64") - generate_install_test_job(periodic_install_test_job) - periodic_install_test_job.with - { - parameters { - stringParam("DISTRO", 'jammy', "Linux release inside LINUX_DISTRO to build packages for") - stringParam("ARCH", 'amd64', "Architecture to build packages for") - stringParam("ROS_DISTRO", ros_distro, "ROS distribution") - stringParam("GZ_VERSION", 'garden', "Gazebo version") +unofficial_combinations.each { gz_release, ros_distros -> + ros_distros.each { ros_distro -> + def periodic_install_test_job = job("ros_gz${gz_release}_bridge-install-pkg_${ros_distro}-ci-jammy-amd64") + generate_install_test_job(periodic_install_test_job) + periodic_install_test_job.with + { + parameters { + stringParam("DISTRO", 'jammy', "Linux release inside LINUX_DISTRO to build packages for") + stringParam("ARCH", 'amd64', "Architecture to build packages for") + stringParam("ROS_DISTRO", ros_distro, "ROS distribution") + stringParam("GZ_VERSION", gz_release, "Gazebo version") + } + + triggers { + scm('@daily') + } } - triggers { - scm('@daily') + logging_list['unofficial_wrappers_install_pkg_ci'].add( + [collection: gz_release, + job_name: periodic_install_test_job.name]) + } +} + + +if (WRITE_JOB_LOG) { + File log_file = new File("jobs.txt") + log_file.withWriter{ file_writer -> + logging_list.each { log_type, items -> + items.each {file_writer.println "${log_type} ${it.collection} ${it.job_name}"} } } } From 914ea4020b5986356b9503cc7914bf7d75f52f76 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 3 Oct 2023 19:28:09 +0200 Subject: [PATCH 488/540] Use choiceParam for non-modificable params Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/ros_gz_bridge.dsl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/jenkins-scripts/dsl/ros_gz_bridge.dsl b/jenkins-scripts/dsl/ros_gz_bridge.dsl index 0d3c470d6..5cf3fb762 100644 --- a/jenkins-scripts/dsl/ros_gz_bridge.dsl +++ b/jenkins-scripts/dsl/ros_gz_bridge.dsl @@ -213,10 +213,10 @@ unofficial_combinations.each { gz_release, ros_distros -> periodic_install_test_job.with { parameters { - stringParam("DISTRO", 'jammy', "Linux release inside LINUX_DISTRO to build packages for") - stringParam("ARCH", 'amd64', "Architecture to build packages for") - stringParam("ROS_DISTRO", ros_distro, "ROS distribution") - stringParam("GZ_VERSION", gz_release, "Gazebo version") + choiceParam("DISTRO", 'jammy', "Linux release inside LINUX_DISTRO to build packages for") + choiceParam("ARCH", 'amd64', "Architecture to build packages for") + choiceParam("ROS_DISTRO", ros_distro, "ROS distribution") + choiceParam("GZ_VERSION", gz_release, "Gazebo version") } triggers { From 74f0c8fd136a34d476330c77d51eebd3def4b3ea Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 3 Oct 2023 19:35:11 +0200 Subject: [PATCH 489/540] Fix choiceParam syntax Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/ros_gz_bridge.dsl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/jenkins-scripts/dsl/ros_gz_bridge.dsl b/jenkins-scripts/dsl/ros_gz_bridge.dsl index 5cf3fb762..0f4ca14aa 100644 --- a/jenkins-scripts/dsl/ros_gz_bridge.dsl +++ b/jenkins-scripts/dsl/ros_gz_bridge.dsl @@ -213,10 +213,10 @@ unofficial_combinations.each { gz_release, ros_distros -> periodic_install_test_job.with { parameters { - choiceParam("DISTRO", 'jammy', "Linux release inside LINUX_DISTRO to build packages for") - choiceParam("ARCH", 'amd64', "Architecture to build packages for") - choiceParam("ROS_DISTRO", ros_distro, "ROS distribution") - choiceParam("GZ_VERSION", gz_release, "Gazebo version") + choiceParam("DISTRO", ['jammy'], "Linux release inside LINUX_DISTRO to build packages for") + choiceParam("ARCH", ['amd64'], "Architecture to build packages for") + choiceParam("ROS_DISTRO", [ros_distro], "ROS distribution") + choiceParam("GZ_VERSION", [gz_release], "Gazebo version") } triggers { From 4b45e74588735b8ae166418858ea0b7cd347eefd Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 4 Oct 2023 15:56:56 +0200 Subject: [PATCH 490/540] Do not override log file entries Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/gazebo_libs.dsl | 6 ++---- jenkins-scripts/dsl/ros_gz_bridge.dsl | 6 ++---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/jenkins-scripts/dsl/gazebo_libs.dsl b/jenkins-scripts/dsl/gazebo_libs.dsl index cff29f441..f76bd5bb6 100644 --- a/jenkins-scripts/dsl/gazebo_libs.dsl +++ b/jenkins-scripts/dsl/gazebo_libs.dsl @@ -201,9 +201,7 @@ configs_per_lib_index.each { lib_name, lib_configs -> if (WRITE_JOB_LOG) { File log_file = new File("jobs.txt") - log_file.withWriter{ file_writer -> - logging_list.each { log_type, items -> - items.each {file_writer.println "${log_type} ${it.collection} ${it.job_name}"} - } + logging_list.each { log_type, items -> + items.each { log_file.append("${log_type} ${it.collection} ${it.job_name}\n") } } } diff --git a/jenkins-scripts/dsl/ros_gz_bridge.dsl b/jenkins-scripts/dsl/ros_gz_bridge.dsl index 0f4ca14aa..bc98b3a4b 100644 --- a/jenkins-scripts/dsl/ros_gz_bridge.dsl +++ b/jenkins-scripts/dsl/ros_gz_bridge.dsl @@ -233,9 +233,7 @@ unofficial_combinations.each { gz_release, ros_distros -> if (WRITE_JOB_LOG) { File log_file = new File("jobs.txt") - log_file.withWriter{ file_writer -> - logging_list.each { log_type, items -> - items.each {file_writer.println "${log_type} ${it.collection} ${it.job_name}"} - } + logging_list.each { log_type, items -> + items.each { log_file.append("${log_type} ${it.collection} ${it.job_name}\n") } } } From 71c54429c0793f47274c57e75e96bff159949651 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 6 Oct 2023 12:52:13 +0200 Subject: [PATCH 491/540] Include CI on README --- bloom/ros_gz/README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/bloom/ros_gz/README.md b/bloom/ros_gz/README.md index 0f44b2c69..09fdd9a08 100644 --- a/bloom/ros_gz/README.md +++ b/bloom/ros_gz/README.md @@ -29,11 +29,11 @@ newer version of Gazebo than the one selected in REP-2000. ### List of active relases -| Gazebo Release | ROS / ROS 2 Release | status | ros_gz branch | -release repository | release cmd | -| ---------------|---------------------|------------|---------------|---------------------|-------------| -| Garden | Humble | stable | humble | https://github.com/gazebo-release/ros_ign-release |
```RELEASE_REPO_URL=https://github.com/gazebo-release/ros_ign-release ./bloom_from_special_env.bash humble garden https://raw.githubusercontent.com/osrf/osrf-rosdep/master/gz/replace_fortress_with_garden/00-replace-gz-fortress-with-garden.list```
| -| Garden | Iron | prerelease | iron | https://github.com/gazebo-release/ros_ign-release |
```RELEASE_REPO_URL=https://github.com/gazebo-release/ros_ign-release ./bloom_from_special_env.bash iron garden https://raw.githubusercontent.com/osrf/osrf-rosdep/master/gz/replace_fortress_with_garden/00-replace-gz-fortress-with-garden.list```
| -| Harmonic | Iron | stable | ros2 | https://github.com/gazebo-release/ros_gz-gzharmonic-release|
```RELEASE_REPO_URL=https://github.com/gazebo-release/ros_gz-gzharmonic-release ./bloom_from_special_env.bash iron harmonic https://raw.githubusercontent.com/osrf/osrf-rosdep```
| +| Gazebo Release | ROS / ROS 2 Release | status | ros_gz branch | -release repository | CI | release cmd | +| ---------------|---------------------|------------|---------------|---------------------|----|-------------| +| Garden | Humble | stable | humble | https://github.com/gazebo-release/ros_ign-release | [![Build Status](https://build.osrfoundation.org/buildStatus/icon?job=ros_gzgarden_bridge-install-pkg_humble-ci-jammy-amd64)](https://build.osrfoundation.org/job/ros_gzgarden_bridge-install-pkg_humble-ci-jammy-amd64/) |
```RELEASE_REPO_URL=https://github.com/gazebo-release/ros_ign-release ./bloom_from_special_env.bash humble garden https://raw.githubusercontent.com/osrf/osrf-rosdep/master/gz/replace_fortress_with_garden/00-replace-gz-fortress-with-garden.list```
| +| Garden | Iron | stable | iron | https://github.com/gazebo-release/ros_ign-release | [![Build Status](https://build.osrfoundation.org/buildStatus/icon?job=ros_gzgarden_bridge-install-pkg_iron-ci-jammy-amd64)](https://build.osrfoundation.org/job/ros_gzgarden_bridge-install-pkg_iron-ci-jammy-amd64/) |
```RELEASE_REPO_URL=https://github.com/gazebo-release/ros_ign-release ./bloom_from_special_env.bash iron garden https://raw.githubusercontent.com/osrf/osrf-rosdep/master/gz/replace_fortress_with_garden/00-replace-gz-fortress-with-garden.list```
| +| Harmonic | Iron | stable | ros2 | https://github.com/gazebo-release/ros_gz-gzharmonic-release | [![Build Status](https://build.osrfoundation.org/buildStatus/icon?job=ros_gzharmonic_bridge-install-pkg_iron-ci-jammy-amd64)](https://build.osrfoundation.org/job/ros_gzharmonic_bridge-install-pkg_iron-ci-jammy-amd64/) |
```RELEASE_REPO_URL=https://github.com/gazebo-release/ros_gz-gzharmonic-release ./bloom_from_special_env.bash iron harmonic https://raw.githubusercontent.com/osrf/osrf-rosdep```
| ### Upstream versions released using this tutorial From 56897bf8d451fd490b4d79290a5a308aab04c8c8 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 12 Oct 2023 23:22:03 +0200 Subject: [PATCH 492/540] Remove nightly generation for Harmonic Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/gz-collections.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/jenkins-scripts/dsl/gz-collections.yaml b/jenkins-scripts/dsl/gz-collections.yaml index 23b79b306..18d3582c9 100644 --- a/jenkins-scripts/dsl/gz-collections.yaml +++ b/jenkins-scripts/dsl/gz-collections.yaml @@ -334,9 +334,6 @@ collections: package_name: ignore_major_version: - gz-harmonic - nightly: - distros: - - jammy - name: 'ionic' libs: - name: gz-cmake From 96de6743907850f9a6d168a8e7dd3a54222d2802 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 12 Oct 2023 23:56:04 +0200 Subject: [PATCH 493/540] Generate source jobs using the new index (#1036) Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/gazebo_libs.dsl | 55 +++++++++++++++--- jenkins-scripts/dsl/gz-collections.yaml | 62 ++++++++++++++++----- jenkins-scripts/dsl/ignition_collection.dsl | 2 +- 3 files changed, 94 insertions(+), 25 deletions(-) diff --git a/jenkins-scripts/dsl/gazebo_libs.dsl b/jenkins-scripts/dsl/gazebo_libs.dsl index f76bd5bb6..bc7f3e92d 100644 --- a/jenkins-scripts/dsl/gazebo_libs.dsl +++ b/jenkins-scripts/dsl/gazebo_libs.dsl @@ -55,12 +55,11 @@ boolean is_testing_enabled(lib_name, ci_config) return ! ci_config.tests_disabled?.contains(lib_name) } - /* * Generate an index that facilitates the operations with the yaml values, * avoiding to parse them several times. * - * ci_configs_by_lib index structure: + * # ci_configs_by_lib index structure: * lib_name : [ ci_config_name : [ .branch .collection ] ] * * The index main keys are the lib names (i.e: gz-cmake) and associated them @@ -71,16 +70,33 @@ boolean is_testing_enabled(lib_name, ci_config) * * index[gz-cmake][jammy] -> [ branch: gz-cmake3, collection: garden , * branch: gz-cmake3, collection: harmonic] + * + * # pkgconf_per_src_inde index structure: + * pkg_src_name : [ packaging_config_name : [ .lib_name .collection ] ] + * + * The index main keys are package source names (i.e gz-cmake3 or gz-harmonic and associated them + * another map of packaging configuration names supported as keys (i.e: jammy) with the + * list of associated items composed by a map: lib_name (canonical name for the source package) + * (and collection) + * + * index[gz-cmake3][jammy] -> [ lib_name: gz-cmake, collection: harmonic ] */ -void generate_ciconfigs_by_lib(config, configs_per_lib_index) +void generate_ciconfigs_by_lib(config, ciconf_per_lib_index, pkgconf_per_src_index) { config.collections.each { collection -> collection.libs.each { lib -> def libName = lib.name def branch = lib.repo.current_branch collection.ci.configs.each { config_name -> - configs_per_lib_index["$libName"]["${config_name}"] = configs_per_lib_index["$libName"]["${config_name}"]?: [] - configs_per_lib_index["$libName"]["${config_name}"].contains(branch) ?: configs_per_lib_index["$libName"]["${config_name}"] << [branch: branch, collection: collection.name] + ciconf_per_lib_index[libName][config_name] = ciconf_per_lib_index[libName][config_name]?: [] + ciconf_per_lib_index[libName][config_name].contains(branch) ?: ciconf_per_lib_index[libName][config_name] << [branch: branch, collection: collection.name] + } + def pkg_name = lib.name + lib.major_version + if (collection.packaging.linux?.ignore_major_version?.contains(libName)) + pkg_name = lib.name + collection.packaging.configs?.each { config_name -> + pkgconf_per_src_index[pkg_name][config_name] = pkgconf_per_src_index[pkg_name][config_name]?: [] + pkgconf_per_src_index[pkg_name][config_name] << [ lib_name: libName, collection: collection.name ] } } } @@ -122,11 +138,12 @@ void generate_ci_job(gz_ci_job, lib_name, branch, ci_config, } } -def configs_per_lib_index = [:].withDefault { [:] } -generate_ciconfigs_by_lib(gz_collections_yaml, configs_per_lib_index) +def ciconf_per_lib_index = [:].withDefault { [:] } +def pkgconf_per_src_index = [:].withDefault { [:] } +generate_ciconfigs_by_lib(gz_collections_yaml, ciconf_per_lib_index, pkgconf_per_src_index) // Generate PR jobs: 1 per ci configuration on each lib -configs_per_lib_index.each { lib_name, lib_configs -> +ciconf_per_lib_index.each { lib_name, lib_configs -> lib_configs.each { ci_configs -> def config_name = ci_configs.getKey() def ci_config = gz_collections_yaml.ci_configs.find{ it.name == config_name } @@ -141,6 +158,7 @@ configs_per_lib_index.each { lib_name, lib_configs -> if (ci_config.exclude.contains(lib_name)) return + // Main PR jobs (-ci-pr_any-) (pulling check every 5 minutes) // -------------------------------------------------------------- def distro = ci_config.system.version @@ -180,7 +198,6 @@ configs_per_lib_index.each { lib_name, lib_configs -> // TODO: remove after testing if (branch_and_collection.collection != 'harmonic') return - branch_name = branch_and_collection.branch def gz_ci_job = job("${gz_job_name_prefix}-ci-${branch_name}-${distro}-${arch}") generate_ci_job(gz_ci_job, lib_name, branch_name, ci_config) @@ -199,6 +216,26 @@ configs_per_lib_index.each { lib_name, lib_configs -> } //en of lib_configs } // end of lib +pkgconf_per_src_index.each { pkg_src, pkg_src_configs -> + pkg_src_configs.each { pkg_src_config -> + def config_name = pkg_src_config.getKey() + def pkg_config = gz_collections_yaml.packaging_configs.find{ it.name == config_name } + // lib_names are the same in all the entries + def canonical_lib_name = pkg_src_config.getValue()[0].lib_name + + if (pkg_config.exclude?.contains(canonical_lib_name)) + return + + def gz_source_job = job("${pkg_src}-source") + OSRFSourceCreation.create(gz_source_job, [ + PACKAGE: pkg_src , + SOURCE_REPO_URI: "https://github.com/gazebosim/${canonical_lib_name}.git"]) + OSRFSourceCreation.call_uploader_and_releasepy(gz_source_job, + 'repository_uploader_packages', + '_releasepy') + } +} + if (WRITE_JOB_LOG) { File log_file = new File("jobs.txt") logging_list.each { log_type, items -> diff --git a/jenkins-scripts/dsl/gz-collections.yaml b/jenkins-scripts/dsl/gz-collections.yaml index 18d3582c9..f3a37adf4 100644 --- a/jenkins-scripts/dsl/gz-collections.yaml +++ b/jenkins-scripts/dsl/gz-collections.yaml @@ -96,8 +96,9 @@ collections: configs: - bionic packaging: + configs: + - bionic linux: - package_name: ignore_major_version: - ign-citadel - name: 'fortress' @@ -174,10 +175,11 @@ collections: configs: - focal packaging: + configs: + - focal linux: - package_name: - ignore_major_version: - - gz-fortress + ignore_major_version: + - gz-fortress - name: 'garden' libs: - name: gz-cmake @@ -252,10 +254,11 @@ collections: configs: - focal packaging: + configs: + - focal linux: - package_name: - ignore_major_version: - - gz-garden + ignore_major_version: + - gz-garden - name: 'harmonic' libs: - name: gz-cmake @@ -330,10 +333,11 @@ collections: configs: - jammy packaging: + configs: + - jammy linux: - package_name: - ignore_major_version: - - gz-harmonic + ignore_major_version: + - gz-harmonic - name: 'ionic' libs: - name: gz-cmake @@ -405,12 +409,11 @@ collections: - jammy packaging: linux: - package_name: - ignore_major_version: - - gz-ionic + ignore_major_version: + - gz-ionic nightly: - distros: - - jammy + distros: + - jammy - name: '__upcoming__' libs: - name: gz-cmake @@ -481,6 +484,8 @@ collections: configs: - jammy packaging: + configs: + - jammy exclude: - __upcoming__ ci_configs: @@ -546,3 +551,30 @@ ci_configs: gz-physics: - "export MAKE_JOBS=1" tests_disabled: +packaging_configs: + - name: bionic + system: + so: linux + distribution: ubuntu + version: bionic + arch: + - i386 + - amd64 + - arm64 + - name: focal + system: + so: linux + distribution: ubuntu + version: focal + arch: + - amd64 + - arm64 + - name: jammy + system: + so: linux + distribution: ubuntu + version: jammy + arch: + - amd64 + - arm64 + - armhf diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index 6d2ac3b7c..6273535c3 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -17,7 +17,7 @@ String get_debbuilder_name(parsed_yaml_lib, parsed_yaml_packaging) { major_version = parsed_yaml_lib.major_version - ignore_major_version = parsed_yaml_packaging.linux?.package_name?.ignore_major_version + ignore_major_version = parsed_yaml_packaging.linux?.ignore_major_version if (ignore_major_version && ignore_major_version.contains(parsed_yaml_lib.name)) major_version = "" From 312fa9549e7dfb7f5e9359a5f53fd06e142d36b1 Mon Sep 17 00:00:00 2001 From: "Addisu Z. Taddese" Date: Thu, 12 Oct 2023 17:15:56 -0500 Subject: [PATCH 494/540] Retry brew update to fix errors about another active Homebrew process being active (#1032) Signed-off-by: Addisu Z. Taddese --- jenkins-scripts/lib/_homebrew_base_setup.bash | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/jenkins-scripts/lib/_homebrew_base_setup.bash b/jenkins-scripts/lib/_homebrew_base_setup.bash index 35d2308fc..dfd607e91 100644 --- a/jenkins-scripts/lib/_homebrew_base_setup.bash +++ b/jenkins-scripts/lib/_homebrew_base_setup.bash @@ -16,7 +16,15 @@ fi git -C $(${BREW_BINARY} --repo) fsck export HOMEBREW_UPDATE_TO_TAG=1 -${BREW_BINARY} update + +# There might be a background process that blocks `brew update`, so we try to +# run it several times until it succeeds. +# See https://github.com/Homebrew/brew/issues/1155 +brew_update_retry_count=0 +until ${BREW_BINARY} update || (( brew_update_retry_count++ > 6 )) +do + sleep 10 +done # manually exclude a ruby warning that jenkins thinks is from clang # https://github.com/osrf/homebrew-simulation/issues/1343 ${BREW_BINARY} install ${BREW_BASE_DEPENDCIES} \ From 5143e362923ed691796e01b81de334fb988becb8 Mon Sep 17 00:00:00 2001 From: "Addisu Z. Taddese" Date: Mon, 16 Oct 2023 10:33:45 -0500 Subject: [PATCH 495/540] Increase `brew update` retries and sleep duration. (#1039) Signed-off-by: Addisu Z. Taddese --- jenkins-scripts/lib/_homebrew_base_setup.bash | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jenkins-scripts/lib/_homebrew_base_setup.bash b/jenkins-scripts/lib/_homebrew_base_setup.bash index dfd607e91..da1623468 100644 --- a/jenkins-scripts/lib/_homebrew_base_setup.bash +++ b/jenkins-scripts/lib/_homebrew_base_setup.bash @@ -21,9 +21,9 @@ export HOMEBREW_UPDATE_TO_TAG=1 # run it several times until it succeeds. # See https://github.com/Homebrew/brew/issues/1155 brew_update_retry_count=0 -until ${BREW_BINARY} update || (( brew_update_retry_count++ > 6 )) +until ${BREW_BINARY} update || (( brew_update_retry_count++ > 10 )) do - sleep 10 + sleep 60 done # manually exclude a ruby warning that jenkins thinks is from clang # https://github.com/osrf/homebrew-simulation/issues/1343 From 5a64f14a18841dd1dae65c1938412aa5e786f0ab Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 17 Oct 2023 11:12:31 +0200 Subject: [PATCH 496/540] Fix Fortress branches prefix: gz -> ign (#1040) * Fix Fortress branches prefix: gz -> ign --------- Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/gz-collections.yaml | 32 ++++++++++++------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/jenkins-scripts/dsl/gz-collections.yaml b/jenkins-scripts/dsl/gz-collections.yaml index f3a37adf4..ea5108707 100644 --- a/jenkins-scripts/dsl/gz-collections.yaml +++ b/jenkins-scripts/dsl/gz-collections.yaml @@ -104,37 +104,37 @@ collections: - name: 'fortress' libs: - name: gz-cmake - major_version: 3 + major_version: 2 repo: - current_branch: gz-cmake2 + current_branch: ign-cmake2 - name: gz-tools major_version: 1 repo: - current_branch: gz-tools1 + current_branch: ign-tools1 - name: gz-utils major_version: 1 repo: - current_branch: gz-utils1 + current_branch: ign-utils1 - name: gz-math major_version: 6 repo: - current_branch: gz-math6 + current_branch: ign-math6 - name: gz-plugin major_version: 1 repo: - current_branch: gz-plugin1 + current_branch: ign-plugin1 - name: gz-common major_version: 4 repo: - current_branch: gz-common4 + current_branch: ign-common4 - name: gz-msgs major_version: 8 repo: - current_branch: gz-msgs8 + current_branch: ign-msgs8 - name: gz-rendering major_version: 6 repo: - current_branch: gz-rendering6 + current_branch: ign-rendering6 - name: sdformat major_version: 12 repo: @@ -142,31 +142,31 @@ collections: - name: gz-fuel-tools major_version: 7 repo: - current_branch: gz-fuel-tools7 + current_branch: ign-fuel-tools7 - name: gz-transport major_version: 11 repo: - current_branch: gz-transport11 + current_branch: ign-transport11 - name: gz-gui major_version: 6 repo: - current_branch: gz-gui6 + current_branch: ign-gui6 - name: gz-sensors major_version: 7 repo: - current_branch: gz-sensors7 + current_branch: ign-sensors7 - name: gz-physics major_version: 6 repo: - current_branch: gz-physics6 + current_branch: ign-physics6 - name: gz-sim major_version: 6 repo: - current_branch: gz-sim6 + current_branch: ign-sim6 - name: gz-launch major_version: 5 repo: - current_branch: gz-launch5 + current_branch: ign-launch5 - name: gz-fortress major_version: 1 repo: From 5894221c9ff851323de88681b302e84d6926205d Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 17 Oct 2023 15:55:31 +0200 Subject: [PATCH 497/540] Change abichecker generation to use gz-collection.yaml (#1038) * Update abichecker to work with gz-* * Rename abichecker bash to gz * Implement gazebo_libs jobs for abichecker * Return true from all ignition abicheckerq previous jobs * Add excludes for abichecker * Added a check for main branches * Added a check for abichecker old jobs ---- Signed-off-by: Jose Luis Rivero Signed-off-by: Addisu Z. Taddese --- .github/workflows/ci.yaml | 27 ++++++++++++ ...ion-abichecker.bash => gz-abichecker.bash} | 44 +++++++++---------- jenkins-scripts/dsl/gazebo_libs.dsl | 44 ++++++++++++++++--- jenkins-scripts/dsl/gz-collections.yaml | 29 +++++++++--- jenkins-scripts/dsl/ignition.dsl | 44 ++++--------------- jenkins-scripts/dsl/ignition_collection.dsl | 2 +- 6 files changed, 118 insertions(+), 72 deletions(-) rename jenkins-scripts/docker/{ignition-abichecker.bash => gz-abichecker.bash} (56%) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index f0ee0cdd8..e3caa33c8 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -52,6 +52,33 @@ jobs: sudo chown -R ${USER} /var/lib/jenkins cd jenkins-scripts/dsl WRITE_JOB_LOG=1 java -jar tools/jobdsl.jar *.dsl + - name: Checks for abichecker jobs + if: steps.dsl_check.outputs.run_job == 'true' + run: | + cd jenkins-scripts/dsl + main=$(grep 'main' *-abichecker-*.xml || true) + if [[ -n ${main} ]]; then + echo "Found a main branch in an abichecker job:" + echo "${main}" + echo "Main branches are not target of abichecking. Fix the code." + exit 1 + fi + # Filter out the previous auto jobs + mkdir /tmp/abichecker_filtered_jobs + cp *-abichecker-*.xml /tmp/abichecker_filtered_jobs + cd /tmp/abichecker_filtered_jobs + rm *ubuntu_auto*.xml + repeated=$(grep '\' *-abichecker-*.xml | awk '{ print $2 }' | sort | uniq -d) + if [[ -n ${repeated} ]]; then + echo "Found a duplicate in an abichecker branch:" + echo "${repeated}" + echo "please exclude one of the versions in the yaml file to reduce the server workload" + exit 1 + fi + - name: Export XML generated configuration for diff + if: steps.dsl_check.outputs.run_job == 'true' + run: | + cd jenkins-scripts/dsl # export files for later diff mkdir /tmp/pr_xml_configuration && mkdir /tmp/pr_log_generated/ mv *.xml /tmp/pr_xml_configuration/ diff --git a/jenkins-scripts/docker/ignition-abichecker.bash b/jenkins-scripts/docker/gz-abichecker.bash similarity index 56% rename from jenkins-scripts/docker/ignition-abichecker.bash rename to jenkins-scripts/docker/gz-abichecker.bash index 1383bfd2a..b44a46eee 100644 --- a/jenkins-scripts/docker/ignition-abichecker.bash +++ b/jenkins-scripts/docker/gz-abichecker.bash @@ -14,13 +14,13 @@ if [[ -z ${DISTRO} ]]; then exit 1 fi -# In ignition this variable is passed directly from the DSL job creation +# In Gazebo this variable is passed directly from the DSL job creation if [[ -z ${ABI_JOB_SOFTWARE_NAME} ]]; then echo "ABI_JOB_SOFTWARE_NAME not set!" exit 1 fi -# convert from ign-package to GZ_PACKAGE_DEPENDENCIES +# convert from gz-package to GZ_PACKAGE_DEPENDENCIES GZ_NAME_PREFIX=$(\ echo ${ABI_JOB_SOFTWARE_NAME} | tr '[:lower:]-' '[:upper:]_') ABI_JOB_PKG_DEPENDENCIES_VAR_NAME=${GZ_NAME_PREFIX}_DEPENDENCIES @@ -42,7 +42,7 @@ export ${GZ_NAME_PREFIX}_MAJOR_VERSION=${GZ_NAME_PREFIX_MAJOR_VERSION} # # Set ABI_JOB_HEADER_PREFIX to match the last subfolder: # gazebo6, math6, math7, sim7 -if [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-gazebo" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -ge 7 ]] +if [[ "${ABI_JOB_SOFTWARE_NAME}" = "gz-sim" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -ge 7 ]] then # special case for gz-sim7+ export ABI_JOB_HEADER_PREFIX=sim[0-9]* @@ -51,29 +51,29 @@ else export ABI_JOB_HEADER_PREFIX=${ABI_JOB_SOFTWARE_NAME/[ignz]*-/}[0-9]* fi -# check if NEED_C17_COMPILER should be set +# check if NEED_C17_COMPILER should be set. Only needed for Bionic so only care about Citadel +# versions if [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-gazebo" ]] || \ [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-physics" ]] || \ [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-sensors" ]] || \ - [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-common" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -ge 3 ]] || \ - [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-gui" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -ge 1 ]] || \ - [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-math" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -ge 6 ]] || \ - [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-msgs" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -ge 3 ]] || \ - [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-plugin" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -ge 1 ]] || \ - [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-rendering" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -ge 1 ]] || \ - [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-transport" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -ge 6 ]] + [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-common" ]] || \ + [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-gui" ]] || \ + [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-math" ]] || \ + [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-msgs" ]] || \ + [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-plugin" ]] || \ + [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-rendering" ]] || \ + [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-transport" ]] then export NEED_C17_COMPILER=true fi # check if OGRE-2.2 include paths are needed -if [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-gazebo" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -eq 6 ]] || \ - [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-gui" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -eq 6 ]] || \ - [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-launch" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -eq 5 ]] || \ - [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-sensors" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -eq 6 ]] || \ - [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-rendering" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -eq 6 ]] +if [[ "${ABI_JOB_SOFTWARE_NAME}" = "gz-sim" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -eq 6 ]] || \ + [[ "${ABI_JOB_SOFTWARE_NAME}" = "gz-gui" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -eq 6 ]] || \ + [[ "${ABI_JOB_SOFTWARE_NAME}" = "gz-launch" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -eq 5 ]] || \ + [[ "${ABI_JOB_SOFTWARE_NAME}" = "gz-sensors" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -eq 6 ]] || \ + [[ "${ABI_JOB_SOFTWARE_NAME}" = "gz-rendering" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -eq 6 ]] then - # OGRE 2.2 is packaged as "Ogre-Next" on jammy if [[ "${DISTRO}" == "jammy" ]] then @@ -92,11 +92,11 @@ then fi fi -if [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-gazebo" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -ge 6 ]] || \ - [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-gui" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -ge 7 ]] || \ - [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-launch" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -ge 6 ]] || \ - [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-sensors" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -ge 7 ]] || \ - [[ "${ABI_JOB_SOFTWARE_NAME}" = "ign-rendering" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -ge 7 ]] +if [[ "${ABI_JOB_SOFTWARE_NAME}" = "gz-sim" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -ge 6 ]] || \ + [[ "${ABI_JOB_SOFTWARE_NAME}" = "gz-gui" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -ge 7 ]] || \ + [[ "${ABI_JOB_SOFTWARE_NAME}" = "gz-launch" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -ge 6 ]] || \ + [[ "${ABI_JOB_SOFTWARE_NAME}" = "gz-sensors" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -ge 7 ]] || \ + [[ "${ABI_JOB_SOFTWARE_NAME}" = "gz-rendering" && ${GZ_NAME_PREFIX_MAJOR_VERSION} -ge 7 ]] then # -fPIC needed to compile Qt export ABI_JOB_EXTRA_GCC_OPTIONS="-fPIC" diff --git a/jenkins-scripts/dsl/gazebo_libs.dsl b/jenkins-scripts/dsl/gazebo_libs.dsl index bc7f3e92d..bc84323b0 100644 --- a/jenkins-scripts/dsl/gazebo_libs.dsl +++ b/jenkins-scripts/dsl/gazebo_libs.dsl @@ -149,25 +149,25 @@ ciconf_per_lib_index.each { lib_name, lib_configs -> def ci_config = gz_collections_yaml.ci_configs.find{ it.name == config_name } def branches_with_collections = ci_configs.getValue() def branch_names = branches_with_collections.collect { it.branch }.unique() - if (ci_config.exclude.contains(lib_name)) + if (ci_config.exclude.all?.contains(lib_name)) return assert(lib_name) assert(branch_names) assert(ci_config) - if (ci_config.exclude.contains(lib_name)) - return - // Main PR jobs (-ci-pr_any-) (pulling check every 5 minutes) // -------------------------------------------------------------- def distro = ci_config.system.version def arch = ci_config.system.arch def gz_job_name_prefix = lib_name.replaceAll('-','_') - def gz_ci_job_name = "${gz_job_name_prefix}-ci-pr_any-${distro}-${arch}" - def gz_ci_any_job = job(gz_ci_job_name) def pre_setup_script = ci_config.pre_setup_script_hook?.get(lib_name)?.join('\n') def extra_cmd = pre_setup_script ?: "" + + // Main PR jobs (-ci-pr_any-) (pulling check every 5 minutes) + // -------------------------------------------------------------- + def gz_ci_job_name = "${gz_job_name_prefix}-ci-pr_any-${distro}-${arch}" + def gz_ci_any_job = job(gz_ci_job_name) OSRFLinuxCompilationAnyGitHub.create(gz_ci_any_job, "gazebosim/${lib_name}", is_testing_enabled(lib_name, ci_config), @@ -193,6 +193,37 @@ ciconf_per_lib_index.each { lib_name, lib_configs -> } // end of steps } // end of ci_any_job + if (! ci_config.exclude.abichecker?.contains(lib_name)) { + // ABI branch jobs (-ci-abichecker-) for non main branches + def abi_job_name = "${gz_job_name_prefix}-abichecker-any_to_any-ubuntu-${distro}-${arch}" + def abi_job = job(abi_job_name) + OSRFLinuxABIGitHub.create(abi_job) + GenericAnyJobGitHub.create(abi_job, + "gazebosim/${lib_name}", + branch_names - [ 'main']) + generate_label_by_requirements(abi_job, lib_name, ci_config.requirements) + abi_job.with + { + steps { + shell("""\ + #!/bin/bash -xe + + export DISTRO=${distro} + + ${GLOBAL_SHELL_CMD} + ${extra_cmd} + + export ARCH=${arch} + export DEST_BRANCH=\${DEST_BRANCH:-\$ghprbTargetBranch} + export SRC_BRANCH=\${SRC_BRANCH:-\$ghprbSourceBranch} + export SRC_REPO=\${SRC_REPO:-\$ghprbAuthorRepoGitUrl} + export ABI_JOB_SOFTWARE_NAME=${lib_name} + /bin/bash -xe ./scripts/jenkins-scripts/docker/gz-abichecker.bash + """.stripIndent()) + } // end of steps + } // end of with + } + // CI branch jobs (-ci-$branch-) (pulling check every 5 minutes) branches_with_collections.each { branch_and_collection -> // TODO: remove after testing @@ -212,7 +243,6 @@ ciconf_per_lib_index.each { lib_name, lib_configs -> [collection: branch_and_collection.collection, job_name: gz_ci_job.name]) } // end_of_branch - } //en of lib_configs } // end of lib diff --git a/jenkins-scripts/dsl/gz-collections.yaml b/jenkins-scripts/dsl/gz-collections.yaml index ea5108707..3af0948af 100644 --- a/jenkins-scripts/dsl/gz-collections.yaml +++ b/jenkins-scripts/dsl/gz-collections.yaml @@ -487,7 +487,7 @@ collections: configs: - jammy exclude: - - __upcoming__ + - __upcoming__ ci_configs: - name: bionic system: @@ -496,7 +496,11 @@ ci_configs: version: bionic arch: amd64 exclude: - - ign-citadel + all: + - ign-citadel + abichecker: + - ign-cmake + - ign-tools requirements: large_memory: - ign-physics @@ -524,8 +528,17 @@ ci_configs: - gz-rendering - gz-sensors exclude: - - gz-fortress - - gz-garden + all: + - gz-fortress + - gz-garden + abichecker: + - gz-cmake + - gz-tools + # These have a Jammy counterpart in Harmonic + - gz-common + - gz-math + - gz-plugin + - gz-utils pre_setup_script_hook: ign-physics: - "export MAKE_JOBS=1" @@ -545,8 +558,12 @@ ci_configs: - gz-rendering - gz-sensors exclude: - - gz-harmonic - - __upcoming__ + all: + - gz-harmonic + - __upcoming__ + abichecker: + - gz-cmake + - gz-tools pre_setup_script_hook: gz-physics: - "export MAKE_JOBS=1" diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index 6fde8a44e..f703fe455 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -389,9 +389,9 @@ void generate_install_job(prefix, gz_sw, major_version, distro, arch) gz_software.each { gz_sw -> supported_arches.each { arch -> // 1 Per library and per linux arch - // 1.1 Per abi_distro + // 1.1 DEPRECATED Per abi_distro // 1.1.1 [job] ABI checker for main branches - // 1.2 Per ci_str_distro + // 1.2 DEPRECATED Per ci_str_distro // 1.2.1 [job] Main PR jobs (-ci-pr_any-) // 1.3 Per all supported_distros // 1.3.1 Per all supported branches on each library @@ -412,39 +412,12 @@ gz_software.each { gz_sw -> abi_job_names[software_name] = "ignition_${software_name}-abichecker-any_to_any-ubuntu_auto-${arch}" def abi_job = job(abi_job_names[software_name]) - checkout_subdir = "ign-${software_name}" - OSRFLinuxABIGitHub.create(abi_job) GenericAnyJobGitHub.create(abi_job, "gazebosim/ign-${software_name}", all_branches(software_name) - [ 'main']) abi_job.with { - extra_str="" - if (gz_sw == 'physics') - { - label Globals.nontest_label("large-memory") - // on ARM native nodes in buildfarm we need to restrict to 1 the - // compilation threads to avoid OOM killer - extra_str += '\nexport MAKE_JOBS=1' - } - - steps { - shell("""\ - #!/bin/bash -xe - - export DISTRO=${distro} - - ${GLOBAL_SHELL_CMD} - ${extra_str} - - export ARCH=${arch} - export DEST_BRANCH=\${DEST_BRANCH:-\$ghprbTargetBranch} - export SRC_BRANCH=\${SRC_BRANCH:-\$ghprbSourceBranch} - export SRC_REPO=\${SRC_REPO:-\$ghprbAuthorRepoGitUrl} - export ABI_JOB_SOFTWARE_NAME=${checkout_subdir} - /bin/bash -xe ./scripts/jenkins-scripts/docker/ignition-abichecker.bash - """.stripIndent()) - } // end of steps + description 'Automatic generated job by DSL jenkins. Stub job for migration, not doing any check' } // end of with } // end of abi_distro @@ -542,12 +515,12 @@ void generate_asan_ci_job(gz_ci_job, gz_sw, branch, distro, arch) generate_ci_job(gz_ci_job, gz_sw, branch, distro, arch, '-DGZ_SANITIZER=Address', Globals.MAKETEST_SKIP_GZ, - 'export ASAN_OPTIONS=check_initialization_order=true:strict_init_order=true') + ['export ASAN_OPTIONS=check_initialization_order=true:strict_init_order=true']) } void generate_ci_job(gz_ci_job, gz_sw, branch, distro, arch, - extra_cmake = '', extra_test = '', extra_cmd = '') + extra_cmake = '', extra_test = '', extra_cmd = []) { OSRFLinuxCompilation.create(gz_ci_job, enable_testing(software_name)) OSRFGitHub.create(gz_ci_job, @@ -559,17 +532,16 @@ void generate_ci_job(gz_ci_job, gz_sw, branch, distro, arch, { if (gz_sw == 'physics') { label Globals.nontest_label("large-memory") - extra_str += '\nexport MAKE_JOBS=1' + extra_cmd += "export MAKE_JOBS=1" } if (gz_sw == 'gazebo') gz_sw = 'sim' steps { - shell("""\ - #!/bin/bash -xe + shell("""#!/bin/bash -xe ${GLOBAL_SHELL_CMD} - ${extra_cmd} + ${extra_cmd.join('\n')} export BUILDING_EXTRA_CMAKE_PARAMS="${extra_cmake}" export BUILDING_EXTRA_MAKETEST_PARAMS="${extra_test}" export DISTRO=${distro} diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index 6273535c3..ea4b72053 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -491,7 +491,7 @@ gz_collections_yaml.collections.each { collection -> distro = ci_config.system.version arch = ci_config.system.arch - if (ci_config.exclude.contains(gz_collection_name)) + if (ci_config.exclude.all?.contains(gz_collection_name)) return // INSTALL JOBS: From 304a148137ce81fce91f29581822f2ae1339050c Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 17 Oct 2023 17:25:38 +0200 Subject: [PATCH 498/540] Move github actions dsl checks to a script (#1042) * Move github actions dsl checks to a script * Added a check for null values --------- Signed-off-by: Jose Luis Rivero --- .github/workflows/ci.yaml | 22 ++----------------- jenkins-scripts/dsl/dsl_checks.bash | 34 +++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 20 deletions(-) create mode 100755 jenkins-scripts/dsl/dsl_checks.bash diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e3caa33c8..02122458a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -52,29 +52,11 @@ jobs: sudo chown -R ${USER} /var/lib/jenkins cd jenkins-scripts/dsl WRITE_JOB_LOG=1 java -jar tools/jobdsl.jar *.dsl - - name: Checks for abichecker jobs + - name: Checks for DSL Code if: steps.dsl_check.outputs.run_job == 'true' run: | cd jenkins-scripts/dsl - main=$(grep 'main' *-abichecker-*.xml || true) - if [[ -n ${main} ]]; then - echo "Found a main branch in an abichecker job:" - echo "${main}" - echo "Main branches are not target of abichecking. Fix the code." - exit 1 - fi - # Filter out the previous auto jobs - mkdir /tmp/abichecker_filtered_jobs - cp *-abichecker-*.xml /tmp/abichecker_filtered_jobs - cd /tmp/abichecker_filtered_jobs - rm *ubuntu_auto*.xml - repeated=$(grep '\' *-abichecker-*.xml | awk '{ print $2 }' | sort | uniq -d) - if [[ -n ${repeated} ]]; then - echo "Found a duplicate in an abichecker branch:" - echo "${repeated}" - echo "please exclude one of the versions in the yaml file to reduce the server workload" - exit 1 - fi + ./dsl_checks.bash - name: Export XML generated configuration for diff if: steps.dsl_check.outputs.run_job == 'true' run: | diff --git a/jenkins-scripts/dsl/dsl_checks.bash b/jenkins-scripts/dsl/dsl_checks.bash new file mode 100755 index 000000000..b87f9bd91 --- /dev/null +++ b/jenkins-scripts/dsl/dsl_checks.bash @@ -0,0 +1,34 @@ +#!/bin/bash -e +# +if [[ -z $(ls -- *.xml) ]]; then + echo "No .xml file founds. Generate them using:" + echo "https://github.com/gazebo-tooling/release-tools/blob/master/jenkins-scripts/README.md#L11" +fi + +not_null=$(grep 'null' -- *.xml || true) +if [[ -n ${not_null} ]]; then + echo "Found a null value in a configuration file:" + echo "${not_null}" + echo "Probably a problem in the code" + exit 1 +fi + +main=$(grep 'main' -- *-abichecker-*.xml || true) +if [[ -n ${main} ]]; then + echo "Found a main branch in an abichecker job:" + echo "${main}" + echo "Main branches are not target of abichecking. Fix the code." + exit 1 +fi + +# Filter out the previous auto jobs +filtered_dir=$(mktemp -d) +cp -- *-abichecker-*.xml "${filtered_dir}" +rm -f "${filtered_dir}"/*-ubuntu_auto*.xml +repeated=$(grep '\' "${filtered_dir}"/*-abichecker-*.xml | awk '{ print $2 }' | sort | uniq -d) +if [[ -n ${repeated} ]]; then + echo "Found a duplicate in an abichecker branch:" + echo "${repeated}" + echo "please exclude one of the versions in the yaml file to reduce the server workload" + exit 1 +fi From 282eec92173136afc355aa686b6d9e4dff673625 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 17 Oct 2023 15:18:50 +0200 Subject: [PATCH 499/540] Fix compilation script path in -any- jobs Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/gazebo_libs.dsl | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/jenkins-scripts/dsl/gazebo_libs.dsl b/jenkins-scripts/dsl/gazebo_libs.dsl index bc84323b0..e2119713a 100644 --- a/jenkins-scripts/dsl/gazebo_libs.dsl +++ b/jenkins-scripts/dsl/gazebo_libs.dsl @@ -50,6 +50,13 @@ void generate_label_by_requirements(job, lib_name, requirements) } } +/* Generate the release-tools script name based on lib_name. The ign replacement can be + * removed after EOL of Bionic */ +String cleanup_library_name(lib_name) +{ + return lib_name.replaceAll('-','_').replaceAll('ign_','gz_').replaceAll('gazebo','sim') +} + boolean is_testing_enabled(lib_name, ci_config) { return ! ci_config.tests_disabled?.contains(lib_name) @@ -105,6 +112,7 @@ void generate_ciconfigs_by_lib(config, ciconf_per_lib_index, pkgconf_per_src_ind void generate_ci_job(gz_ci_job, lib_name, branch, ci_config, extra_cmake = '', extra_test = '', extra_cmd = '') { + def script_name_prefix = cleanup_library_name(lib_name) def distro = ci_config.system.version def arch = ci_config.system.arch def pre_setup_script = ci_config.pre_setup_script_hook?.get(lib_name)?.join('\n') @@ -116,11 +124,6 @@ void generate_ci_job(gz_ci_job, lib_name, branch, ci_config, generate_label_by_requirements(gz_ci_job, lib_name, ci_config.requirements) gz_ci_job.with { - if (lib_name == 'physics') - label Globals.nontest_label("large-memory") - if (lib_name == 'gazebo') - lib_name = 'sim' - steps { shell("""\ #!/bin/bash -xe @@ -132,7 +135,7 @@ void generate_ci_job(gz_ci_job, lib_name, branch, ci_config, export BUILDING_SOFTWARE_DIRECTORY="${lib_name}" export DISTRO=${distro} export ARCH=${arch} - /bin/bash -xe ./scripts/jenkins-scripts/docker/${lib_name.replaceAll('-','_')}-compilation.bash + /bin/bash -xe ./scripts/jenkins-scripts/docker/${script_name_prefix}-compilation.bash """.stripIndent()) } } @@ -155,9 +158,9 @@ ciconf_per_lib_index.each { lib_name, lib_configs -> assert(branch_names) assert(ci_config) - // Main PR jobs (-ci-pr_any-) (pulling check every 5 minutes) // -------------------------------------------------------------- + def script_name_prefix = cleanup_library_name(lib_name) def distro = ci_config.system.version def arch = ci_config.system.arch def gz_job_name_prefix = lib_name.replaceAll('-','_') @@ -188,7 +191,7 @@ ciconf_per_lib_index.each { lib_name, lib_configs -> export BUILDING_SOFTWARE_DIRECTORY=${lib_name} export ARCH=${arch} - /bin/bash -xe ./scripts/jenkins-scripts/docker/${gz_job_name_prefix}-compilation.bash + /bin/bash -xe ./scripts/jenkins-scripts/docker/${script_name_prefix}-compilation.bash """.stripIndent()) } // end of steps } // end of ci_any_job From afd3b89d21c702212a6781490a4cba256c1e1ca1 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 17 Oct 2023 17:56:07 +0200 Subject: [PATCH 500/540] Remove death code in brew jobs Removed code was pointing to a removed scripts. There was already using the rest of the code left in the snippet. Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/ignition.dsl | 24 ++++++------------------ 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index f703fe455..dd1e63e6e 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -614,15 +614,9 @@ gz_software.each { gz_sw -> shell("""\ #!/bin/bash -xe - export HOMEBREW_SCRIPT="./scripts/jenkins-scripts/ign_${software_name}-default-devel-homebrew-amd64.bash" - if [ -s "\$HOMEBREW_SCRIPT" ] - then - /bin/bash -xe "\$HOMEBREW_SCRIPT" - else - software_name="gz-${software_name}" - [[ ${software_name} == 'gazebo' ]] && software_name="gz-sim" - /bin/bash -xe "./scripts/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash" "\${software_name}" - fi + software_name="gz-${software_name}" + [[ ${software_name} == 'gazebo' ]] && software_name="gz-sim" + /bin/bash -xe "./scripts/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash" "\${software_name}" """.stripIndent()) } } @@ -651,15 +645,9 @@ gz_software.each { gz_sw -> shell("""\ #!/bin/bash -xe - export HOMEBREW_SCRIPT="./scripts/jenkins-scripts/ign_${software_name}-default-devel-homebrew-amd64.bash" - if [ -s "\$HOMEBREW_SCRIPT" ] - then - /bin/bash -xe "\$HOMEBREW_SCRIPT" - else - software_name="gz-${software_name}" - [[ ${software_name} == 'gazebo' ]] && software_name="gz-sim" - /bin/bash -xe "./scripts/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash" "\${software_name}" - fi + software_name="gz-${software_name}" + [[ ${software_name} == 'gazebo' ]] && software_name="gz-sim" + /bin/bash -xe "./scripts/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash" "\${software_name}" """.stripIndent()) } } From c2433a587760e64f74aaaa358d83e80683a22874 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 17 Oct 2023 17:58:42 +0200 Subject: [PATCH 501/540] Add check for existing scripts Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/dsl_checks.bash | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/jenkins-scripts/dsl/dsl_checks.bash b/jenkins-scripts/dsl/dsl_checks.bash index b87f9bd91..2841b7515 100755 --- a/jenkins-scripts/dsl/dsl_checks.bash +++ b/jenkins-scripts/dsl/dsl_checks.bash @@ -13,10 +13,17 @@ if [[ -n ${not_null} ]]; then exit 1 fi -main=$(grep 'main' -- *-abichecker-*.xml || true) -if [[ -n ${main} ]]; then +# Check for existing scripts +for f in $(grep -Eh -o './scripts/.*.bash' -- *.xml | sort | uniq); do + if ! test -f "${f}"; then + echo "${f} script not found in the repository" + fi +done + +abichecker_main=$(grep 'main' -- *-abichecker-*.xml || true) +if [[ -n ${abichecker_main} ]]; then echo "Found a main branch in an abichecker job:" - echo "${main}" + echo "${abichecker_main}" echo "Main branches are not target of abichecking. Fix the code." exit 1 fi From 55656a479577984c8987fb5bb9ee9ffe10e025de Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Mon, 23 Oct 2023 17:11:50 +0200 Subject: [PATCH 502/540] Be sure that we use a final / in S3_UPLOAD_PATH Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/_configs_/Globals.groovy | 2 +- jenkins-scripts/lib/repository_uploader.bash | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/jenkins-scripts/dsl/_configs_/Globals.groovy b/jenkins-scripts/dsl/_configs_/Globals.groovy index 3da1e8c38..7de2b82e8 100644 --- a/jenkins-scripts/dsl/_configs_/Globals.groovy +++ b/jenkins-scripts/dsl/_configs_/Globals.groovy @@ -137,7 +137,7 @@ class Globals } static String s3_releases_dir(String package_name) { - return get_canonical_package_name(package_name) + '/releases' + return get_canonical_package_name(package_name) + '/releases/' } static String s3_upload_tarball_path(String package_name) { diff --git a/jenkins-scripts/lib/repository_uploader.bash b/jenkins-scripts/lib/repository_uploader.bash index f45ee35b8..5e0a8949e 100644 --- a/jenkins-scripts/lib/repository_uploader.bash +++ b/jenkins-scripts/lib/repository_uploader.bash @@ -144,7 +144,7 @@ for pkg in ${S3_FILES_TO_UPLOAD}; do fi # Seems important to upload the path with a final slash - S3_upload "${pkgs_path}/${pkg}" "${S3_UPLOAD_PATH}" + S3_upload "${pkgs_path}/${pkg}" "${S3_UPLOAD_PATH%%*(/)}/" done # .bottle | brew binaries From defef687d827bbe8f3a0b9ceaebbcb9d4bf70852 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Mon, 23 Oct 2023 16:51:59 +0200 Subject: [PATCH 503/540] Make OSRFRelease to fail if release.py fails Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/_configs_/OSRFReleasepy.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/_configs_/OSRFReleasepy.groovy b/jenkins-scripts/dsl/_configs_/OSRFReleasepy.groovy index e799ddea1..2dac2bce2 100644 --- a/jenkins-scripts/dsl/_configs_/OSRFReleasepy.groovy +++ b/jenkins-scripts/dsl/_configs_/OSRFReleasepy.groovy @@ -75,7 +75,7 @@ class OSRFReleasepy python3 ./scripts/release.py \${dry_run_str} "\${PACKAGE}" "\${VERSION}" "\${PASS}" \${extra_osrf_repo} \ --source-tarball-uri \${SOURCE_TARBALL_URI} \ --release-repo-branch \${RELEASE_REPO_BRANCH} \ - --upload-to-repo \${UPLOAD_TO_REPO} > log || echo "MARK_AS_UNSTABLE" + --upload-to-repo \${UPLOAD_TO_REPO} > log echo " - done" """.stripIndent()) } From 9c5ad135b37ccb1799bca60c2dab96735c4bfd84 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Mon, 23 Oct 2023 19:18:03 +0200 Subject: [PATCH 504/540] Patch S3_UPLOAD_PATH and leave the releases_dir without final / Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/_configs_/Globals.groovy | 2 +- jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/jenkins-scripts/dsl/_configs_/Globals.groovy b/jenkins-scripts/dsl/_configs_/Globals.groovy index 7de2b82e8..3da1e8c38 100644 --- a/jenkins-scripts/dsl/_configs_/Globals.groovy +++ b/jenkins-scripts/dsl/_configs_/Globals.groovy @@ -137,7 +137,7 @@ class Globals } static String s3_releases_dir(String package_name) { - return get_canonical_package_name(package_name) + '/releases/' + return get_canonical_package_name(package_name) + '/releases' } static String s3_upload_tarball_path(String package_name) { diff --git a/jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy b/jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy index 4e61ca7d9..6297244df 100644 --- a/jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy +++ b/jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy @@ -136,7 +136,7 @@ class OSRFSourceCreation parameters { currentBuild() predefinedProps([PROJECT_NAME_TO_COPY_ARTIFACTS: '${JOB_NAME}', - S3_UPLOAD_PATH: Globals.s3_releases_dir(package_name)]) // relative path + S3_UPLOAD_PATH: "${Globals.s3_releases_dir(package_name)}/"]) // relative path with a final / propertiesFile(properties_file) // S3_FILES_TO_UPLOAD } } From b08020518b1909f81d8f1d33e595f7f3398b1ce7 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 24 Oct 2023 19:08:44 +0200 Subject: [PATCH 505/540] OSRFSourceCreation: use git refs directly instead of rebuild it (#1046) * OSRFSourceCreation: use git refs instead of calculate it * Include context for the null check --------- Signed-off-by: Jose Luis Rivero --- jenkins-scripts/docker/gz-source-generation.bash | 2 +- jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy | 9 ++++++--- jenkins-scripts/dsl/dsl_checks.bash | 2 +- jenkins-scripts/dsl/gazebo_libs.dsl | 2 +- jenkins-scripts/dsl/test.dsl | 5 +++-- 5 files changed, 12 insertions(+), 8 deletions(-) diff --git a/jenkins-scripts/docker/gz-source-generation.bash b/jenkins-scripts/docker/gz-source-generation.bash index 66252259e..a56709217 100644 --- a/jenkins-scripts/docker/gz-source-generation.bash +++ b/jenkins-scripts/docker/gz-source-generation.bash @@ -22,7 +22,7 @@ BUILD_DIR=\$SOURCES_DIR/build cd \${WORKSPACE} rm -fr \$SOURCES_DIR && mkdir \$SOURCES_DIR -git clone --depth 1 --branch ${PACKAGE}_${VERSION/\~/-} ${SOURCE_REPO_URI} \${SOURCES_DIR} +git clone --depth 1 --branch ${SOURCE_REPO_REF} ${SOURCE_REPO_URI} \${SOURCES_DIR} rm -fr \$BUILD_DIR && mkdir \$BUILD_DIR cd \${BUILD_DIR} cmake .. -DPACKAGE_SOURCE_ONLY:BOOL=ON diff --git a/jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy b/jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy index 6297244df..ecd3f94c3 100644 --- a/jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy +++ b/jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy @@ -15,12 +15,12 @@ class OSRFSourceCreation job.with { parameters { - choiceParam('PACKAGE', - [default_params.find{ it.key == "PACKAGE"}?.value], - "Package name (can not be modified)") choiceParam('SOURCE_REPO_URI', [default_params.find{ it.key == "SOURCE_REPO_URI"}?.value], "Software repository URL (can not be modified)") + stringParam('SOURCE_REPO_REF', + default_params.find{ it.key == "SOURCE_REPO_REF"}?.value, + "Git branch or tag to build sources from") stringParam("VERSION", default_params.find{ it.key == "VERSION"}?.value, "Packages version to be built or nightly (enable nightly build mode)") @@ -33,6 +33,9 @@ class OSRFSourceCreation stringParam("DISTRO", default_params.find{ it.key == "DISTRO"}?.value, "Linux release inside LINUX_DISTRO to generate sources on") + choiceParam('PACKAGE', + [default_params.find{ it.key == "PACKAGE"}?.value], + "For downstream use: Package name (can not be modified)") stringParam("RELEASE_VERSION", default_params.find{ it.key == "RELEASE_VERSION"}?.value, "For downstream jobs: Packages release version") diff --git a/jenkins-scripts/dsl/dsl_checks.bash b/jenkins-scripts/dsl/dsl_checks.bash index 2841b7515..5aa45c7f5 100755 --- a/jenkins-scripts/dsl/dsl_checks.bash +++ b/jenkins-scripts/dsl/dsl_checks.bash @@ -5,7 +5,7 @@ if [[ -z $(ls -- *.xml) ]]; then echo "https://github.com/gazebo-tooling/release-tools/blob/master/jenkins-scripts/README.md#L11" fi -not_null=$(grep 'null' -- *.xml || true) +not_null=$(grep -3 'null' -- *.xml || true) if [[ -n ${not_null} ]]; then echo "Found a null value in a configuration file:" echo "${not_null}" diff --git a/jenkins-scripts/dsl/gazebo_libs.dsl b/jenkins-scripts/dsl/gazebo_libs.dsl index e2119713a..015172f7b 100644 --- a/jenkins-scripts/dsl/gazebo_libs.dsl +++ b/jenkins-scripts/dsl/gazebo_libs.dsl @@ -261,7 +261,7 @@ pkgconf_per_src_index.each { pkg_src, pkg_src_configs -> def gz_source_job = job("${pkg_src}-source") OSRFSourceCreation.create(gz_source_job, [ - PACKAGE: pkg_src , + PACKAGE: pkg_src, SOURCE_REPO_URI: "https://github.com/gazebosim/${canonical_lib_name}.git"]) OSRFSourceCreation.call_uploader_and_releasepy(gz_source_job, 'repository_uploader_packages', diff --git a/jenkins-scripts/dsl/test.dsl b/jenkins-scripts/dsl/test.dsl index 4f3fc45d9..56af6a360 100644 --- a/jenkins-scripts/dsl/test.dsl +++ b/jenkins-scripts/dsl/test.dsl @@ -29,8 +29,9 @@ releasepy_job.with { // gz source testing job def gz_source_job = job("_test_gz_source") OSRFSourceCreation.create(gz_source_job, [ - PACKAGE: "gz-plugin2" , - SOURCE_REPO_URI: "https://github.com/gazebosim/gz-plugin.git"]) + PACKAGE: "gz-plugin2", + SOURCE_REPO_URI: "https://github.com/gazebosim/gz-plugin.git", + SOURCE_REPO_REF: "gz-plugin2"]) OSRFSourceCreation.call_uploader_and_releasepy(gz_source_job, '_test_repository_uploader', '_test_releasepy') From 773f6cc55d8222dc85416d42aaf4d7e0bc32157b Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 25 Oct 2023 19:04:56 +0200 Subject: [PATCH 506/540] Fix problem with quoting on repository_uploader Signed-off-by: Jose Luis Rivero --- jenkins-scripts/lib/repository_uploader.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/lib/repository_uploader.bash b/jenkins-scripts/lib/repository_uploader.bash index 5e0a8949e..b7ce2bd4f 100644 --- a/jenkins-scripts/lib/repository_uploader.bash +++ b/jenkins-scripts/lib/repository_uploader.bash @@ -144,7 +144,7 @@ for pkg in ${S3_FILES_TO_UPLOAD}; do fi # Seems important to upload the path with a final slash - S3_upload "${pkgs_path}/${pkg}" "${S3_UPLOAD_PATH%%*(/)}/" + S3_upload "${pkgs_path}/${pkg}" "${S3_UPLOAD_PATH%%*(/)}"/ done # .bottle | brew binaries From 742b33abb4bc694ae547175dab2d7761bbc40146 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 26 Oct 2023 15:03:05 +0200 Subject: [PATCH 507/540] Fix slash check in repository_uploader Signed-off-by: Jose Luis Rivero --- jenkins-scripts/lib/repository_uploader.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/lib/repository_uploader.bash b/jenkins-scripts/lib/repository_uploader.bash index b7ce2bd4f..e8c06068c 100644 --- a/jenkins-scripts/lib/repository_uploader.bash +++ b/jenkins-scripts/lib/repository_uploader.bash @@ -144,7 +144,7 @@ for pkg in ${S3_FILES_TO_UPLOAD}; do fi # Seems important to upload the path with a final slash - S3_upload "${pkgs_path}/${pkg}" "${S3_UPLOAD_PATH%%*(/)}"/ + S3_upload "${pkgs_path}/${pkg}" "${S3_UPLOAD_PATH%*/}"/ done # .bottle | brew binaries From 8c8880e00854bde1cbc4091e8e0c61759d1f03fe Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 26 Oct 2023 22:24:01 +0200 Subject: [PATCH 508/540] Change daily Linux ci jobs (-ci-${branch}) to be based on gz-collections.yaml (#1047) * Move all collection daily -ci- on Linux to gazebo_libs * Remove __upcoming__ duplicates with Ionic * Remove sdformat related jobs * Missing jamy config in packaging for Ionic --------- Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/gazebo_libs.dsl | 3 - jenkins-scripts/dsl/gz-collections.yaml | 62 +----------------- jenkins-scripts/dsl/ignition.dsl | 10 --- jenkins-scripts/dsl/sdformat.dsl | 85 ------------------------- 4 files changed, 2 insertions(+), 158 deletions(-) diff --git a/jenkins-scripts/dsl/gazebo_libs.dsl b/jenkins-scripts/dsl/gazebo_libs.dsl index 015172f7b..0536abae2 100644 --- a/jenkins-scripts/dsl/gazebo_libs.dsl +++ b/jenkins-scripts/dsl/gazebo_libs.dsl @@ -229,9 +229,6 @@ ciconf_per_lib_index.each { lib_name, lib_configs -> // CI branch jobs (-ci-$branch-) (pulling check every 5 minutes) branches_with_collections.each { branch_and_collection -> - // TODO: remove after testing - if (branch_and_collection.collection != 'harmonic') - return branch_name = branch_and_collection.branch def gz_ci_job = job("${gz_job_name_prefix}-ci-${branch_name}-${distro}-${arch}") generate_ci_job(gz_ci_job, lib_name, branch_name, ci_config) diff --git a/jenkins-scripts/dsl/gz-collections.yaml b/jenkins-scripts/dsl/gz-collections.yaml index 3af0948af..8e0b6329a 100644 --- a/jenkins-scripts/dsl/gz-collections.yaml +++ b/jenkins-scripts/dsl/gz-collections.yaml @@ -408,6 +408,8 @@ collections: configs: - jammy packaging: + configs: + - jammy linux: ignore_major_version: - gz-ionic @@ -416,70 +418,10 @@ collections: - jammy - name: '__upcoming__' libs: - - name: gz-cmake - major_version: 4 - repo: - current_branch: main - name: gz-tools major_version: 3 repo: current_branch: main - - name: gz-utils - major_version: 3 - repo: - current_branch: main - - name: gz-math - major_version: 8 - repo: - current_branch: main - - name: gz-plugin - major_version: 3 - repo: - current_branch: main - - name: gz-common - major_version: 6 - repo: - current_branch: main - - name: gz-msgs - major_version: 11 - repo: - current_branch: main - - name: gz-rendering - major_version: 9 - repo: - current_branch: main - - name: sdformat - major_version: 15 - repo: - current_branch: main - - name: gz-fuel-tools - major_version: 10 - repo: - current_branch: main - - name: gz-transport - major_version: 14 - repo: - current_branch: main - - name: gz-gui - major_version: 9 - repo: - current_branch: main - - name: gz-sensors - major_version: 9 - repo: - current_branch: main - - name: gz-physics - major_version: 8 - repo: - current_branch: main - - name: gz-sim - major_version: 9 - repo: - current_branch: main - - name: gz-launch - major_version: 8 - repo: - current_branch: main ci: configs: - jammy diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index dd1e63e6e..c845386a7 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -452,16 +452,6 @@ gz_software.each { gz_sw -> if (gz_sw == 'sim') software_name = "gazebo" - // 1.3.1.1 Branch jobs -ci-$branch- - // -------------------------------------------------------------- - def gz_ci_job = job("ignition_${software_name}-ci-${branch}-${distro}-${arch}") - generate_ci_job(gz_ci_job, software_name, branch, distro, arch) - gz_ci_job.with - { - triggers { - scm('@daily') - } - } // 1.3.1.2 Branch ASAN jobs -ci_asan-$branch- // -------------------------------------------------------------- def gz_ci_asan_job = job("ignition_${software_name}-ci_asan-${branch}-${distro}-${arch}") diff --git a/jenkins-scripts/dsl/sdformat.dsl b/jenkins-scripts/dsl/sdformat.dsl index f93b42396..dd67dc9f6 100644 --- a/jenkins-scripts/dsl/sdformat.dsl +++ b/jenkins-scripts/dsl/sdformat.dsl @@ -63,29 +63,6 @@ abi_distro.each { distro -> // CI JOBS @ SCM/5 min ci_distro.each { distro -> supported_arches.each { arch -> - // -------------------------------------------------------------- - // 1. Create the main ci jobs - def sdformat_ci_job = job("sdformat-ci-main-${distro}-${arch}") - OSRFLinuxCompilation.create(sdformat_ci_job) - OSRFGitHub.create(sdformat_ci_job, "gazebosim/sdformat", "main") - - sdformat_ci_job.with - { - triggers { - scm('*/5 * * * *') - } - - steps { - shell("""\ - #!/bin/bash -xe - - export DISTRO=${distro} - export ARCH=${arch} - /bin/bash -xe ./scripts/jenkins-scripts/docker/sdformat-compilation.bash - """.stripIndent()) - } - } - // -------------------------------------------------------------- // 2. Create a fake any job for migrating // TODO: remove the job once the migration in https://github.com/gazebo-tooling/release-tools/issues/1010 @@ -96,33 +73,6 @@ ci_distro.each { distro -> } // end of arch } // end of distro -// OTHER CI SUPPORTED JOBS (main branch) @ SCM/DAILY -other_supported_distros.each { distro -> - supported_arches.each { arch -> - // ci_main job for the rest of arches / scm@daily - def sdformat_ci_job = job("sdformat-ci-main-${distro}-${arch}") - OSRFLinuxCompilation.create(sdformat_ci_job) - OSRFGitHub.create(sdformat_ci_job, "gazebosim/sdformat", "main") - - sdformat_ci_job.with - { - triggers { - scm('@daily') - } - - steps { - shell("""\ - #!/bin/bash -xe - - export DISTRO=${distro} - export ARCH=${arch} - /bin/bash -xe ./scripts/jenkins-scripts/docker/sdformat-compilation.bash - """.stripIndent()) - } - } - } // end of arch -} // end of distro - void generate_asan_ci_job(sdformat_ci_job, version, distro, arch) { generate_ci_job(sdformat_ci_job, version, distro, arch, @@ -159,15 +109,6 @@ void generate_ci_job(sdformat_ci_job, version, distro, arch, sdformat_supported_versions.each { version -> ci_distro.each { distro -> supported_arches.each { arch -> - // ci job for the rest of arches / scm@daily - def sdformat_ci_job = job("sdformat-ci-${version}-${distro}-${arch}") - generate_ci_job(sdformat_ci_job, version, distro, arch) - sdformat_ci_job.with - { - triggers { - scm('@daily') - } - } // ci_asan job for the rest of arches / scm@weekend def sdformat_ci_asan_job = job("sdformat-ci_asan-${version}-${distro}-${arch}") generate_asan_ci_job(sdformat_ci_asan_job, version, distro, arch) @@ -181,32 +122,6 @@ sdformat_supported_versions.each { version -> } } -// EXPERIMENTAL ARCHES @ SCM/WEEKLY -ci_distro.each { distro -> - experimental_arches.each { arch -> - def sdformat_ci_job = job("sdformat-ci-main-${distro}-${arch}") - OSRFLinuxCompilation.create(sdformat_ci_job) - OSRFGitHub.create(sdformat_ci_job, "gazebosim/sdformat", "main") - - sdformat_ci_job.with - { - triggers { - scm('@weekly') - } - - steps { - shell("""\ - #!/bin/bash -xe - - export DISTRO=${distro} - export ARCH=${arch} - /bin/bash -xe ./scripts/jenkins-scripts/docker/sdformat-compilation.bash - """.stripIndent()) - } - } - } -} - // INSTALL LINUX -DEV PACKAGES ALL PLATFORMS @ CRON/DAILY sdformat_supported_versions.each { version -> ci_distro.each { distro -> From aeaae9e918a0167649a4661929913d8505c37385 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Crist=C3=B3bal=20Arroyo?= Date: Fri, 27 Oct 2023 14:58:46 -0500 Subject: [PATCH 509/540] Fix sensors and physics versions (#1049) --- jenkins-scripts/dsl/gz-collections.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/jenkins-scripts/dsl/gz-collections.yaml b/jenkins-scripts/dsl/gz-collections.yaml index 8e0b6329a..9a0549acb 100644 --- a/jenkins-scripts/dsl/gz-collections.yaml +++ b/jenkins-scripts/dsl/gz-collections.yaml @@ -152,13 +152,13 @@ collections: repo: current_branch: ign-gui6 - name: gz-sensors - major_version: 7 + major_version: 6 repo: - current_branch: ign-sensors7 + current_branch: ign-sensors6 - name: gz-physics - major_version: 6 + major_version: 5 repo: - current_branch: ign-physics6 + current_branch: ign-physics5 - name: gz-sim major_version: 6 repo: From e35e68bbf6b258f9d8e795666dd107660949eb9e Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 1 Nov 2023 17:08:07 +0100 Subject: [PATCH 510/540] Fix dsl to assure that BUILDING_SOFTWARE_DIRECTORY and git checkout dir are in sync (#1052) Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/gazebo_libs.dsl | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/jenkins-scripts/dsl/gazebo_libs.dsl b/jenkins-scripts/dsl/gazebo_libs.dsl index 0536abae2..700f98913 100644 --- a/jenkins-scripts/dsl/gazebo_libs.dsl +++ b/jenkins-scripts/dsl/gazebo_libs.dsl @@ -116,11 +116,14 @@ void generate_ci_job(gz_ci_job, lib_name, branch, ci_config, def distro = ci_config.system.version def arch = ci_config.system.arch def pre_setup_script = ci_config.pre_setup_script_hook?.get(lib_name)?.join('\n') + def ws_checkout_dir = lib_name extra_cmd = [extra_cmd, pre_setup_script].findAll({ it != null }).join() OSRFLinuxCompilation.create(gz_ci_job, is_testing_enabled(lib_name, ci_config)) OSRFGitHub.create(gz_ci_job, - "gazebosim/${lib_name}", "${branch}") + "gazebosim/${lib_name}", + branch, + ws_checkout_dir) generate_label_by_requirements(gz_ci_job, lib_name, ci_config.requirements) gz_ci_job.with { @@ -132,7 +135,7 @@ void generate_ci_job(gz_ci_job, lib_name, branch, ci_config, ${extra_cmd} export BUILDING_EXTRA_CMAKE_PARAMS="${extra_cmake}" export BUILDING_EXTRA_MAKETEST_PARAMS="${extra_test}" - export BUILDING_SOFTWARE_DIRECTORY="${lib_name}" + export BUILDING_SOFTWARE_DIRECTORY="${ws_checkout_dir}" export DISTRO=${distro} export ARCH=${arch} /bin/bash -xe ./scripts/jenkins-scripts/docker/${script_name_prefix}-compilation.bash From 2d79621ccc053d25e95aae69034e4f3a3ca576be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Crist=C3=B3bal=20Arroyo?= Date: Wed, 1 Nov 2023 12:17:57 -0500 Subject: [PATCH 511/540] Update outdated-job-runner job list (#1050) --- .../tools/outdated-job-runner.groovy | 74 +++++++++---------- 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/jenkins-scripts/tools/outdated-job-runner.groovy b/jenkins-scripts/tools/outdated-job-runner.groovy index bf9fc5e16..3d958e311 100644 --- a/jenkins-scripts/tools/outdated-job-runner.groovy +++ b/jenkins-scripts/tools/outdated-job-runner.groovy @@ -67,22 +67,47 @@ def runJobsInAvailableNodes(LinkedHashMap outdatedJobs) { // Currently is the result of https://github.com/osrf/buildfarmer/blob/main/scripts/gazebo_non_nightly_jobs.py def trackedJobsList = [ 'gz_cmake-ci-gz-cmake3-jammy-amd64', + 'gz_cmake-ci-ign-cmake2-focal-amd64', 'gz_common-ci-gz-common5-jammy-amd64', + 'gz_common-ci-ign-common4-focal-amd64', + 'gz_fuel_tools-ci-gz-fuel-tools8-focal-amd64', 'gz_fuel_tools-ci-gz-fuel-tools9-jammy-amd64', + 'gz_fuel_tools-ci-ign-fuel-tools7-focal-amd64', + 'gz_gui-ci-gz-gui7-focal-amd64', 'gz_gui-ci-gz-gui8-jammy-amd64', + 'gz_gui-ci-ign-gui6-focal-amd64', + 'gz_launch-ci-gz-launch6-focal-amd64', 'gz_launch-ci-gz-launch7-jammy-amd64', + 'gz_launch-ci-ign-launch5-focal-amd64', 'gz_math-ci-gz-math7-jammy-amd64', + 'gz_math-ci-ign-math6-focal-amd64', 'gz_msgs-ci-gz-msgs10-jammy-amd64', + 'gz_msgs-ci-gz-msgs9-focal-amd64', + 'gz_msgs-ci-ign-msgs8-focal-amd64', + 'gz_physics-ci-gz-physics6-focal-amd64', 'gz_physics-ci-gz-physics7-jammy-amd64', + 'gz_physics-ci-ign-physics5-focal-amd64', 'gz_plugin-ci-gz-plugin2-jammy-amd64', + 'gz_plugin-ci-ign-plugin1-focal-amd64', + 'gz_rendering-ci-gz-rendering7-focal-amd64', 'gz_rendering-ci-gz-rendering8-jammy-amd64', + 'gz_rendering-ci-ign-rendering6-focal-amd64', + 'gz_sensors-ci-gz-sensors7-focal-amd64', 'gz_sensors-ci-gz-sensors8-jammy-amd64', + 'gz_sensors-ci-ign-sensors6-focal-amd64', + 'gz_sim-ci-gz-sim7-focal-amd64', 'gz_sim-ci-gz-sim8-jammy-amd64', + 'gz_sim-ci-ign-gazebo6-focal-amd64', 'gz_tools-ci-gz-tools2-jammy-amd64', + 'gz_tools-ci-ign-tools1-focal-amd64', + 'gz_transport-ci-gz-transport12-focal-amd64', 'gz_transport-ci-gz-transport13-jammy-amd64', + 'gz_transport-ci-ign-transport11-focal-amd64', 'gz_utils-ci-gz-utils2-jammy-amd64', + 'gz_utils-ci-ign-utils1-focal-amd64', 'ign_cmake-gz-3-win', 'ign_cmake-ign-2-win', + 'ign_common-ci-ign-common3-bionic-amd64', 'ign_common-gz-5-win', 'ign_common-ign-3-win', 'ign_common-ign-4-win', @@ -90,40 +115,49 @@ def trackedJobsList = [ 'ign_fuel-tools-gz-9-win', 'ign_fuel-tools-ign-4-win', 'ign_fuel-tools-ign-7-win', + 'ign_fuel_tools-ci-ign-fuel-tools4-bionic-amd64', + 'ign_gazebo-ci-ign-gazebo3-bionic-amd64', 'ign_gazebo-gz-7-win', 'ign_gazebo-gz-8-win', 'ign_gazebo-ign-3-win', 'ign_gazebo-ign-6-win', + 'ign_gui-ci-ign-gui3-bionic-amd64', 'ign_gui-gz-7-win', 'ign_gui-gz-8-win', 'ign_gui-ign-3-win', 'ign_gui-ign-6-win', + 'ign_launch-ci-ign-launch2-bionic-amd64', 'ign_launch-gz-6-win', 'ign_launch-gz-7-win', 'ign_launch-ign-2-win', 'ign_launch-ign-5-win', 'ign_math-gz-7-win', 'ign_math-ign-6-win', + 'ign_msgs-ci-ign-msgs5-bionic-amd64', 'ign_msgs-gz-10-win', 'ign_msgs-gz-9-win', 'ign_msgs-ign-5-win', 'ign_msgs-ign-8-win', + 'ign_physics-ci-ign-physics2-bionic-amd64', 'ign_physics-gz-6-win', 'ign_physics-gz-7-win', 'ign_physics-ign-2-win', 'ign_physics-ign-5-win', 'ign_plugin-gz-2-win', 'ign_plugin-ign-1-win', + 'ign_rendering-ci-ign-rendering3-bionic-amd64', 'ign_rendering-gz-7-win', 'ign_rendering-gz-8-win', 'ign_rendering-ign-3-win', 'ign_rendering-ign-6-win', + 'ign_sensors-ci-ign-sensors3-bionic-amd64', 'ign_sensors-gz-7-win', 'ign_sensors-gz-8-win', 'ign_sensors-ign-3-win', 'ign_sensors-ign-6-win', 'ign_tools-gz-2-win', 'ign_tools-ign-1-win', + 'ign_transport-ci-ign-transport8-bionic-amd64', 'ign_transport-gz-12-win', 'ign_transport-gz-13-win', 'ign_transport-ign-11-win', @@ -131,95 +165,61 @@ def trackedJobsList = [ 'ign_utils-gz-2-win', 'ign_utils-ign-1-win', 'ignition_cmake-ci-gz-cmake3-homebrew-amd64', - 'ignition_cmake-ci-ign-cmake2-focal-amd64', 'ignition_cmake-ci-ign-cmake2-homebrew-amd64', 'ignition_common-ci-gz-common5-homebrew-amd64', - 'ignition_common-ci-ign-common3-focal-amd64', 'ignition_common-ci-ign-common3-homebrew-amd64', - 'ignition_common-ci-ign-common4-focal-amd64', 'ignition_common-ci-ign-common4-homebrew-amd64', - 'ignition_fuel-tools-ci-gz-fuel-tools8-focal-amd64', 'ignition_fuel-tools-ci-gz-fuel-tools8-homebrew-amd64', 'ignition_fuel-tools-ci-gz-fuel-tools9-homebrew-amd64', - 'ignition_fuel-tools-ci-ign-fuel-tools4-focal-amd64', 'ignition_fuel-tools-ci-ign-fuel-tools4-homebrew-amd64', - 'ignition_fuel-tools-ci-ign-fuel-tools7-focal-amd64', 'ignition_fuel-tools-ci-ign-fuel-tools7-homebrew-amd64', - 'ignition_gazebo-ci-gz-sim7-focal-amd64', 'ignition_gazebo-ci-gz-sim7-homebrew-amd64', 'ignition_gazebo-ci-gz-sim8-homebrew-amd64', - 'ignition_gazebo-ci-ign-gazebo3-focal-amd64', 'ignition_gazebo-ci-ign-gazebo3-homebrew-amd64', - 'ignition_gazebo-ci-ign-gazebo6-focal-amd64', 'ignition_gazebo-ci-ign-gazebo6-homebrew-amd64', - 'ignition_gui-ci-gz-gui7-focal-amd64', 'ignition_gui-ci-gz-gui7-homebrew-amd64', 'ignition_gui-ci-gz-gui8-homebrew-amd64', - 'ignition_gui-ci-ign-gui3-focal-amd64', 'ignition_gui-ci-ign-gui3-homebrew-amd64', - 'ignition_gui-ci-ign-gui6-focal-amd64', 'ignition_gui-ci-ign-gui6-homebrew-amd64', - 'ignition_launch-ci-gz-launch6-focal-amd64', 'ignition_launch-ci-gz-launch6-homebrew-amd64', 'ignition_launch-ci-gz-launch7-homebrew-amd64', - 'ignition_launch-ci-ign-launch2-focal-amd64', 'ignition_launch-ci-ign-launch2-homebrew-amd64', - 'ignition_launch-ci-ign-launch5-focal-amd64', 'ignition_launch-ci-ign-launch5-homebrew-amd64', 'ignition_math-ci-gz-math7-homebrew-amd64', - 'ignition_math-ci-ign-math6-focal-amd64', 'ignition_math-ci-ign-math6-homebrew-amd64', 'ignition_msgs-ci-gz-msgs10-homebrew-amd64', - 'ignition_msgs-ci-gz-msgs9-focal-amd64', 'ignition_msgs-ci-gz-msgs9-homebrew-amd64', - 'ignition_msgs-ci-ign-msgs5-focal-amd64', 'ignition_msgs-ci-ign-msgs5-homebrew-amd64', - 'ignition_msgs-ci-ign-msgs8-focal-amd64', 'ignition_msgs-ci-ign-msgs8-homebrew-amd64', - 'ignition_physics-ci-gz-physics6-focal-amd64', 'ignition_physics-ci-gz-physics6-homebrew-amd64', 'ignition_physics-ci-gz-physics7-homebrew-amd64', - 'ignition_physics-ci-ign-physics2-focal-amd64', 'ignition_physics-ci-ign-physics2-homebrew-amd64', - 'ignition_physics-ci-ign-physics5-focal-amd64', 'ignition_physics-ci-ign-physics5-homebrew-amd64', 'ignition_plugin-ci-gz-plugin2-homebrew-amd64', - 'ignition_plugin-ci-ign-plugin1-focal-amd64', 'ignition_plugin-ci-ign-plugin1-homebrew-amd64', - 'ignition_rendering-ci-gz-rendering7-focal-amd64', 'ignition_rendering-ci-gz-rendering7-homebrew-amd64', 'ignition_rendering-ci-gz-rendering8-homebrew-amd64', - 'ignition_rendering-ci-ign-rendering3-focal-amd64', 'ignition_rendering-ci-ign-rendering3-homebrew-amd64', - 'ignition_rendering-ci-ign-rendering6-focal-amd64', 'ignition_rendering-ci-ign-rendering6-homebrew-amd64', - 'ignition_sensors-ci-gz-sensors7-focal-amd64', 'ignition_sensors-ci-gz-sensors7-homebrew-amd64', 'ignition_sensors-ci-gz-sensors8-homebrew-amd64', - 'ignition_sensors-ci-ign-sensors3-focal-amd64', 'ignition_sensors-ci-ign-sensors3-homebrew-amd64', - 'ignition_sensors-ci-ign-sensors6-focal-amd64', 'ignition_sensors-ci-ign-sensors6-homebrew-amd64', 'ignition_tools-ci-gz-tools2-homebrew-amd64', - 'ignition_tools-ci-ign-tools1-focal-amd64', 'ignition_tools-ci-ign-tools1-homebrew-amd64', - 'ignition_transport-ci-gz-transport12-focal-amd64', 'ignition_transport-ci-gz-transport12-homebrew-amd64', 'ignition_transport-ci-gz-transport13-homebrew-amd64', - 'ignition_transport-ci-ign-transport11-focal-amd64', 'ignition_transport-ci-ign-transport11-homebrew-amd64', - 'ignition_transport-ci-ign-transport8-focal-amd64', 'ignition_transport-ci-ign-transport8-homebrew-amd64', 'ignition_utils-ci-gz-utils2-homebrew-amd64', - 'ignition_utils-ci-ign-utils1-focal-amd64', 'ignition_utils-ci-ign-utils1-homebrew-amd64', + 'sdformat-ci-sdf12-focal-amd64', + 'sdformat-ci-sdf13-focal-amd64', 'sdformat-ci-sdf14-jammy-amd64', - 'sdformat-ci-sdformat12-focal-amd64', 'sdformat-ci-sdformat12-homebrew-amd64', - 'sdformat-ci-sdformat13-focal-amd64', 'sdformat-ci-sdformat13-homebrew-amd64', 'sdformat-ci-sdformat14-homebrew-amd64', - 'sdformat-ci-sdformat9-focal-amd64', + 'sdformat-ci-sdformat9-bionic-amd64', 'sdformat-ci-sdformat9-homebrew-amd64', 'sdformat-sdf-12-win', 'sdformat-sdf-13-win', From 22bde724cace079fe16cc85421eb17c85ec91e51 Mon Sep 17 00:00:00 2001 From: Ian Chen Date: Wed, 1 Nov 2023 15:51:33 -0700 Subject: [PATCH 512/540] Remove references to answers.gazebosim.org (#1053) Signed-off-by: Ian Chen --- .github/ISSUE_TEMPLATE/new_collection.md | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/new_collection.md b/.github/ISSUE_TEMPLATE/new_collection.md index 481d816da..94f1d1810 100644 --- a/.github/ISSUE_TEMPLATE/new_collection.md +++ b/.github/ISSUE_TEMPLATE/new_collection.md @@ -22,4 +22,3 @@ When opening PRs, add a link back to this issue for easier tracking. - [ ] Add new collection label: - [ ] Add to pr-collection-labeler ([example](https://github.com/gazebo-tooling/pr-collection-labeler/pull/11)) - [ ] Add to all libraries ([handy script](https://github.com/chapulina/github-label-maker/tree/chapulina/ign#ignition)) -- [ ] Add new collection tag to Gazebo Answers [here](https://answers.gazebosim.org/settings/FORUM_DATA_RULES/) From 82ce89e4ba8d198df94a1a1dcfcfdfb0b5d312cf Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Wed, 1 Nov 2023 15:53:57 -0700 Subject: [PATCH 513/540] brew CI: use full tap name (#1051) Use the fully qualified formula name (with the osrf/simulation/ prefix) when checking for dependencies. Remove logic for tinyxml2@2.6.2 dependencies since it is not used by any current formulae. Signed-off-by: Steve Peters Co-authored-by: Jose Luis Rivero --- .../project-default-devel-homebrew-amd64.bash | 23 +++++++------------ 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash b/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash index df94a79b1..5f05056f5 100644 --- a/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash +++ b/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash @@ -136,25 +136,18 @@ if brew ruby -e "exit ! '${PROJECT_FORMULA}'.f.recursive_dependencies.map(&:name CMAKE_ARGS='-DQWT_WIN_INCLUDE_DIR=/usr/local/opt/qwt-qt5/lib/qwt.framework/Headers -DQWT_WIN_LIBRARY_DIR=/usr/local/opt/qwt-qt5/lib' fi # Workaround for cmake@3.21.4: set PATH -if brew ruby -e "exit ! '${PROJECT_FORMULA}'.f.recursive_dependencies.map(&:name).keep_if { |d| d == 'cmake@3.21.4' }.empty?"; then +if brew ruby -e "exit ! '${PROJECT_FORMULA}'.f.recursive_dependencies.map(&:name).keep_if { |d| d == 'osrf/simulation/cmake@3.21.4' }.empty?"; then export PATH=/usr/local/opt/cmake@3.21.4/bin:${PATH} fi -# Workaround for ffmpeg 4: set CMAKE_PREFIX_PATH and PKG_CONFIG_PATH if we are using ffmpeg@4 -if brew ruby -e "exit ! '${PROJECT_FORMULA}'.f.recursive_dependencies.map(&:name).keep_if { |d| d == 'ffmpeg@4' }.empty?"; then +# Workaround for ffmpeg 4: set PKG_CONFIG_PATH if we are using ffmpeg@4 +if brew ruby -e "exit ! '${PROJECT_FORMULA}'.f.recursive_dependencies.map(&:name).keep_if { |d| d == 'osrf/simulation/ffmpeg@4' }.empty?"; then export PKG_CONFIG_PATH=${PKG_CONFIG_PATH}:/usr/local/opt/ffmpeg@4/lib/pkgconfig fi # Workaround for tbb@2020_u3: set CPATH, LIBRARY_PATH, and CMAKE_PREFIX_PATH -if brew ruby -e "exit ! '${PROJECT_FORMULA}'.f.recursive_dependencies.map(&:name).keep_if { |d| d == 'tbb@2020_u3' }.empty?"; then - if [ "${ghprbSourceBranch}" != "replace_deprecated_tbb_task" ]; then - export CMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH}:/usr/local/opt/tbb@2020_u3 - export CPATH=${CPATH}:/usr/local/opt/tbb@2020_u3/include - export LIBRARY_PATH=${LIBRARY_PATH}:/usr/local/opt/tbb@2020_u3/lib - fi -fi -# Workaround for tinyxml2 6.2.0: set CMAKE_PREFIX_PATH and PKG_CONFIG_PATH if we are using tinyxml2@6.2.0 -if brew ruby -e "exit ! '${PROJECT_FORMULA}'.f.recursive_dependencies.map(&:name).keep_if { |d| d == 'tinyxml2@6.2.0' }.empty?"; then - export CMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH}:/usr/local/opt/tinyxml2@6.2.0 - export PKG_CONFIG_PATH=${PKG_CONFIG_PATH}:/usr/local/opt/tinyxml2@6.2.0/lib/pkgconfig +if brew ruby -e "exit ! '${PROJECT_FORMULA}'.f.recursive_dependencies.map(&:name).keep_if { |d| d == 'osrf/simulation/tbb@2020_u3' }.empty?"; then + export CMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH}:/usr/local/opt/tbb@2020_u3 + export CPATH=${CPATH}:/usr/local/opt/tbb@2020_u3/include + export LIBRARY_PATH=${LIBRARY_PATH}:/usr/local/opt/tbb@2020_u3/lib fi # if we are using gts, need to add gettext library path since it is keg-only if brew ruby -e "exit ! '${PROJECT_FORMULA}'.f.recursive_dependencies.map(&:name).keep_if { |d| d == 'gettext' }.empty?"; then @@ -166,7 +159,7 @@ if brew ruby -e "exit ! '${PROJECT_FORMULA}'.f.recursive_dependencies.map(&:name fi # if we are using dart@6.10.0 (custom OR port), need to add dartsim library path since it is keg-only -if brew ruby -e "exit ! '${PROJECT_FORMULA}'.f.recursive_dependencies.map(&:name).keep_if { |d| d == 'dartsim@6.10.0' }.empty?"; then +if brew ruby -e "exit ! '${PROJECT_FORMULA}'.f.recursive_dependencies.map(&:name).keep_if { |d| d == 'osrf/simulation/dartsim@6.10.0' }.empty?"; then export CMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH}:/usr/local/opt/dartsim@6.10.0 export DYLD_FALLBACK_LIBRARY_PATH=${DYLD_FALLBACK_LIBRARY_PATH}:/usr/local/opt/dartsim@6.10.0/lib:/usr/local/opt/octomap/local export PKG_CONFIG_PATH=${PKG_CONFIG_PATH}:/usr/local/opt/dartsim@6.10.0/lib/pkgconfig From 3f3e34d0d35c1c9fede14741c4a64e9bcdb42e67 Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Thu, 2 Nov 2023 15:14:40 -0700 Subject: [PATCH 514/540] Add ci_matching_branch/ for bottle install jobs (#1005) If the release-tools branch (RTOOLS_BRANCH) starts with ci_matching_branch/ then try to check out a matching branch from the osrf/homebrew-simulation tap for bottle install jobs. Signed-off-by: Steve Peters Co-authored-by: Jose Luis Rivero --- jenkins-scripts/lib/project-install-homebrew.bash | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/jenkins-scripts/lib/project-install-homebrew.bash b/jenkins-scripts/lib/project-install-homebrew.bash index b9f35eaf5..164524883 100644 --- a/jenkins-scripts/lib/project-install-homebrew.bash +++ b/jenkins-scripts/lib/project-install-homebrew.bash @@ -46,6 +46,16 @@ echo '# BEGIN SECTION: setup the osrf/simulation tap' brew tap osrf/simulation echo '# END SECTION' +if python3 "${SCRIPT_DIR}/tools/detect_ci_matching_branch.py" "${RTOOLS_BRANCH}" +then + echo "# BEGIN SECTION: trying to checkout branch ${RTOOLS_BRANCH} from osrf/simulation" + pushd "$(brew --repo osrf/simulation)" + git fetch origin "${RTOOLS_BRANCH}" || true + git checkout "${RTOOLS_BRANCH}" || true + popd + echo '# END SECTION' +fi + echo "# BEGIN SECTION: install ${BOTTLE_NAME}" brew install --include-test ${BOTTLE_NAME} From 9c7d545c55bfc145d2ce9d9a26ba86c00d5feaa7 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 3 Nov 2023 17:35:54 +0100 Subject: [PATCH 515/540] Change release.py to use the new centralized tarball approach (#1048) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Remove all code related to tarball generation * Implement new parameters and logic for calling -source or -debbuidlers when needed * Implement a test script for release.py and use it in Github Actions. --------- Signed-off-by: Jose Luis Rivero Co-authored-by: Steven! Ragnarök Co-authored-by: Steve Peters --- .github/workflows/ci.yaml | 3 +- check_releasepy.bash | 82 ++++++++ release.py | 382 ++++++++++++++++++-------------------- 3 files changed, 265 insertions(+), 202 deletions(-) create mode 100755 check_releasepy.bash diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 02122458a..c74dad645 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -40,10 +40,11 @@ jobs: with: distribution: 'temurin' java-version: '11' + - name: Run release.py script tests + run: ./check_releasepy.bash - name: Download and setup job dsl jar if: steps.dsl_check.outputs.run_job == 'true' run: ./jenkins-scripts/dsl/tools/setup_local_generation.bash - - name: Generate all DSL files if: steps.dsl_check.outputs.run_job == 'true' run: | diff --git a/check_releasepy.bash b/check_releasepy.bash new file mode 100755 index 000000000..5e71fb168 --- /dev/null +++ b/check_releasepy.bash @@ -0,0 +1,82 @@ +#!/bin/bash -e + +test_dir=$(mktemp -d) +mkdir -p ${test_dir}/{focal,jammy,ubuntu}/debian +export _RELEASEPY_TEST_RELEASE_REPO=${test_dir} + +exec_releasepy_test() +{ + test_params=${1} + + ./release.py \ + --dry-run \ + --no-sanity-checks \ + gz-foo 1.2.3 token ${test_params}"" +} + +expect_job_run() +{ + output="${1}" job="${2}" + + if ! grep -q "job/${job}/buildWith" <<< "${output}"; then + echo "${job} not found in test output" + exit 1 + fi +} + +expect_job_not_run() +{ + output="${1}" job="${2}" + + if grep -q "job/${job}/buildWith" <<< "${output}"; then + echo "${job} found in test output. Should not appear." + exit 1 + fi +} + +expect_number_of_jobs() +{ + output=${1} njobs=${2} + + if [[ $(grep -c "job/.*/buildWithParameters" <<< "${output}") != "${njobs}" ]]; then + echo "Number of jobs called is not the expected ${njobs}" + exit 1 + fi +} + +expect_param() +{ + output="${1}" param="${2}" + + if ! grep -q "[&\?]${param}" <<< "${output}"; then + echo "${param} not found in test output" + exit 1 + fi + +} + +source_repo_uri_test=$(exec_releasepy_test "--source-repo-uri https://github.org/gazebosim/gz-foo") +expect_job_run "${source_repo_uri_test}" "gz-foo-source" +expect_job_not_run "${source_repo_uri_test}" "gz-foo-debbuilder" +expect_number_of_jobs "${source_repo_uri_test}" "1" + +source_tarball_uri_test=$(exec_releasepy_test "--source-tarball-uri https://gazebosim/gz-foo-1.2.3.tar.gz") +expect_job_run "${source_tarball_uri_test}" "gz-foo-debbuilder" +expect_job_run "${source_tarball_uri_test}" "generic-release-homebrew_pull_request_updater" +expect_job_not_run "${source_tarball_uri_test}" "gz-foo-source" +expect_number_of_jobs "${source_tarball_uri_test}" "7" +expect_param "${source_tarball_uri_test}" "SOURCE_TARBALL_URI=https%3A%2F%2Fgazebosim%2Fgz-foo-1.2.3.tar.gz" + +nightly_test=$(exec_releasepy_test "--nightly-src-branch my-nightly-branch3 --upload-to-repo nightly") +expect_job_run "${nightly_test}" "gz-foo-debbuilder" +expect_job_not_run "${nightly_test}" "generic-release-homebrew_pull_request_updater" +expect_job_not_run "${nightly_test}" "gz-foo-source" +expect_number_of_jobs "${nightly_test}" "2" +expect_param "${nightly_test}" "SOURCE_TARBALL_URI=my-nightly-branch3" + +bump_linux_test=$(exec_releasepy_test "--source-tarball-uri https://gazebosim/gz-foo-1.2.3.tar.gz --only-bump-revision-linux -r 2") +expect_job_run "${bump_linux_test}" "gz-foo-debbuilder" +expect_job_not_run "${bump_linux_test}" "generic-release-homebrew_pull_request_updater" +expect_job_not_run "${bump_linux_test}" "gz-foo-source" +expect_number_of_jobs "${bump_linux_test}" "6" +expect_param "${bump_linux_test}" "RELEASE_VERSION=2" diff --git a/release.py b/release.py index fe0730460..db1bd2c35 100755 --- a/release.py +++ b/release.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 from __future__ import print_function +from argparse import RawTextHelpFormatter import subprocess import sys import tempfile @@ -9,7 +10,6 @@ import urllib.request import argparse import shutil -import re USAGE = 'release.py ' try: @@ -18,14 +18,12 @@ JENKINS_URL = 'http://build.osrfoundation.org' JOB_NAME_PATTERN = '%s-debbuilder' GENERIC_BREW_PULLREQUEST_JOB = 'generic-release-homebrew_pull_request_updater' -UPLOAD_DEST_PATTERN = 's3://osrf-distributions/%s/releases/' -DOWNLOAD_URI_PATTERN = 'https://osrf-distributions.s3.amazonaws.com/%s/releases/' LINUX_DISTROS = ['ubuntu', 'debian'] SUPPORTED_ARCHS = ['amd64', 'armhf', 'arm64'] RELEASEPY_NO_ARCH_PREFIX = '.releasepy_NO_ARCH_' -OSRF_REPOS_SUPPORTED = "stable prerelease nightly testing" +OSRF_REPOS_SUPPORTED = "stable prerelease nightly testing none" DRY_RUN = False NIGHTLY = False @@ -92,16 +90,6 @@ def github_repo_exists(url): return True -def generate_package_source(srcdir, builddir): - cmake_cmd = ["cmake"] - - # configure and make package_source - os.mkdir(builddir) - os.chdir(builddir) - check_call(cmake_cmd + [srcdir]) - check_call(['make', 'package_source']) - - def exists_main_branch(github_url): check_main_cmd = ['git', 'ls-remote', '--exit-code', '--heads', github_url, 'main'] try: @@ -116,7 +104,21 @@ def parse_args(argv): global NIGHTLY global PRERELEASE - parser = argparse.ArgumentParser(description='Make releases.') + parser = argparse.ArgumentParser(formatter_class=RawTextHelpFormatter, + description=""" +Script to handle the release process for the Gazebo devs. +Examples: +A) Generate source: local repository tag + call source job: + $ release.py + (auto calculate source-repo-uri from local directory) + +B) Call builders: reuse existing tarball version + call build jobs: + $ release.py --source-tarball-uri + (no call to source job, directly build jobs with tarball URL) + +C) Nightly builds (linux) + $ release.py --source-repo-existing-ref --upload-to-repo nightly + """) parser.add_argument('package', help='which package to release') parser.add_argument('version', help='which version to release') parser.add_argument('jenkins_token', help='secret token to allow access to Jenkins to start builds') @@ -133,8 +135,18 @@ def parse_args(argv): help='Release version suffix; usually 1 (e.g., 1') parser.add_argument('--no-sanity-checks', dest='no_sanity_checks', action='store_true', default=False, help='no-sanity-checks; i.e. skip sanity checks commands') - parser.add_argument('--no-generate-source-file', dest='no_source_file', action='store_true', default=False, - help='Do not generate source file when building') + parser.add_argument('--source-repo-uri', + dest='source_repo_uri', + default=None, + help='Indicate the repository URL to grab the source from (overriding the heristics to calculate it from the local directory)') # NOQA + parser.add_argument('--source-repo-existing-ref', + dest='source_repo_ref', + default=None, + help='Optionally, when using --source-repo-uri, indicate the Git reference (branch|tag) to grab the release sources from.\ + If used: avoid to tag the local repository. If not used: tag the local repository with and use it as ref') # NOQA + parser.add_argument('--source-tarball-uri', + dest='source_tarball_uri', default=None, + help='Indicate the URL of the sources to grab the release sources from.') # NOQA parser.add_argument('--upload-to-repo', dest='upload_to_repository', default="stable", help='OSRF repo to upload: stable | prerelease | nightly') parser.add_argument('--extra-osrf-repo', dest='extra_repo', default="", @@ -158,9 +170,6 @@ def parse_args(argv): NIGHTLY = True if args.upload_to_repository == 'prerelease': PRERELEASE = True - # Nightly do not generate a tar.bz2 file - if NIGHTLY: - args.no_source_file = True return args @@ -172,21 +181,25 @@ def get_release_repository_info(package): error("release repository not found in github.com/gazebo-release") + def download_release_repository(package, release_branch): - vcs, url = get_release_repository_info(package) - release_tmp_dir = tempfile.mkdtemp() + try: + return os.environ['_RELEASEPY_TEST_RELEASE_REPO'], 'main' + except KeyError: + vcs, url = get_release_repository_info(package) + release_tmp_dir = tempfile.mkdtemp() - if vcs == "git" and release_branch == "default": - release_branch = "master" + if vcs == "git" and release_branch == "default": + release_branch = "master" - # If main branch exists, prefer it over master - if release_branch == "master": - if exists_main_branch(url): - release_branch = 'main' + # If main branch exists, prefer it over master + if release_branch == "master": + if exists_main_branch(url): + release_branch = 'main' - cmd = [vcs, "clone", "-b", release_branch, url, release_tmp_dir] - check_call(cmd, IGNORE_DRY_RUN) - return release_tmp_dir, release_branch + cmd = [vcs, "clone", "-b", release_branch, url, release_tmp_dir] + check_call(cmd, IGNORE_DRY_RUN) + return release_tmp_dir, release_branch def sanity_package_name_underscore(package, package_alias): @@ -267,7 +280,8 @@ def sanity_check_repo_name(repo_name): if repo_name in OSRF_REPOS_SUPPORTED: return - error("Upload repo value: " + repo_name + " is not valid. stable | prerelease | nightly") + error(f"Upload repo value: {repo_name} is not valid. \ + Supported values are: {OSRF_REPOS_SUPPORTED}") def sanity_project_package_in_stable(version, repo_name): @@ -283,12 +297,32 @@ def sanity_project_package_in_stable(version, repo_name): return +def sanity_check_source_repo_uri(source_repo_uri): + # Check if the scheme is "https" and the path ends with ".git" + parsed_uri = urllib.parse.urlparse(source_repo_uri) + if not parsed_uri.scheme == "https" or \ + not parsed_uri.path.endswith(".git"): + error("--source-repo-uri parameter should start with https:// and end with .git") + + +def sanity_check_bump_linux(source_tarball_uri): + if (not source_tarball_uri): + error('--only-bump-revision-linux needs --source-tarball-uri argument' + 'to call builders and not source generation') + + def sanity_checks(args, repo_dir): print("Safety checks:") sanity_package_name_underscore(args.package, args.package_alias) sanity_package_name(repo_dir, args.package, args.package_alias) sanity_check_repo_name(args.upload_to_repository) + if (args.bump_rev_linux_only): + sanity_check_bump_linux(args.source_tarball_uri) + + if args.source_repo_uri: + sanity_check_source_repo_uri(args.source_repo_uri) + if not NIGHTLY: sanity_package_version(repo_dir, args.version, str(args.release_version)) sanity_check_sdformat_versions(args.package, args.version) @@ -336,7 +370,7 @@ def discover_distros(repo_dir): arches_supported = [x for x in SUPPORTED_ARCHS if x not in excluded_arches] distro_arch_list[d] = arches_supported - print('Releasing for distributions: ') + print('Distributions in release-repo:') for k in distro_arch_list: print("- " + k + " (" + ', '.join(distro_arch_list[k]) + ")") @@ -371,38 +405,6 @@ def check_call(cmd, ignore_dry_run=False): return out, err -# Returns tarball name: package name/alias without versions -def create_tarball_name(args): - return re.sub(r'[0-9]+$', '', args.package) - - -# Returns: sha, tarball file name, tarball full path -def create_tarball_path(tarball_name, version, builddir, dry_run): - tarball_fname = '%s-%s.tar.bz2' % (tarball_name, version) - # Try using the tarball_name as it is - tarball_path = os.path.join(builddir, tarball_fname) - - if not os.path.isfile(tarball_path): - # Try looking for special project names using underscores - alt_tarball_name = "_".join(tarball_name.rsplit("-", 1)) - alt_tarball_fname = '%s-%s.tar.bz2' % (alt_tarball_name, version) - alt_tarball_path = os.path.join(builddir, alt_tarball_fname) - if (not dry_run): - if not os.path.isfile(alt_tarball_path): - error("Can not find a tarball at: " + tarball_path + " or at " + alt_tarball_path) - else: - tarball_fname = alt_tarball_fname - tarball_path = alt_tarball_path - - out, err = check_call(['shasum', '--algorithm', '256', tarball_path]) - if err: - error("shasum returned an error: " + err.decode()) - if isinstance(out, bytes): - out = out.decode() - - return out.split(' ')[0], tarball_fname, tarball_path - - def tag_repo(args): try: # tilde is not a valid character in git @@ -426,78 +428,45 @@ def tag_repo(args): return tag -def generate_upload_tarball(args): - ################################################### - # Platform-agnostic stuff. - # The goal is to tag the repo and prepare a tarball. - - sourcedir = os.getcwd() - tmpdir = tempfile.mkdtemp() - builddir = os.path.join(tmpdir, 'build') - - # Note for bump_rev_linux_only: there are some adjustment to the tarball name - # that are done after generating it, even if the tarball upload is not - # needed, it should be generated to get changes in the name - if args.bump_rev_linux_only: - print('\nINFO: bump revision is enabled. It needs to generate a local tarball although it will not upload it') - - # Check for uncommitted changes; abort if present - cmd = ['git', 'diff-index', 'HEAD'] - out, err = check_call(cmd) - if out: - print('git says that you have uncommitted changes') - print('Please clean up your working copy so that "%s" outputs nothing' % (' '.join(cmd))) - print('stdout: %s' % (out.decode())) +def generate_source_repository_uri(args): + org_repo = f"gazebosim/{get_canonical_package_name(args.package_alias)}" + out, err = check_call(['git', 'ls-remote', '--get-url', 'origin'], + IGNORE_DRY_RUN) + if err: + print(f"An error happened running git ls-remote: ${err}") sys.exit(1) - # Make a clean copy, to avoid pulling in other stuff that the user has - # sitting in the working copy. - srcdir = os.path.join(tmpdir, 'src') - os.mkdir(srcdir) - tmp_tar = os.path.join(tmpdir, 'orig.tar') - check_call(['git', 'archive', '--format=tar', 'HEAD', '-o', tmp_tar]) - check_call(['tar', 'xf', tmp_tar, '-C', srcdir]) - if not args.dry_run: - os.remove(tmp_tar) - - # use cmake to generate package_source - generate_package_source(srcdir, builddir) - # For ignition, we use the alias without version numbers as package name - tarball_name = re.sub(r'[0-9]+$', '', args.package_alias) - tarball_sha, tarball_fname, tarball_path = create_tarball_path(tarball_name, args.version, builddir, args.dry_run) - - # If we're releasing under a different name, then rename the tarball (the - # package itself doesn't know anything about this). - if args.package != args.package_alias: - tarball_fname = '%s-%s.tar.bz2' % (args.package_alias, args.version) - if (not args.dry_run): - dest_file = os.path.join(builddir, tarball_fname) - # Do not copy if files are the same - if not (tarball_path == dest_file): - shutil.copyfile(tarball_path, dest_file) - tarball_path = dest_file - - s3_tarball_directory = UPLOAD_DEST_PATTERN % get_canonical_package_name(args.package) - source_tarball_uri = DOWNLOAD_URI_PATTERN % get_canonical_package_name(args.package) + tarball_fname - - # If the release only bump revision does not need to upload tarball but - # checkout that the one that should be already uploaded exists - if args.bump_rev_linux_only: - if urllib.request.urlopen(source_tarball_uri).getcode() == '404': - print('Can not find tarball: %s' % (source_tarball_uri)) - sys.exit(1) - else: - check_call(['s3cmd', 'sync', tarball_path, s3_tarball_directory]) - shutil.rmtree(tmpdir) + git_remote = out.decode().split('\n')[0] + if org_repo not in git_remote: + print(f""" !! Automatic calculation of source_repo_uri failed.\ + \n * git remote origin is: {git_remote}\ + \n * Package name generated org/repo: {org_repo}\ + \n >> Please use --source-repo-uri parameter""") + sys.exit(1) + + return f"https://github.com/{org_repo}.git" # NOQA + - # Tag repo - os.chdir(sourcedir) - _ = tag_repo(args) +def generate_source_params(args): + params = {} + # 1. NIGHTLY (launch builders): + # Pass the nightly branch if NIGHTLY enabled + # 2. Launch builders + # Using args.source_tarball_uri if present + # 3. Launch source jobs (SOURCE_REPO_URI) + # 1.1 using args.source_repo_uri (if it was passed) + # 1.2 autogenerating it + + if NIGHTLY: + params['SOURCE_TARBALL_URI'] = args.nightly_branch + elif args.source_tarball_uri: + params['SOURCE_TARBALL_URI'] = args.source_tarball_uri + else: + params['SOURCE_REPO_URI'] = \ + args.source_repo_uri if args.source_repo_uri else \ + generate_source_repository_uri(args) - # TODO: Consider auto-updating the Ubuntu changelog. It requires - # cloning the -release repo, making a change, and pushing it back. - # Until we do that, the user must have first updated it manually. - return source_tarball_uri, tarball_sha + return params def call_jenkins_build(job_name, params, output_string): @@ -525,20 +494,10 @@ def go(argv): if not args.no_sanity_checks: sanity_checks(args, repo_dir) - source_tarball_uri = '' - source_tarball_sha = '' - - # Do not generate source file if not needed or impossible - if not args.no_source_file: - source_tarball_uri, source_tarball_sha = generate_upload_tarball(args) - - # Kick off Jenkins jobs - params = {} + params = generate_source_params(args) params['token'] = args.jenkins_token params['PACKAGE'] = args.package - params['VERSION'] = args.version - params['SOURCE_TARBALL_URI'] = source_tarball_uri - params['SOURCE_TARBALL_SHA'] = source_tarball_sha + params['VERSION'] = args.version if not NIGHTLY else 'nightly' params['RELEASE_REPO_BRANCH'] = args.release_repo_branch params['PACKAGE_ALIAS'] = args.package_alias params['RELEASE_VERSION'] = args.release_version @@ -548,65 +507,86 @@ def go(argv): if args.extra_repo: params['OSRF_REPOS_TO_USE'] += " " + args.extra_repo - if NIGHTLY: - params['VERSION'] = 'nightly' - # reuse SOURCE_TARBALL_URI to indicate the nightly branch - # name must be modified in the future - params['SOURCE_TARBALL_URI'] = args.nightly_branch - - # RELEASING FOR BREW - if not NIGHTLY and not args.bump_rev_linux_only: - call_jenkins_build(GENERIC_BREW_PULLREQUEST_JOB, - params, 'Brew') - - # RELEASING FOR LINUX - for l in LINUX_DISTROS: - if (l == 'ubuntu'): - distros_dic = ubuntu_distros - elif (l == 'debian'): - if (PRERELEASE or NIGHTLY): - continue - if not debian_distros: - continue - distros_dic = debian_distros - else: - error("Distro not supported in code") - for d in distros_dic: - for a in distros_dic[d]: - # Filter prerelease and nightly architectures + # a) Mode nightly or builders: + if NIGHTLY or args.source_tarball_uri: + # RELEASING FOR BREW + if not NIGHTLY and not args.bump_rev_linux_only: + call_jenkins_build(GENERIC_BREW_PULLREQUEST_JOB, + params, 'Brew') + # RELEASING FOR LINUX + for l in LINUX_DISTROS: + if (l == 'ubuntu'): + distros_dic = ubuntu_distros + elif (l == 'debian'): if (PRERELEASE or NIGHTLY): + continue + if not debian_distros: + continue + distros_dic = debian_distros + else: + error("Distro not supported in code") + + for d in distros_dic: + for a in distros_dic[d]: + # Filter prerelease and nightly architectures + if (PRERELEASE or NIGHTLY): + if (a == 'armhf' or a == 'arm64'): + continue + + linux_platform_params = params.copy() + linux_platform_params['ARCH'] = a + linux_platform_params['LINUX_DISTRO'] = l + linux_platform_params['DISTRO'] = d + if (a == 'armhf' or a == 'arm64'): - continue - - linux_platform_params = params.copy() - linux_platform_params['ARCH'] = a - linux_platform_params['LINUX_DISTRO'] = l - linux_platform_params['DISTRO'] = d - - if (a == 'armhf' or a == 'arm64'): - # No sid releases for arm64/armhf lack of docker image - # https://hub.docker.com/r/aarch64/debian/ fails on Jenkins - if (d == 'sid'): - continue - # Need to use JENKINS_NODE_TAG parameter for large memory nodes - # since it runs qemu emulation - linux_platform_params['JENKINS_NODE_TAG'] = 'linux-' + a - elif ('ignition-physics' in args.package_alias) or \ - ('gz-physics' in args.package_alias): - linux_platform_params['JENKINS_NODE_TAG'] = 'large-memory' - - # control nightly generation using a single machine to process - # all distribution builds to avoid race conditions. Note: this - # assumes that large-memory nodes are beind used for nightly - # tags. - # https://github.com/gazebo-tooling/release-tools/issues/644 - if (NIGHTLY): - assert a == 'amd64', f'Nightly tag assumed amd64 but arch is {a}' - linux_platform_params['JENKINS_NODE_TAG'] = 'linux-nightly-' + d - - call_jenkins_build(f"{args.package}-debbuilder", - linux_platform_params, f"{l} {d}/{a}") + # No sid releases for arm64/armhf lack of docker image + # https://hub.docker.com/r/aarch64/debian/ fails on Jenkins + if (d == 'sid'): + continue + # Need to use JENKINS_NODE_TAG parameter for large memory nodes + # since it runs qemu emulation + linux_platform_params['JENKINS_NODE_TAG'] = 'linux-' + a + elif ('ignition-physics' in args.package_alias) or \ + ('gz-physics' in args.package_alias): + linux_platform_params['JENKINS_NODE_TAG'] = 'large-memory' + + # control nightly generation using a single machine to process + # all distribution builds to avoid race conditions. Note: this + # assumes that large-memory nodes are beind used for nightly + # tags. + # https://github.com/gazebo-tooling/release-tools/issues/644 + if (NIGHTLY): + assert a == 'amd64', f'Nightly tag assumed amd64 but arch is {a}' + linux_platform_params['JENKINS_NODE_TAG'] = 'linux-nightly-' + d + + call_jenkins_build(f"{args.package_alias}-debbuilder", + linux_platform_params, f"{l} {d}/{a}") + else: + # b) Mode generate source + # Choose platform to run gz-source on. It will need to install gz-cmake + # Take the first key in the supported distros since all them should be + # able to install the needed gz-cmake. + if ubuntu_distros: + params['LINUX_DISTRO'] = 'ubuntu' + params['DISTRO'] = list(ubuntu_distros.keys())[0] + elif debian_distros: + params['LINUX_DISTRO'] = 'debian' + params['DISTRO'] = list(debian_distros.keys())[0] + else: + error("No distributions where found in the release repo") + + # Tag should not go before any method or step that can fail and just + # before the calls to the servers. + if not args.source_repo_ref: + print('INFO: no --source-repo-existing-ref used, tag the local' + 'repository as the reference for the source code of the' + 'release') + + params['SOURCE_REPO_REF'] = tag_repo(args) \ + if not args.source_repo_ref else args.source_repo_ref + + call_jenkins_build(f"{args.package_alias}-source", params, 'Source') if __name__ == '__main__': From 40743f1d9c8303132045ec8639febbeb5b710ce3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Crist=C3=B3bal=20Arroyo?= Date: Mon, 6 Nov 2023 09:48:20 -0500 Subject: [PATCH 516/540] Update ign-sim6 name (#1056) Signed-off-by: Crola1702 --- jenkins-scripts/tools/outdated-job-runner.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/tools/outdated-job-runner.groovy b/jenkins-scripts/tools/outdated-job-runner.groovy index 3d958e311..a085d6650 100644 --- a/jenkins-scripts/tools/outdated-job-runner.groovy +++ b/jenkins-scripts/tools/outdated-job-runner.groovy @@ -97,7 +97,7 @@ def trackedJobsList = [ 'gz_sensors-ci-ign-sensors6-focal-amd64', 'gz_sim-ci-gz-sim7-focal-amd64', 'gz_sim-ci-gz-sim8-jammy-amd64', - 'gz_sim-ci-ign-gazebo6-focal-amd64', + 'gz_sim-ci-ign-sim6-focal-amd64', 'gz_tools-ci-gz-tools2-jammy-amd64', 'gz_tools-ci-ign-tools1-focal-amd64', 'gz_transport-ci-gz-transport12-focal-amd64', From 691ba1dbc8bc82db7f7319b150996d1fc0600f91 Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Mon, 6 Nov 2023 10:10:49 -0800 Subject: [PATCH 517/540] ignition_collection.dsl: add harmonic bottle jobs (#1055) Signed-off-by: Steve Peters --- jenkins-scripts/dsl/ignition_collection.dsl | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index ea4b72053..fcb2fb7a9 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -306,23 +306,40 @@ gz_collection_jobs = 'gz_transport-ci-gz-transport13-jammy-amd64', 'gz_utils-ci-gz-utils2-jammy-amd64', 'ignition_cmake-ci-gz-cmake3-homebrew-amd64', + 'ignition_cmake3-install_bottle-homebrew-amd64', 'ignition_common-ci-gz-common5-homebrew-amd64', + 'ignition_common5-install_bottle-homebrew-amd64', 'ignition_fuel-tools-ci-gz-fuel-tools9-homebrew-amd64', + 'ignition_fuel-tools9-install_bottle-homebrew-amd64', 'ignition_harmonic-ci-main-homebrew-amd64', + 'ignition_harmonic-install_bottle-homebrew-amd64', 'ignition_gazebo-ci-gz-sim8-homebrew-amd64', + 'ignition_sim8-install_bottle-homebrew-amd64', 'ignition_gui-ci-gz-gui8-homebrew-amd64', + 'ignition_gui8-install_bottle-homebrew-amd64', 'ignition_launch-gz-launch7-homebrew-amd64', + 'ignition_launch7-install_bottle-homebrew-amd64', 'ignition_math-ci-gz-math7-homebrew-amd64', + 'ignition_math7-install_bottle-homebrew-amd64', 'ignition_msgs-ci-gz-msgs10-homebrew-amd64', + 'ignition_msgs10-install_bottle-homebrew-amd64', 'ignition_physics-ci-gz-physics7-homebrew-amd64', + 'ignition_physics7-install_bottle-homebrew-amd64', 'ignition_plugin-ci-gz-plugin2-homebrew-amd64', + 'ignition_plugin2-install_bottle-homebrew-amd64', 'ignition_rendering-ci-gz-rendering8-homebrew-amd64', + 'ignition_rendering8-install_bottle-homebrew-amd64', 'ignition_sensors-ci-gz-sensors8-homebrew-amd64', + 'ignition_sensors8-install_bottle-homebrew-amd64', 'ignition_tools-ci-gz-tools2-homebrew-amd64', + 'ignition_tools2-install_bottle-homebrew-amd64', 'ignition_transport-ci-gz-transport13-homebrew-amd64', + 'ignition_transport13-install_bottle-homebrew-amd64', 'ignition_utils-ci-gz-utils2-homebrew-amd64', + 'ignition_utils2-install_bottle-homebrew-amd64', 'sdformat-ci-sdformat14-jammy-amd64', 'sdformat-ci-sdformat14-homebrew-amd64', + 'sdformat14-install_bottle-homebrew-amd64', 'sdformat-sdf-14-win' ], 'ionic' : [ From 1e64f565f45eb7df789b5f382f994fb9827d2b5f Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 7 Nov 2023 12:47:42 +0100 Subject: [PATCH 518/540] Move brew ci-pr_any- to gazebo_libs (#1054) * Add brew ci configuration to yaml file * Generate brew pr_any jobs in gazebo_libs * Include GITHUB_SUPPORT_ALL_BRANCHES in -ci-pr_any- * Move current brew ci-pr_any jobs to return always true * Workaround to exclude brew but keep current jobs --------- Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/gazebo_libs.dsl | 169 ++++++++++++-------- jenkins-scripts/dsl/gz-collections.yaml | 35 ++++ jenkins-scripts/dsl/ignition.dsl | 27 +--- jenkins-scripts/dsl/ignition_collection.dsl | 9 +- 4 files changed, 147 insertions(+), 93 deletions(-) diff --git a/jenkins-scripts/dsl/gazebo_libs.dsl b/jenkins-scripts/dsl/gazebo_libs.dsl index 700f98913..897477039 100644 --- a/jenkins-scripts/dsl/gazebo_libs.dsl +++ b/jenkins-scripts/dsl/gazebo_libs.dsl @@ -8,6 +8,8 @@ GLOBAL_SHELL_CMD='' // GZ COLLECTIONS ENABLE_CPPCHECK = true +GITHUB_SUPPORT_ALL_BRANCHES = [] +ENABLE_GITHUB_PR_INTEGRATION = true def WRITE_JOB_LOG = System.getenv('WRITE_JOB_LOG') ?: false logging_list = [:] @@ -62,6 +64,11 @@ boolean is_testing_enabled(lib_name, ci_config) return ! ci_config.tests_disabled?.contains(lib_name) } +boolean are_cmake_warnings_enabled(lib_name, ci_config) +{ + return ! ci_config.cmake_warnings_disabled?.contains(lib_name) +} + /* * Generate an index that facilitates the operations with the yaml values, * avoiding to parse them several times. @@ -155,63 +162,33 @@ ciconf_per_lib_index.each { lib_name, lib_configs -> def ci_config = gz_collections_yaml.ci_configs.find{ it.name == config_name } def branches_with_collections = ci_configs.getValue() def branch_names = branches_with_collections.collect { it.branch }.unique() + def script_name_prefix = cleanup_library_name(lib_name) + def gz_job_name_prefix = lib_name.replaceAll('-','_') if (ci_config.exclude.all?.contains(lib_name)) return assert(lib_name) assert(branch_names) assert(ci_config) - // Main PR jobs (-ci-pr_any-) (pulling check every 5 minutes) - // -------------------------------------------------------------- - def script_name_prefix = cleanup_library_name(lib_name) - def distro = ci_config.system.version - def arch = ci_config.system.arch - def gz_job_name_prefix = lib_name.replaceAll('-','_') - def pre_setup_script = ci_config.pre_setup_script_hook?.get(lib_name)?.join('\n') - def extra_cmd = pre_setup_script ?: "" - - // Main PR jobs (-ci-pr_any-) (pulling check every 5 minutes) - // -------------------------------------------------------------- - def gz_ci_job_name = "${gz_job_name_prefix}-ci-pr_any-${distro}-${arch}" - def gz_ci_any_job = job(gz_ci_job_name) - OSRFLinuxCompilationAnyGitHub.create(gz_ci_any_job, - "gazebosim/${lib_name}", - is_testing_enabled(lib_name, ci_config), - ENABLE_CPPCHECK, - branch_names) - generate_label_by_requirements(gz_ci_any_job, lib_name, ci_config.requirements) - gz_ci_any_job.with - { - steps - { - shell("""\ - #!/bin/bash -xe - - export DISTRO=${distro} - - ${GLOBAL_SHELL_CMD} - ${extra_cmd} - - export BUILDING_SOFTWARE_DIRECTORY=${lib_name} - export ARCH=${arch} - /bin/bash -xe ./scripts/jenkins-scripts/docker/${script_name_prefix}-compilation.bash - """.stripIndent()) - } // end of steps - } // end of ci_any_job - - if (! ci_config.exclude.abichecker?.contains(lib_name)) { - // ABI branch jobs (-ci-abichecker-) for non main branches - def abi_job_name = "${gz_job_name_prefix}-abichecker-any_to_any-ubuntu-${distro}-${arch}" - def abi_job = job(abi_job_name) - OSRFLinuxABIGitHub.create(abi_job) - GenericAnyJobGitHub.create(abi_job, - "gazebosim/${lib_name}", - branch_names - [ 'main']) - generate_label_by_requirements(abi_job, lib_name, ci_config.requirements) - abi_job.with + if (ci_config.system.so == 'linux') { + def distro = ci_config.system.version + def arch = ci_config.system.arch + def pre_setup_script = ci_config.pre_setup_script_hook?.get(lib_name)?.join('\n') + def extra_cmd = pre_setup_script ?: "" + + def gz_ci_job_name = "${gz_job_name_prefix}-ci-pr_any-${distro}-${arch}" + def gz_ci_any_job = job(gz_ci_job_name) + OSRFLinuxCompilationAnyGitHub.create(gz_ci_any_job, + "gazebosim/${lib_name}", + is_testing_enabled(lib_name, ci_config), + ENABLE_CPPCHECK, + branch_names) + generate_label_by_requirements(gz_ci_any_job, lib_name, ci_config.requirements) + gz_ci_any_job.with { - steps { - shell("""\ + steps + { + shell("""\ #!/bin/bash -xe export DISTRO=${distro} @@ -219,33 +196,85 @@ ciconf_per_lib_index.each { lib_name, lib_configs -> ${GLOBAL_SHELL_CMD} ${extra_cmd} + export BUILDING_SOFTWARE_DIRECTORY=${lib_name} export ARCH=${arch} - export DEST_BRANCH=\${DEST_BRANCH:-\$ghprbTargetBranch} - export SRC_BRANCH=\${SRC_BRANCH:-\$ghprbSourceBranch} - export SRC_REPO=\${SRC_REPO:-\$ghprbAuthorRepoGitUrl} - export ABI_JOB_SOFTWARE_NAME=${lib_name} - /bin/bash -xe ./scripts/jenkins-scripts/docker/gz-abichecker.bash + /bin/bash -xe ./scripts/jenkins-scripts/docker/${script_name_prefix}-compilation.bash """.stripIndent()) } // end of steps - } // end of with - } + } // end of ci_any_job + + if (! ci_config.exclude.abichecker?.contains(lib_name)) { + // ABI branch jobs (-ci-abichecker-) for non main branches + def abi_job_name = "${gz_job_name_prefix}-abichecker-any_to_any-ubuntu-${distro}-${arch}" + def abi_job = job(abi_job_name) + OSRFLinuxABIGitHub.create(abi_job) + GenericAnyJobGitHub.create(abi_job, + "gazebosim/${lib_name}", + branch_names - [ 'main']) + generate_label_by_requirements(abi_job, lib_name, ci_config.requirements) + abi_job.with + { + steps { + shell("""\ + #!/bin/bash -xe + + export DISTRO=${distro} + + ${GLOBAL_SHELL_CMD} + ${extra_cmd} + + export ARCH=${arch} + export DEST_BRANCH=\${DEST_BRANCH:-\$ghprbTargetBranch} + export SRC_BRANCH=\${SRC_BRANCH:-\$ghprbSourceBranch} + export SRC_REPO=\${SRC_REPO:-\$ghprbAuthorRepoGitUrl} + export ABI_JOB_SOFTWARE_NAME=${lib_name} + /bin/bash -xe ./scripts/jenkins-scripts/docker/gz-abichecker.bash + """.stripIndent()) + } // end of steps + } // end of with + } + + // CI branch jobs (-ci-$branch-) (pulling check every 5 minutes) + branches_with_collections.each { branch_and_collection -> + branch_name = branch_and_collection.branch + def gz_ci_job = job("${gz_job_name_prefix}-ci-${branch_name}-${distro}-${arch}") + generate_ci_job(gz_ci_job, lib_name, branch_name, ci_config) + gz_ci_job.with + { + triggers { + scm('@daily') + } + } - // CI branch jobs (-ci-$branch-) (pulling check every 5 minutes) - branches_with_collections.each { branch_and_collection -> - branch_name = branch_and_collection.branch - def gz_ci_job = job("${gz_job_name_prefix}-ci-${branch_name}-${distro}-${arch}") - generate_ci_job(gz_ci_job, lib_name, branch_name, ci_config) - gz_ci_job.with + logging_list['branch_ci'].add( + [collection: branch_and_collection.collection, + job_name: gz_ci_job.name]) + } // end_of_branch + } else if (ci_config.system.so == 'darwin') { + // -------------------------------------------------------------- + def gz_brew_ci_any_job_name = "${gz_job_name_prefix}-ci-pr_any-homebrew-amd64" + def gz_brew_ci_any_job = job(gz_brew_ci_any_job_name) + // TODO: Instead of GITHUB_SUPPORT_ALL_BRANCHES the right configuration would be $branch_names + // leave all the branches since we only support one brew platform and it covers the + // cases of PRs targeting not stable branches, see: + // https://github.com/gazebo-tooling/release-tools/issues/1044 + OSRFBrewCompilationAnyGitHub.create(gz_brew_ci_any_job, + "gazebosim/${lib_name}", + is_testing_enabled(lib_name, ci_config), + GITHUB_SUPPORT_ALL_BRANCHES, + ENABLE_GITHUB_PR_INTEGRATION, + are_cmake_warnings_enabled(lib_name, ci_config)) + gz_brew_ci_any_job.with { - triggers { - scm('@daily') + steps { + shell("""\ + #!/bin/bash -xe + + /bin/bash -xe "./scripts/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash" "${lib_name}" + """.stripIndent()) } } - - logging_list['branch_ci'].add( - [collection: branch_and_collection.collection, - job_name: gz_ci_job.name]) - } // end_of_branch + } } //en of lib_configs } // end of lib diff --git a/jenkins-scripts/dsl/gz-collections.yaml b/jenkins-scripts/dsl/gz-collections.yaml index 9a0549acb..f65ca7473 100644 --- a/jenkins-scripts/dsl/gz-collections.yaml +++ b/jenkins-scripts/dsl/gz-collections.yaml @@ -95,6 +95,7 @@ collections: ci: configs: - bionic + - brew packaging: configs: - bionic @@ -174,6 +175,7 @@ collections: ci: configs: - focal + - brew packaging: configs: - focal @@ -253,6 +255,7 @@ collections: ci: configs: - focal + - brew packaging: configs: - focal @@ -332,6 +335,7 @@ collections: ci: configs: - jammy + - brew packaging: configs: - jammy @@ -407,6 +411,7 @@ collections: ci: configs: - jammy + - brew packaging: configs: - jammy @@ -425,6 +430,7 @@ collections: ci: configs: - jammy + - brew packaging: configs: - jammy @@ -510,6 +516,35 @@ ci_configs: gz-physics: - "export MAKE_JOBS=1" tests_disabled: + - name: brew + system: + so: darwin + distribution: macOSX + version: all + arch: amd64 + exclude: + all: + - ign-citadel + - gz-fortress + - gz-garden + - gz-harmonic + - gz-ionic + - __upcoming__ + cmake_warnings_disabled: + - gz-cmake + - gz-common + - gz-fuel-tools + - gz-sim + - gz-gui + - gz-launch + - gz-math + - gz-msgs + - gz-physics + - gz-rendering + - gz-sensors + - gz-tools + - gz-transport + - gz-utils packaging_configs: - name: bionic system: diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index c845386a7..2816c8856 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -589,33 +589,16 @@ gz_software.each { gz_sw -> software_name = "gazebo" String gz_brew_ci_any_job_name = "ignition_${software_name}-ci-pr_any-homebrew-amd64" - - def gz_brew_ci_any_job = job(gz_brew_ci_any_job_name) - OSRFBrewCompilationAnyGitHub.create(gz_brew_ci_any_job, - "gazebosim/ign-${software_name}", - enable_testing(software_name), - GITHUB_SUPPORT_ALL_BRANCHES, - ENABLE_GITHUB_PR_INTEGRATION, - enable_cmake_warnings(software_name)) + GenericAnyJobGitHub.create(gz_brew_ci_any_job, + "gazebosim/ign-${software_name}", + GITHUB_SUPPORT_ALL_BRANCHES, + ENABLE_GITHUB_PR_INTEGRATION) gz_brew_ci_any_job.with { - steps { - shell("""\ - #!/bin/bash -xe - - software_name="gz-${software_name}" - [[ ${software_name} == 'gazebo' ]] && software_name="gz-sim" - /bin/bash -xe "./scripts/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash" "\${software_name}" - """.stripIndent()) - } + description 'Automatic generated job by DSL jenkins. Stub job for migration, not doing any check' } - // do not add gazebo twice - // add ci-pr_any to the list for CIWorkflow - if (gz_sw != 'sim') - ci_pr_any_list[software_name] << gz_brew_ci_any_job_name - // 2. main, release branches all_branches("${software_name}").each { branch -> def gz_brew_ci_job = job("ignition_${software_name}-ci-${branch}-homebrew-amd64") diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index fcb2fb7a9..7a046c26c 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -508,7 +508,14 @@ gz_collections_yaml.collections.each { collection -> distro = ci_config.system.version arch = ci_config.system.arch - if (ci_config.exclude.all?.contains(gz_collection_name)) + // This should really include these two conditions to be effective: + // ci_config.exclude.all?.contains("gz-" + gz_collection_name) + // ci_config.exclude.all?.contains("ign-" + gz_collection_name) + // However the collection package being processed here are superseed by the migration + // to yaml efforts. Remove from here when -install- and -win- jobs are implemented + // in gazebo_libs + if (ci_config.exclude.all?.contains(gz_collection_name) || + ci_config.name == "brew" ) return // INSTALL JOBS: From a33c07eb52472680994b26a0e3d74977b44b177e Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 7 Nov 2023 14:15:23 +0100 Subject: [PATCH 519/540] Sort log generated files from DSL (#1057) Signed-off-by: Jose Luis Rivero --- .github/workflows/ci.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c74dad645..02a6c8290 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -64,6 +64,8 @@ jobs: cd jenkins-scripts/dsl # export files for later diff mkdir /tmp/pr_xml_configuration && mkdir /tmp/pr_log_generated/ + # sort jobs.txt + sort jobs.txt -o jobs.txt mv *.xml /tmp/pr_xml_configuration/ mv *.txt /tmp/pr_log_generated/ - name: Generate master DSL files @@ -74,6 +76,8 @@ jobs: cd jenkins-scripts/dsl WRITE_JOB_LOG=1 java -jar tools/jobdsl.jar *.dsl mkdir /tmp/current_xml_configuration && mkdir /tmp/current_log_generated/ + # sort jobs.txt + sort jobs.txt -o jobs.txt mv *.xml /tmp/current_xml_configuration/ mv *.txt /tmp/current_log_generated/ || true - name: Generating diffs From ca67293e9bb267cef6ba089066c1e698c4768ad6 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 7 Nov 2023 18:31:11 +0100 Subject: [PATCH 520/540] Alternative keyserver for Bionic (#1059) Signed-off-by: Jose Luis Rivero --- .../docker/lib/docker_generate_dockerfile.bash | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash b/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash index 9b8831017..4346eb904 100644 --- a/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash +++ b/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash @@ -206,6 +206,11 @@ DELIM_DOCKER_DIRMNGR # Install necessary repositories using gzdev dockerfile_install_gzdev_repos +KEYSERVER="keyserver.ubuntu.com" +if [ "${DISTRO}" == 'bionic' ]; then + KEYSERVER="keys.openpgp.org" +fi + if ${USE_ROS_REPO}; then if ${ROS2}; then cat >> Dockerfile << DELIM_ROS_REPO @@ -224,14 +229,14 @@ DELIM_ROS_REPO cat >> Dockerfile << DELIM_ROS_REPO RUN echo "deb http://repos.ros.org/repos/ros_bootstrap/ ${DISTRO} main" > \\ /etc/apt/sources.list.d/ros_bootstrap.list -RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 8EDB2EF661FC880E +RUN apt-key adv --keyserver ${KEYSERVER} --recv-keys 8EDB2EF661FC880E DELIM_ROS_REPO else cat >> Dockerfile << DELIM_ROS_REPO # Note that ROS uses ubuntu hardcoded in the paths of repositories RUN echo "deb http://packages.ros.org/${ROS_REPO_NAME}/ubuntu ${DISTRO} main" > \\ /etc/apt/sources.list.d/ros.list -RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys F42ED6FBAB17C654 +RUN apt-key adv --keyserver ${KEYSERVER} --recv-keys F42ED6FBAB17C654 DELIM_ROS_REPO # Need ros stable for the cases of ros-testing if [[ ${ROS_REPO_NAME} != "ros" ]]; then From cfa04bd783545b11d537dfc7a262de784fee4d52 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 7 Nov 2023 18:42:18 +0100 Subject: [PATCH 521/540] Cleanup old chaning mechanism for workspace (#1060) Signed-off-by: Jose Luis Rivero --- .../lib/project-default-devel-homebrew-amd64.bash | 9 --------- 1 file changed, 9 deletions(-) diff --git a/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash b/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash index 5f05056f5..a32af3d84 100644 --- a/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash +++ b/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash @@ -13,16 +13,7 @@ export HOMEBREW_MAKE_JOBS=${MAKE_JOBS} PROJECT=$1 # project will have the major version included (ex gazebo2) PROJECT_ARGS=${2} -# TODO(chapulina) Use gz path instead of legacy ign PROJECT_PATH=${PROJECT} -if [[ ${PROJECT/gz} != ${PROJECT} ]]; then - PROJECT_PATH="ign${PROJECT/gz}" - PROJECT_PATH="${PROJECT_PATH/[0-9]*}" -fi - -# Temporary fix for gz-sim -PROJECT=${PROJECT/gz-gazebo/gz-sim} -PROJECT_PATH=${PROJECT_PATH/ign-sim/ign-gazebo} # Check for major version number # the PROJECT_FORMULA variable is only used for dependency resolution From da4fd32dcb57639c426c7a13ba9ebb1da39bf955 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 7 Nov 2023 19:05:30 +0100 Subject: [PATCH 522/540] Do not integrate all brew pr_any job with all branches (#1061) Remove the workaround since it won't work with ign- jobs generated for Citadel in modern branches since there is no ign- formulas anymore. Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/gazebo_libs.dsl | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/jenkins-scripts/dsl/gazebo_libs.dsl b/jenkins-scripts/dsl/gazebo_libs.dsl index 897477039..3cb1322aa 100644 --- a/jenkins-scripts/dsl/gazebo_libs.dsl +++ b/jenkins-scripts/dsl/gazebo_libs.dsl @@ -254,14 +254,10 @@ ciconf_per_lib_index.each { lib_name, lib_configs -> // -------------------------------------------------------------- def gz_brew_ci_any_job_name = "${gz_job_name_prefix}-ci-pr_any-homebrew-amd64" def gz_brew_ci_any_job = job(gz_brew_ci_any_job_name) - // TODO: Instead of GITHUB_SUPPORT_ALL_BRANCHES the right configuration would be $branch_names - // leave all the branches since we only support one brew platform and it covers the - // cases of PRs targeting not stable branches, see: - // https://github.com/gazebo-tooling/release-tools/issues/1044 OSRFBrewCompilationAnyGitHub.create(gz_brew_ci_any_job, "gazebosim/${lib_name}", is_testing_enabled(lib_name, ci_config), - GITHUB_SUPPORT_ALL_BRANCHES, + branch_names, ENABLE_GITHUB_PR_INTEGRATION, are_cmake_warnings_enabled(lib_name, ci_config)) gz_brew_ci_any_job.with From d273caebb0f22fd756377ea8bc08bf781c3ca48b Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 9 Nov 2023 16:56:15 +0100 Subject: [PATCH 523/540] Change homebrew -ci- daily jobs to use gz-collections.yaml (#1058) * Change homebrew -ci- daily jobs to use gz-collections.yaml * Remove brew CI daily jobs from ignition.dsl * Remove relevant code generated in sdformat.dsl * Add a test to check for github orgs expected --------- Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/dsl_checks.bash | 13 +++++ jenkins-scripts/dsl/gazebo_libs.dsl | 67 ++++++++++++++++++------- jenkins-scripts/dsl/gz-collections.yaml | 1 + jenkins-scripts/dsl/ignition.dsl | 27 ---------- jenkins-scripts/dsl/ros_gz_bridge.dsl | 2 +- jenkins-scripts/dsl/sdformat.dsl | 43 +--------------- 6 files changed, 64 insertions(+), 89 deletions(-) diff --git a/jenkins-scripts/dsl/dsl_checks.bash b/jenkins-scripts/dsl/dsl_checks.bash index 5aa45c7f5..d08afb440 100755 --- a/jenkins-scripts/dsl/dsl_checks.bash +++ b/jenkins-scripts/dsl/dsl_checks.bash @@ -3,6 +3,7 @@ if [[ -z $(ls -- *.xml) ]]; then echo "No .xml file founds. Generate them using:" echo "https://github.com/gazebo-tooling/release-tools/blob/master/jenkins-scripts/README.md#L11" + exit 1 fi not_null=$(grep -3 'null' -- *.xml || true) @@ -28,6 +29,18 @@ if [[ -n ${abichecker_main} ]]; then exit 1 fi +# Check authorized/expected github organizations and/or repositories +non_github_orgs=$(grep -R "https?://github.com/" -- *.xml | \ + grep -E -v '/gazebosim/|/gazebo-tooling/|/gazebo-release/|/gazebo-forks/' | \ + grep -E -v '/osrf/|/ros-simulation/|/ros2?-gbp/' | \ + grep -E -v '/j-rivero/ratt/' || true) +if [[ -n ${non_github_orgs} ]]; then + echo "Unexpected github orgs in XML files:" + echo "${non_github_orgs}" + echo "either update this test or fix the DSL code" + exit 1 +fi + # Filter out the previous auto jobs filtered_dir=$(mktemp -d) cp -- *-abichecker-*.xml "${filtered_dir}" diff --git a/jenkins-scripts/dsl/gazebo_libs.dsl b/jenkins-scripts/dsl/gazebo_libs.dsl index 3cb1322aa..b801ec114 100644 --- a/jenkins-scripts/dsl/gazebo_libs.dsl +++ b/jenkins-scripts/dsl/gazebo_libs.dsl @@ -151,6 +151,28 @@ void generate_ci_job(gz_ci_job, lib_name, branch, ci_config, } } +void generate_brew_ci_job(gz_brew_ci_job, lib_name, branch, ci_config) +{ + def script_name_prefix = cleanup_library_name(lib_name) + OSRFBrewCompilation.create(gz_brew_ci_job, + is_testing_enabled(lib_name, ci_config), + are_cmake_warnings_enabled(lib_name, ci_config)) + OSRFGitHub.create(gz_brew_ci_job, + "gazebosim/${lib_name}", + branch, + lib_name) + gz_brew_ci_job.with + { + steps { + shell("""\ + #!/bin/bash -xe + + /bin/bash -xe ./scripts/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash "${lib_name}" + """.stripIndent()) + } + } +} + def ciconf_per_lib_index = [:].withDefault { [:] } def pkgconf_per_src_index = [:].withDefault { [:] } generate_ciconfigs_by_lib(gz_collections_yaml, ciconf_per_lib_index, pkgconf_per_src_index) @@ -164,15 +186,39 @@ ciconf_per_lib_index.each { lib_name, lib_configs -> def branch_names = branches_with_collections.collect { it.branch }.unique() def script_name_prefix = cleanup_library_name(lib_name) def gz_job_name_prefix = lib_name.replaceAll('-','_') + def distro = ci_config.system.version + def arch = ci_config.system.arch if (ci_config.exclude.all?.contains(lib_name)) return assert(lib_name) assert(branch_names) assert(ci_config) + // CI branch jobs (-ci-$branch-) (pulling check every 5 minutes) + branches_with_collections.each { branch_and_collection -> + def gz_ci_job + branch_name = branch_and_collection.branch + if (ci_config.system.so == 'linux') { + gz_ci_job = job("${gz_job_name_prefix}-ci-${branch_name}-${distro}-${arch}") + generate_ci_job(gz_ci_job, lib_name, branch_name, ci_config) + } else if (ci_config.system.so == 'darwin') { + gz_ci_job = job("${gz_job_name_prefix}-ci-${branch_name}-homebrew-${arch}") + generate_brew_ci_job(gz_ci_job, lib_name, branch_name, ci_config) + } + + gz_ci_job.with + { + triggers { + scm('@daily') + } + } + + logging_list['branch_ci'].add( + [collection: branch_and_collection.collection, + job_name: gz_ci_job.name]) + } // end_of_branch + if (ci_config.system.so == 'linux') { - def distro = ci_config.system.version - def arch = ci_config.system.arch def pre_setup_script = ci_config.pre_setup_script_hook?.get(lib_name)?.join('\n') def extra_cmd = pre_setup_script ?: "" @@ -233,23 +279,6 @@ ciconf_per_lib_index.each { lib_name, lib_configs -> } // end of steps } // end of with } - - // CI branch jobs (-ci-$branch-) (pulling check every 5 minutes) - branches_with_collections.each { branch_and_collection -> - branch_name = branch_and_collection.branch - def gz_ci_job = job("${gz_job_name_prefix}-ci-${branch_name}-${distro}-${arch}") - generate_ci_job(gz_ci_job, lib_name, branch_name, ci_config) - gz_ci_job.with - { - triggers { - scm('@daily') - } - } - - logging_list['branch_ci'].add( - [collection: branch_and_collection.collection, - job_name: gz_ci_job.name]) - } // end_of_branch } else if (ci_config.system.so == 'darwin') { // -------------------------------------------------------------- def gz_brew_ci_any_job_name = "${gz_job_name_prefix}-ci-pr_any-homebrew-amd64" diff --git a/jenkins-scripts/dsl/gz-collections.yaml b/jenkins-scripts/dsl/gz-collections.yaml index f65ca7473..f96a1a503 100644 --- a/jenkins-scripts/dsl/gz-collections.yaml +++ b/jenkins-scripts/dsl/gz-collections.yaml @@ -522,6 +522,7 @@ ci_configs: distribution: macOSX version: all arch: amd64 + requirements: exclude: all: - ign-citadel diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index 2816c8856..0c2296cd9 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -599,33 +599,6 @@ gz_software.each { gz_sw -> description 'Automatic generated job by DSL jenkins. Stub job for migration, not doing any check' } - // 2. main, release branches - all_branches("${software_name}").each { branch -> - def gz_brew_ci_job = job("ignition_${software_name}-ci-${branch}-homebrew-amd64") - OSRFBrewCompilation.create(gz_brew_ci_job, - enable_testing(software_name), - enable_cmake_warnings(software_name)) - OSRFGitHub.create(gz_brew_ci_job, - "gazebosim/ign-${software_name}", - "${branch}", "ign-${software_name}") - gz_brew_ci_job.with - { - triggers { - scm('@daily') - } - - steps { - shell("""\ - #!/bin/bash -xe - - software_name="gz-${software_name}" - [[ ${software_name} == 'gazebo' ]] && software_name="gz-sim" - /bin/bash -xe "./scripts/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash" "\${software_name}" - """.stripIndent()) - } - } - } - // 3. install jobs to test bottles supported_install_pkg_branches(gz_sw).each { major_version, supported_distros -> def install_default_job = job("ignition_${gz_sw}${major_version}-install_bottle-homebrew-amd64") diff --git a/jenkins-scripts/dsl/ros_gz_bridge.dsl b/jenkins-scripts/dsl/ros_gz_bridge.dsl index bc98b3a4b..dc61f62cf 100644 --- a/jenkins-scripts/dsl/ros_gz_bridge.dsl +++ b/jenkins-scripts/dsl/ros_gz_bridge.dsl @@ -57,7 +57,7 @@ bridge_packages.each { pkg -> stringParam("ARCH", "amd64", "Architecture to build packages for") stringParam('ROS_DISTRO', null,'ROS DISTRO to build pakcages for') stringParam("UPLOAD_TO_REPO", 'stable', "OSRF repo name to upload the package to") - stringParam('UPSTREAM_RELEASE_REPO', 'https://github.com/j-rivero/ros_ign-release', 'Release repository url') + stringParam('UPSTREAM_RELEASE_REPO', 'https://github.com/ros2-gbp/ros_ign-release', 'Release repository url') } // Blocks to control dependencies diff --git a/jenkins-scripts/dsl/sdformat.dsl b/jenkins-scripts/dsl/sdformat.dsl index dd67dc9f6..9ef8b57b0 100644 --- a/jenkins-scripts/dsl/sdformat.dsl +++ b/jenkins-scripts/dsl/sdformat.dsl @@ -182,47 +182,7 @@ all_debbuild_versions.each { version -> // -------------------------------------------------------------- // BREW: CI jobs -// 1. ANY job @ SCM/5min -String ci_build_any_job_name_brew = "sdformat-ci-pr_any-homebrew-amd64" -def sdformat_brew_ci_any_job = job(ci_build_any_job_name_brew) -OSRFBrewCompilationAnyGitHub.create(sdformat_brew_ci_any_job, - "gazebosim/sdformat") -sdformat_brew_ci_any_job.with -{ - steps { - shell("""\ - #!/bin/bash -xe - - /bin/bash -xe ./scripts/jenkins-scripts/sdformat-default-devel-homebrew-amd64.bash - """.stripIndent()) - } -} - -// 2. main in all branches @SCM/daily -all_versions = sdformat_supported_versions + 'main' -all_versions.each { version -> - def sdformat_brew_ci_job = job("sdformat-ci-${version}-homebrew-amd64") - OSRFBrewCompilation.create(sdformat_brew_ci_job) - OSRFGitHub.create(sdformat_brew_ci_job, "gazebosim/sdformat", - get_sdformat_branch_name(version)) - - sdformat_brew_ci_job.with - { - triggers { - scm('@daily') - } - - steps { - shell("""\ - #!/bin/bash -xe - - /bin/bash -xe ./scripts/jenkins-scripts/sdformat-default-devel-homebrew-amd64.bash - """.stripIndent()) - } - } -} - -// 3. install jobs to test bottles +// install jobs to test bottles sdformat_supported_versions.each { version -> def install_default_job = job("${version}-install_bottle-homebrew-amd64") OSRFBrewInstall.create(install_default_job) @@ -299,5 +259,4 @@ all_versions.each { version -> def sdformat_ci_main = pipelineJob("sdformat-ci-manual_any") OSRFCIWorkFlowMultiAnyGitHub.create(sdformat_ci_main, [ci_build_any_job_name_linux, - ci_build_any_job_name_brew, ci_build_any_job_name_win7]) From 76407219095f533cb97223c4068be4aea118b530 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 10 Nov 2023 18:06:38 +0100 Subject: [PATCH 524/540] Remove the ubuntu_auto -ci-pr_any- fake jobs (#1065) Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/ignition.dsl | 23 ----------------------- jenkins-scripts/dsl/sdformat.dsl | 18 +----------------- 2 files changed, 1 insertion(+), 40 deletions(-) diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index 0c2296cd9..98347699e 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -421,29 +421,6 @@ gz_software.each { gz_sw -> } // end of with } // end of abi_distro - // 1.2.1 Main PR jobs (-ci-pr_any-) (pulling check every 5 minutes) - // -------------------------------------------------------------- - software_name = gz_sw // Necessary substitution. gz_sw won't overwrite - - if (gz_sw == 'sim') - software_name = "gazebo" - - def gz_ci_job_name = "ignition_${software_name}-ci-pr_any-ubuntu_auto-${arch}" - def gz_ci_any_job = job(gz_ci_job_name) - def gz_checkout_dir = "ign-${software_name}" - GenericAnyJobGitHub.create(gz_ci_any_job, - "gazebosim/${gz_checkout_dir}", - [], - true) - gz_ci_any_job.with - { - description 'Automatic generated job by DSL jenkins. Stub job for migration, not doing any check' - } // end of ci_any_job - - // add ci-pr_any to the list for CIWorkflow - if (gz_sw != 'sim') - ci_pr_any_list[software_name] << gz_ci_job_name - all_supported_distros.each { distro -> all_branches("${gz_sw}").each { branch -> // 1. Standard CI diff --git a/jenkins-scripts/dsl/sdformat.dsl b/jenkins-scripts/dsl/sdformat.dsl index 9ef8b57b0..af288e9c7 100644 --- a/jenkins-scripts/dsl/sdformat.dsl +++ b/jenkins-scripts/dsl/sdformat.dsl @@ -17,7 +17,6 @@ def supported_arches = Globals.get_supported_arches() def experimental_arches = Globals.get_experimental_arches() String ci_distro_str = ci_distro[0] -String ci_build_any_job_name_linux = "sdformat-ci-pr_any-ubuntu_auto-amd64" // Need to be used in ci_pr String abi_job_name = '' @@ -59,20 +58,6 @@ abi_distro.each { distro -> } // end of arch } // end of distro -// MAIN CI job -// CI JOBS @ SCM/5 min -ci_distro.each { distro -> - supported_arches.each { arch -> - // -------------------------------------------------------------- - // 2. Create a fake any job for migrating - // TODO: remove the job once the migration in https://github.com/gazebo-tooling/release-tools/issues/1010 - // is done. - String sdf_repo = "gazebosim/sdformat" - def sdformat_ci_any_job = job(ci_build_any_job_name_linux) - GenericAnyJobGitHub.create(sdformat_ci_any_job, sdf_repo) - } // end of arch -} // end of distro - void generate_asan_ci_job(sdformat_ci_job, version, distro, arch) { generate_ci_job(sdformat_ci_job, version, distro, arch, @@ -258,5 +243,4 @@ all_versions.each { version -> // Create the manual all-platforms jobs def sdformat_ci_main = pipelineJob("sdformat-ci-manual_any") OSRFCIWorkFlowMultiAnyGitHub.create(sdformat_ci_main, - [ci_build_any_job_name_linux, - ci_build_any_job_name_win7]) + [ci_build_any_job_name_win7]) From f526ba54933aaf2a1c23617375e77ef4eea3552e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Crist=C3=B3bal=20Arroyo?= Date: Fri, 10 Nov 2023 12:17:23 -0500 Subject: [PATCH 525/540] Fix ign-gazebo6 branch name (#1066) --- jenkins-scripts/dsl/gz-collections.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/gz-collections.yaml b/jenkins-scripts/dsl/gz-collections.yaml index f96a1a503..19e7da36f 100644 --- a/jenkins-scripts/dsl/gz-collections.yaml +++ b/jenkins-scripts/dsl/gz-collections.yaml @@ -163,7 +163,7 @@ collections: - name: gz-sim major_version: 6 repo: - current_branch: ign-sim6 + current_branch: ign-gazebo6 - name: gz-launch major_version: 5 repo: From 6a0cf51ac22d77e0c11ac0042e84ce42dc027a7b Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Mon, 13 Nov 2023 17:24:44 +0100 Subject: [PATCH 526/540] INVALIDATE_DOCKER_CACHE in Bionic and Focal for the next two months (#1067) * INVALIDATE_DOCKER_CACHE in Bionic for the next two months * Run INVALIDATION on Focal too --------- Signed-off-by: Jose Luis Rivero --- .../docker/lib/docker_generate_dockerfile.bash | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash b/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash index 4346eb904..d5487e461 100644 --- a/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash +++ b/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash @@ -29,6 +29,15 @@ fi [[ -z ${INSTALL_C17_COMPILER} ]] && INSTALL_C17_COMPILER=false +# Bionic|Focal builds were affected by a "gpg: keyserver receive failed" in apt-key execution +# that poisoned a lot of docker cache in different builds and nodes. Force invalidation +# during a couple of month to rotate images +ref_date='2023-11-13' +if [[ "${DISTRO}" == 'bionic' || "${DISTRO}" == 'focal' ]] && \ + [[ "$(date '+%s')" -lt "$(date -d "${ref_date}+60 days" '+%s')" ]]; then + export INVALIDATE_DOCKER_CACHE=true +fi + export APT_PARAMS= GZDEV_DIR=${WORKSPACE}/gzdev From 5d88d9c76f45eac3e513742175813811205e54fc Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Mon, 13 Nov 2023 17:45:58 +0100 Subject: [PATCH 527/540] Use keys.openpgp.org keyserver on gzdev for Bionic/Focal (#1070) * Use hkps://pgp.surf.nl keyserver on gzdev Bionic/Focal Signed-off-by: Jose Luis Rivero --- .../docker/lib/docker_generate_dockerfile.bash | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash b/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash index d5487e461..e7f66c419 100644 --- a/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash +++ b/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash @@ -46,6 +46,11 @@ if python3 ${SCRIPT_DIR}/../tools/detect_ci_matching_branch.py "${ghprbSourceBra GZDEV_TRY_BRANCH=$ghprbSourceBranch fi +KEYSERVER="keyserver.ubuntu.com" +if [[ "${DISTRO}" == 'bionic' || "${DISTRO}" == 'focal' ]]; then + KEYSERVER="hkps://pgp.surf.nl" +fi + dockerfile_install_gzdev_repos() { cat >> Dockerfile << DELIM_OSRF_REPO_GIT @@ -62,18 +67,18 @@ if [[ -n ${GZDEV_PROJECT_NAME} ]]; then # debian sid docker images does not return correct name so we need to use # force-linux-distro cat >> Dockerfile << DELIM_OSRF_REPO_GZDEV -RUN ${GZDEV_DIR}/gzdev.py repository enable --project=${GZDEV_PROJECT_NAME} --force-linux-distro=${DISTRO} || ( git -C ${GZDEV_DIR} pull origin ${GZDEV_BRANCH} && \ +RUN ${GZDEV_DIR}/gzdev.py repository enable --project=${GZDEV_PROJECT_NAME} --force-linux-distro=${DISTRO} --keyserver ${KEYSERVER} || ( git -C ${GZDEV_DIR} pull origin ${GZDEV_BRANCH} && \ if [ -n $GZDEV_TRY_BRANCH ]; then git -C ${GZDEV_DIR} checkout $GZDEV_TRY_BRANCH; fi || true && \ - ${GZDEV_DIR}/gzdev.py repository enable --project=${GZDEV_PROJECT_NAME} --force-linux-distro=${DISTRO} ) + ${GZDEV_DIR}/gzdev.py repository enable --project=${GZDEV_PROJECT_NAME} --force-linux-distro=${DISTRO} --keyserver ${KEYSERVER}) DELIM_OSRF_REPO_GZDEV fi # This could duplicate repositories enabled in the step above. gzdev should warn about it without failing. for repo in ${OSRF_REPOS_TO_USE}; do cat >> Dockerfile << DELIM_OSRF_REPO -RUN ${GZDEV_DIR}/gzdev.py repository enable osrf ${repo} --force-linux-distro=${DISTRO} || ( git -C ${GZDEV_DIR} pull origin ${GZDEV_BRANCH} && \ +RUN ${GZDEV_DIR}/gzdev.py repository enable osrf ${repo} --force-linux-distro=${DISTRO} --keyserver ${KEYSERVER} || ( git -C ${GZDEV_DIR} pull origin ${GZDEV_BRANCH} && \ if [ -n $GZDEV_TRY_BRANCH ]; then git -C ${GZDEV_DIR} checkout $GZDEV_TRY_BRANCH; fi || true && \ - ${GZDEV_DIR}/gzdev.py repository enable osrf ${repo} --force-linux-distro=${DISTRO} ) + ${GZDEV_DIR}/gzdev.py repository enable osrf ${repo} --force-linux-distro=${DISTRO} --keyserver ${KEYSERVER}) DELIM_OSRF_REPO done } @@ -215,11 +220,6 @@ DELIM_DOCKER_DIRMNGR # Install necessary repositories using gzdev dockerfile_install_gzdev_repos -KEYSERVER="keyserver.ubuntu.com" -if [ "${DISTRO}" == 'bionic' ]; then - KEYSERVER="keys.openpgp.org" -fi - if ${USE_ROS_REPO}; then if ${ROS2}; then cat >> Dockerfile << DELIM_ROS_REPO From dd4deeb2cf808a0253aebb69f073468df8c68e5f Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Mon, 13 Nov 2023 18:53:16 +0100 Subject: [PATCH 528/540] Fix brew ign- formula names (#1068) * Change project-default-devel-homebrew to support explicit input values * Use PROJECT_PATH and PROJECT_FORMULA in gazebo_libs * Use common function and remove PROJECT_FORMULA --------- Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/gazebo_libs.dsl | 39 ++++++++++--------- .../project-default-devel-homebrew-amd64.bash | 5 +-- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/jenkins-scripts/dsl/gazebo_libs.dsl b/jenkins-scripts/dsl/gazebo_libs.dsl index b801ec114..93252621a 100644 --- a/jenkins-scripts/dsl/gazebo_libs.dsl +++ b/jenkins-scripts/dsl/gazebo_libs.dsl @@ -151,28 +151,37 @@ void generate_ci_job(gz_ci_job, lib_name, branch, ci_config, } } -void generate_brew_ci_job(gz_brew_ci_job, lib_name, branch, ci_config) +void add_brew_shell_build_step(gz_brew_ci_job, lib_name, ws_checkout_dir) { - def script_name_prefix = cleanup_library_name(lib_name) - OSRFBrewCompilation.create(gz_brew_ci_job, - is_testing_enabled(lib_name, ci_config), - are_cmake_warnings_enabled(lib_name, ci_config)) - OSRFGitHub.create(gz_brew_ci_job, - "gazebosim/${lib_name}", - branch, - lib_name) + // ignition formulas does not match the lib name, expand the prefix + lib_name = lib_name.replaceAll(/^ign-/, 'ignition-') gz_brew_ci_job.with { steps { shell("""\ #!/bin/bash -xe + export PROJECT_PATH="${ws_checkout_dir}" /bin/bash -xe ./scripts/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash "${lib_name}" """.stripIndent()) } } } +void generate_brew_ci_job(gz_brew_ci_job, lib_name, branch, ci_config) +{ + def script_name_prefix = cleanup_library_name(lib_name) + def ws_checkout_dir = lib_name + OSRFBrewCompilation.create(gz_brew_ci_job, + is_testing_enabled(lib_name, ci_config), + are_cmake_warnings_enabled(lib_name, ci_config)) + OSRFGitHub.create(gz_brew_ci_job, + "gazebosim/${lib_name}", + branch, + ws_checkout_dir) + add_brew_shell_build_step(gz_brew_ci_job, lib_name, ws_checkout_dir) +} + def ciconf_per_lib_index = [:].withDefault { [:] } def pkgconf_per_src_index = [:].withDefault { [:] } generate_ciconfigs_by_lib(gz_collections_yaml, ciconf_per_lib_index, pkgconf_per_src_index) @@ -283,22 +292,14 @@ ciconf_per_lib_index.each { lib_name, lib_configs -> // -------------------------------------------------------------- def gz_brew_ci_any_job_name = "${gz_job_name_prefix}-ci-pr_any-homebrew-amd64" def gz_brew_ci_any_job = job(gz_brew_ci_any_job_name) + def ws_checkout_dir = lib_name OSRFBrewCompilationAnyGitHub.create(gz_brew_ci_any_job, "gazebosim/${lib_name}", is_testing_enabled(lib_name, ci_config), branch_names, ENABLE_GITHUB_PR_INTEGRATION, are_cmake_warnings_enabled(lib_name, ci_config)) - gz_brew_ci_any_job.with - { - steps { - shell("""\ - #!/bin/bash -xe - - /bin/bash -xe "./scripts/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash" "${lib_name}" - """.stripIndent()) - } - } + add_brew_shell_build_step(gz_brew_ci_any_job, lib_name, ws_checkout_dir) } } //en of lib_configs } // end of lib diff --git a/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash b/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash index a32af3d84..f93373d83 100644 --- a/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash +++ b/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash @@ -12,9 +12,8 @@ export HOMEBREW_MAKE_JOBS=${MAKE_JOBS} # Get project name as first argument to this script PROJECT=$1 # project will have the major version included (ex gazebo2) PROJECT_ARGS=${2} - -PROJECT_PATH=${PROJECT} - +# PROJECT_PATH can be passed as env variable or assume that is the same than project name +PROJECT_PATH=${PROJECT_PATH:-$PROJECT} # Check for major version number # the PROJECT_FORMULA variable is only used for dependency resolution PROJECT_FORMULA=${PROJECT//[0-9]}$(\ From 37e22107c76c49e2575f799a035bb60243aa276e Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Mon, 13 Nov 2023 10:04:12 -0800 Subject: [PATCH 529/540] Fixes for gazebo-install-one_liner-homebrew-amd64 (#1064) * _homebrew_cleanup.bash: fix brew command * one-line gazebo.sh: fix brew doctor --------- Signed-off-by: Steve Peters Co-authored-by: Jose Luis Rivero --- jenkins-scripts/gazebo-one_liner-homebrew.bash | 6 ++++-- jenkins-scripts/lib/_homebrew_cleanup.bash | 2 +- one-line-installations/gazebo.sh | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/jenkins-scripts/gazebo-one_liner-homebrew.bash b/jenkins-scripts/gazebo-one_liner-homebrew.bash index 8716ede3b..36d2e4a26 100644 --- a/jenkins-scripts/gazebo-one_liner-homebrew.bash +++ b/jenkins-scripts/gazebo-one_liner-homebrew.bash @@ -9,6 +9,8 @@ echo '# BEGIN SECTION: cleanup brew installation' echo '# END SECTION' echo '# BEGIN SECTION: run the one-liner installation' -# TODO: change this to 'curl -ssL https://get.gazebosim.org' once that supports https -curl -ssL https://github.com/gazebo-tooling/release-tools/raw/master/one-line-installations/gazebo.sh | sh -x +curl -sSL https://get.gazebosim.org | sh -x +# to test undeployed changes to gazebo.sh, comment out the +# curl invocation and uncomment the line below +# sh -x ${SCRIPT_DIR}/../one-line-installations/gazebo.sh echo '# END SECTION' diff --git a/jenkins-scripts/lib/_homebrew_cleanup.bash b/jenkins-scripts/lib/_homebrew_cleanup.bash index 40d386055..9222a53f1 100644 --- a/jenkins-scripts/lib/_homebrew_cleanup.bash +++ b/jenkins-scripts/lib/_homebrew_cleanup.bash @@ -31,7 +31,7 @@ for t in $(HOMEBREW_NO_AUTO_UPDATE=1 \ | grep -v '^homebrew/core$'); do ${BREW_BINARY} untap $t done -brew cleanup --prune-prefix +${BREW_BINARY} cleanup --prune-prefix pushd $(${BREW_BINARY} --prefix)/Homebrew/Library 2> /dev/null git stash && git clean -d -f diff --git a/one-line-installations/gazebo.sh b/one-line-installations/gazebo.sh index ed4b6252e..3b4c2b470 100644 --- a/one-line-installations/gazebo.sh +++ b/one-line-installations/gazebo.sh @@ -287,7 +287,7 @@ do_install() { exit 1 fi - export PATH=/usr/local/bin:${PATH} + export PATH=/usr/local/bin:/usr/local/sbin:${PATH} if ! command_exists brew; then echo "Installing Homebrew:" ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" From 519b7feb9fcafda74243b3e5f85a27eb9efd802d Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 15 Nov 2023 17:27:09 +0100 Subject: [PATCH 530/540] Exclude sdformat CMake warnings on brew in gz-collections.yaml (#1072) Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/gz-collections.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/jenkins-scripts/dsl/gz-collections.yaml b/jenkins-scripts/dsl/gz-collections.yaml index 19e7da36f..ed7143d79 100644 --- a/jenkins-scripts/dsl/gz-collections.yaml +++ b/jenkins-scripts/dsl/gz-collections.yaml @@ -546,6 +546,7 @@ ci_configs: - gz-tools - gz-transport - gz-utils + - sdformat packaging_configs: - name: bionic system: From 092738becd56e261eee0083c2aaa7cef94c9011d Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 16 Nov 2023 12:24:56 +0100 Subject: [PATCH 531/540] Remove previous brew -ci-pr_any- jobs (#1073) Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/ignition.dsl | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index 98347699e..2e0d56b3f 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -560,23 +560,7 @@ all_debbuilders().each { debbuilder_name -> // 1. any job gz_software.each { gz_sw -> - software_name = gz_sw // Necessary substitution. gz_sw won't overwrite - - if (gz_sw == 'sim') - software_name = "gazebo" - - String gz_brew_ci_any_job_name = "ignition_${software_name}-ci-pr_any-homebrew-amd64" - def gz_brew_ci_any_job = job(gz_brew_ci_any_job_name) - GenericAnyJobGitHub.create(gz_brew_ci_any_job, - "gazebosim/ign-${software_name}", - GITHUB_SUPPORT_ALL_BRANCHES, - ENABLE_GITHUB_PR_INTEGRATION) - gz_brew_ci_any_job.with - { - description 'Automatic generated job by DSL jenkins. Stub job for migration, not doing any check' - } - - // 3. install jobs to test bottles + // Install jobs to test bottles supported_install_pkg_branches(gz_sw).each { major_version, supported_distros -> def install_default_job = job("ignition_${gz_sw}${major_version}-install_bottle-homebrew-amd64") OSRFBrewInstall.create(install_default_job) From 627b27d8e58d84fbcd1f136c0268ebe1400b8662 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 16 Nov 2023 16:03:29 +0100 Subject: [PATCH 532/540] Migrate Windows jobs for -ci-pr_any- and -ci-${branch}- to use gz-collections.yaml (#1071) 1. bat scripting: * Rename all .bat files to use gz (fix fuel tools expected .bat name) * Use VCS_DIRECTORY directly in DSL for checkout dir 2. gz-collections.yaml: Added win to ci_configs and use it in all collections 3. DSL generation: * gazebo_libs: add new ci-win jobs and add -ci-pr_any- win jobs * Remove -ci- daily from ignition.dsl * Remove sdformat generated Windows jobs * Replace the vcpkg tester job by using hardcode gz-sim * Testing * Change dsl_check to include all scripts * Sort the files changed in CI --------- Signed-off-by: Jose Luis Rivero --- .github/workflows/ci.yaml | 2 +- jenkins-scripts/dsl/core.dsl | 5 +- jenkins-scripts/dsl/dsl_checks.bash | 13 ++++- jenkins-scripts/dsl/gazebo_libs.dsl | 49 ++++++++++++++++- jenkins-scripts/dsl/gz-collections.yaml | 51 ++++++++++++++++++ jenkins-scripts/dsl/ignition.dsl | 52 +------------------ jenkins-scripts/dsl/ignition_collection.dsl | 3 +- jenkins-scripts/dsl/sdformat.dsl | 46 ---------------- ... gz_cmake-default-devel-windows-amd64.bat} | 2 +- ...gz_common-default-devel-windows-amd64.bat} | 2 +- ...uel_tools-default-devel-windows-amd64.bat} | 2 +- ...=> gz_gui-default-devel-windows-amd64.bat} | 2 +- ...gz_launch-default-devel-windows-amd64.bat} | 2 +- ...> gz_math-default-devel-windows-amd64.bat} | 2 +- ...> gz_msgs-default-devel-windows-amd64.bat} | 2 +- ...z_physics-default-devel-windows-amd64.bat} | 2 +- ...gz_plugin-default-devel-windows-amd64.bat} | 2 +- ...rendering-default-devel-windows-amd64.bat} | 2 +- ...> gz_rndf-default-devel-windows-amd64.bat} | 2 +- ...z_sensors-default-devel-windows-amd64.bat} | 2 +- ...=> gz_sim-default-devel-windows-amd64.bat} | 2 +- ... gz_tools-default-devel-windows-amd64.bat} | 2 +- ...transport-default-devel-windows-amd64.bat} | 2 +- ... gz_utils-default-devel-windows-amd64.bat} | 2 +- .../sdformat-default-devel-windows-amd64.bat | 2 +- 25 files changed, 133 insertions(+), 122 deletions(-) rename jenkins-scripts/{ign_cmake-default-devel-windows-amd64.bat => gz_cmake-default-devel-windows-amd64.bat} (83%) rename jenkins-scripts/{ign_common-default-devel-windows-amd64.bat => gz_common-default-devel-windows-amd64.bat} (83%) rename jenkins-scripts/{ign_fuel-tools-default-devel-windows-amd64.bat => gz_fuel_tools-default-devel-windows-amd64.bat} (74%) rename jenkins-scripts/{ign_gui-default-devel-windows-amd64.bat => gz_gui-default-devel-windows-amd64.bat} (76%) rename jenkins-scripts/{ign_launch-default-devel-windows-amd64.bat => gz_launch-default-devel-windows-amd64.bat} (75%) rename jenkins-scripts/{ign_math-default-devel-windows-amd64.bat => gz_math-default-devel-windows-amd64.bat} (76%) rename jenkins-scripts/{ign_msgs-default-devel-windows-amd64.bat => gz_msgs-default-devel-windows-amd64.bat} (75%) rename jenkins-scripts/{ign_physics-default-devel-windows-amd64.bat => gz_physics-default-devel-windows-amd64.bat} (75%) rename jenkins-scripts/{ign_plugin-default-devel-windows-amd64.bat => gz_plugin-default-devel-windows-amd64.bat} (75%) rename jenkins-scripts/{ign_rendering-default-devel-windows-amd64.bat => gz_rendering-default-devel-windows-amd64.bat} (75%) rename jenkins-scripts/{ign_rndf-default-devel-windows-amd64.bat => gz_rndf-default-devel-windows-amd64.bat} (69%) rename jenkins-scripts/{ign_sensors-default-devel-windows-amd64.bat => gz_sensors-default-devel-windows-amd64.bat} (75%) rename jenkins-scripts/{ign_gazebo-default-devel-windows-amd64.bat => gz_sim-default-devel-windows-amd64.bat} (76%) rename jenkins-scripts/{ign_tools-default-devel-windows-amd64.bat => gz_tools-default-devel-windows-amd64.bat} (92%) rename jenkins-scripts/{ign_transport-default-devel-windows-amd64.bat => gz_transport-default-devel-windows-amd64.bat} (74%) rename jenkins-scripts/{ign_utils-default-devel-windows-amd64.bat => gz_utils-default-devel-windows-amd64.bat} (79%) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 02a6c8290..524dd498f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -84,7 +84,7 @@ jobs: if: steps.dsl_check.outputs.run_job == 'true' run: | # somehow the Jenkins views changed the portlet_ id on every run. - diff -qr -I '.*dashboard_portlet_.*.*' /tmp/current_xml_configuration /tmp/pr_xml_configuration > /tmp/xml_config_files_changed.diff || true + diff -qr -I '.*dashboard_portlet_.*.*' /tmp/current_xml_configuration /tmp/pr_xml_configuration | sort > /tmp/xml_config_files_changed.diff || true diff -ur -I '.*dashboard_portlet_.*.*' /tmp/current_xml_configuration /tmp/pr_xml_configuration > /tmp/xml_config_content_changed.diff || true diff -ur /tmp/current_log_generated /tmp/pr_log_generated > /tmp/log_content_changed.diff || true - name: Archive files changes diff --git a/jenkins-scripts/dsl/core.dsl b/jenkins-scripts/dsl/core.dsl index add1a9785..dde1485cb 100644 --- a/jenkins-scripts/dsl/core.dsl +++ b/jenkins-scripts/dsl/core.dsl @@ -31,10 +31,9 @@ update_vcpkg_snapshot_job.with } } -def ignition_testing_software = 'gazebo' def testing_vcpkg_job = job("_vcpkg_testing_snapshot") OSRFWinCompilationAnyGitHub.create(testing_vcpkg_job, - "gazebosim/ign-${ignition_testing_software}", + "gazebosim/gz-sim", NO_TESTING, NO_BRANCHES, NO_GITHUB_PR_INTEGRATION) testing_vcpkg_job.with { @@ -48,7 +47,7 @@ testing_vcpkg_job.with batchFile("""\ call "%WORKSPACE%/scripts/jenkins-scripts/vcpkg-bootstrap.bat" || exit /B %errorlevel% - call "%WORKSPACE%/scripts/jenkins-scripts/ign_${ignition_testing_software}-default-devel-windows-amd64.bat" + call "%WORKSPACE%/scripts/jenkins-scripts/gz_sim-default-devel-windows-amd64.bat" """.stripIndent()) } } diff --git a/jenkins-scripts/dsl/dsl_checks.bash b/jenkins-scripts/dsl/dsl_checks.bash index d08afb440..b6774e219 100755 --- a/jenkins-scripts/dsl/dsl_checks.bash +++ b/jenkins-scripts/dsl/dsl_checks.bash @@ -14,8 +14,17 @@ if [[ -n ${not_null} ]]; then exit 1 fi -# Check for existing scripts -for f in $(grep -Eh -o './scripts/.*.bash' -- *.xml | sort | uniq); do +# Check for existing scripts. Lines: +# 1. lookg for ./scripts/. and exclude comment lnes +# 2. replaces %WORKSPACE% by . +# 3. grab only the path from ./scripts/ +# 4. remove spurious " +# 5. sor and uniq to get clean output +for f in $(grep -Eh './scripts/.*' -- *.xml | grep -v '//' | \ + sed 's/%WORKSPACE%/./g' | \ + grep -Eh -o './scripts/.*' | awk '{print $1}' | \ + sed 's/"//g' | \ + sort | uniq); do if ! test -f "${f}"; then echo "${f} script not found in the repository" fi diff --git a/jenkins-scripts/dsl/gazebo_libs.dsl b/jenkins-scripts/dsl/gazebo_libs.dsl index 93252621a..09feb2e34 100644 --- a/jenkins-scripts/dsl/gazebo_libs.dsl +++ b/jenkins-scripts/dsl/gazebo_libs.dsl @@ -182,6 +182,33 @@ void generate_brew_ci_job(gz_brew_ci_job, lib_name, branch, ci_config) add_brew_shell_build_step(gz_brew_ci_job, lib_name, ws_checkout_dir) } +void add_win_devel_bat_call(gz_win_ci_job, lib_name, ws_checkout_dir) +{ + def script_name_prefix = cleanup_library_name(lib_name) + gz_win_ci_job.with + { + steps { + batchFile("""\ + set VCS_DIRECTORY=${ws_checkout_dir} + call "./scripts/jenkins-scripts/${script_name_prefix}-default-devel-windows-amd64.bat" + """.stripIndent()) + } + } +} + +void generate_win_ci_job(gz_win_ci_job, lib_name, branch, ci_config) +{ + def ws_checkout_dir = lib_name + OSRFWinCompilation.create(gz_win_ci_job, + is_testing_enabled(lib_name, ci_config), + are_cmake_warnings_enabled(lib_name, ci_config)) + OSRFGitHub.create(gz_win_ci_job, + "gazebosim/${lib_name}", + branch, + ws_checkout_dir) + add_win_devel_bat_call(gz_win_ci_job, lib_name, ws_checkout_dir) +} + def ciconf_per_lib_index = [:].withDefault { [:] } def pkgconf_per_src_index = [:].withDefault { [:] } generate_ciconfigs_by_lib(gz_collections_yaml, ciconf_per_lib_index, pkgconf_per_src_index) @@ -197,6 +224,7 @@ ciconf_per_lib_index.each { lib_name, lib_configs -> def gz_job_name_prefix = lib_name.replaceAll('-','_') def distro = ci_config.system.version def arch = ci_config.system.arch + def ws_checkout_dir = lib_name if (ci_config.exclude.all?.contains(lib_name)) return assert(lib_name) @@ -213,6 +241,14 @@ ciconf_per_lib_index.each { lib_name, lib_configs -> } else if (ci_config.system.so == 'darwin') { gz_ci_job = job("${gz_job_name_prefix}-ci-${branch_name}-homebrew-${arch}") generate_brew_ci_job(gz_ci_job, lib_name, branch_name, ci_config) + } else if (ci_config.system.so == 'windows') { + branch_number = branch_name - lib_name + Globals.gazebodistro_branch = true + gz_ci_job = job("${gz_job_name_prefix}-${branch_number}-win") + generate_win_ci_job(gz_ci_job, lib_name, branch_name, ci_config) + Globals.gazebodistro_branch = false + } else { + assert false : "Unexpected config.system.so type: ${ci_config.system.so}" } gz_ci_job.with @@ -292,7 +328,6 @@ ciconf_per_lib_index.each { lib_name, lib_configs -> // -------------------------------------------------------------- def gz_brew_ci_any_job_name = "${gz_job_name_prefix}-ci-pr_any-homebrew-amd64" def gz_brew_ci_any_job = job(gz_brew_ci_any_job_name) - def ws_checkout_dir = lib_name OSRFBrewCompilationAnyGitHub.create(gz_brew_ci_any_job, "gazebosim/${lib_name}", is_testing_enabled(lib_name, ci_config), @@ -300,6 +335,18 @@ ciconf_per_lib_index.each { lib_name, lib_configs -> ENABLE_GITHUB_PR_INTEGRATION, are_cmake_warnings_enabled(lib_name, ci_config)) add_brew_shell_build_step(gz_brew_ci_any_job, lib_name, ws_checkout_dir) + } else if (ci_config.system.so == 'windows') { + def gz_win_ci_any_job_name = "${gz_job_name_prefix}-pr-win" + def gz_win_ci_any_job = job(gz_win_ci_any_job_name) + Globals.gazebodistro_branch = true + OSRFWinCompilationAnyGitHub.create(gz_win_ci_any_job, + "gazebosim/${lib_name}", + is_testing_enabled(lib_name, ci_config), + branch_names, + ENABLE_GITHUB_PR_INTEGRATION, + are_cmake_warnings_enabled(lib_name, ci_config)) + add_win_devel_bat_call(gz_win_ci_any_job, lib_name, ws_checkout_dir) + Globals.gazebodistro_branch = false } } //en of lib_configs } // end of lib diff --git a/jenkins-scripts/dsl/gz-collections.yaml b/jenkins-scripts/dsl/gz-collections.yaml index ed7143d79..300235db5 100644 --- a/jenkins-scripts/dsl/gz-collections.yaml +++ b/jenkins-scripts/dsl/gz-collections.yaml @@ -96,6 +96,7 @@ collections: configs: - bionic - brew + - win packaging: configs: - bionic @@ -176,6 +177,7 @@ collections: configs: - focal - brew + - win packaging: configs: - focal @@ -256,6 +258,7 @@ collections: configs: - focal - brew + - win packaging: configs: - focal @@ -336,6 +339,7 @@ collections: configs: - jammy - brew + - win packaging: configs: - jammy @@ -412,6 +416,7 @@ collections: configs: - jammy - brew + - win packaging: configs: - jammy @@ -431,6 +436,7 @@ collections: configs: - jammy - brew + - win packaging: configs: - jammy @@ -547,6 +553,51 @@ ci_configs: - gz-transport - gz-utils - sdformat + - name: win + system: + so: windows + distribution: windows + version: "10" + arch: amd64 + requirements: + exclude: + all: + - ign-citadel + - gz-fortress + - gz-garden + - gz-harmonic + - gz-ionic + - __upcoming__ + cmake_warnings_disabled: + - gz-cmake + - gz-common + - gz-fuel-tools + - gz-sim + - gz-gui + - gz-launch + - gz-math + - gz-msgs + - gz-physics + - gz-rendering + - gz-sensors + - gz-tools + - gz-transport + - gz-utils + - ign-cmake + - ign-common + - ign-fuel-tools + - ign-sim + - ign-gui + - ign-launch + - ign-math + - ign-msgs + - ign-physics + - ign-rendering + - ign-sensors + - ign-tools + - ign-transport + - ign-utils + - sdformat packaging_configs: - name: bionic system: diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index 2e0d56b3f..a0b6be5dd 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -642,61 +642,11 @@ gz_software.each { gz_sw -> enable_cmake_warnings(gz_sw)) gz_win_ci_any_job.with { - steps { - batchFile("""\ - call "./scripts/jenkins-scripts/ign_${gz_sw}-default-devel-windows-amd64.bat" - """.stripIndent()) - } + description 'Automatic generated job by DSL jenkins. Stub job for migration, not doing any check' } // add ci-pr_any to the list for CIWorkflow ci_pr_any_list[gz_sw] << gz_win_ci_any_job_name - - // 2. main, release branches - all_branches("${gz_sw}").each { branch -> - if (is_a_colcon_package(gz_sw)) { - // colcon uses long paths and windows has a hard limit of 260 chars. Keep - // names minimal - if (branch == 'main') - branch_name = "ci" - else - branch_name = branch - gz_sw - // Deal with the special case of changing gazebo name by sim - branch_name = branch_name.replace('gz-sim','gz-') - gz_win_ci_job_name = "ign_${gz_sw}-${branch_name}-win" - } else { - gz_win_ci_job_name = "ignition_${gz_sw}-ci-${branch}-windows7-amd64" - } - - def gz_win_ci_job = job(gz_win_ci_job_name) - OSRFWinCompilation.create(gz_win_ci_job, - enable_testing(gz_sw), - enable_cmake_warnings(gz_sw)) - OSRFGitHub.create(gz_win_ci_job, - "gazebosim/gz-${gz_sw}", - "${branch}") - - gz_win_ci_job.with - { - // ign-gazebo only works on Windows from ign-gazebo5 - if (branch == 'ign-gazebo3') - disabled() - - // ign-launch was not ported to windows until 5 - if (branch == 'ign-launch2') - disabled() - - triggers { - scm('@daily') - } - - steps { - batchFile("""\ - call "./scripts/jenkins-scripts/ign_${gz_sw}-default-devel-windows-amd64.bat" - """.stripIndent()) - } - } - } } // Main CI workflow diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index 7a046c26c..d5096bf3e 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -515,7 +515,8 @@ gz_collections_yaml.collections.each { collection -> // to yaml efforts. Remove from here when -install- and -win- jobs are implemented // in gazebo_libs if (ci_config.exclude.all?.contains(gz_collection_name) || - ci_config.name == "brew" ) + ci_config.system.so == 'darwin' || + ci_config.system.so == 'windows') return // INSTALL JOBS: diff --git a/jenkins-scripts/dsl/sdformat.dsl b/jenkins-scripts/dsl/sdformat.dsl index af288e9c7..01f9803b9 100644 --- a/jenkins-scripts/dsl/sdformat.dsl +++ b/jenkins-scripts/dsl/sdformat.dsl @@ -198,49 +198,3 @@ sdformat_supported_versions.each { version -> } } } - -// -------------------------------------------------------------- -// WINDOWS: CI job - -// 1. any - String ci_build_any_job_name_win7 = "sdformat-pr-win" - def sdformat_win_ci_any_job = job(ci_build_any_job_name_win7) - OSRFWinCompilationAnyGitHub.create(sdformat_win_ci_any_job, - "gazebosim/sdformat") - sdformat_win_ci_any_job.with - { - steps { - batchFile("""\ - call "./scripts/jenkins-scripts/sdformat-default-devel-windows-amd64.bat" - """.stripIndent()) - } - } - -// 2. main / @ SCM/Daily -all_versions = sdformat_supported_versions + 'main' -all_versions.each { version -> - // Use replace to get branch names to sync with ignition packages and the - // format of $branch-$version - def sdformat_win_ci_job = job("sdformat-sdf-" + version.replace('sdformat','') + "-win") - OSRFWinCompilation.create(sdformat_win_ci_job) - OSRFGitHub.create(sdformat_win_ci_job, "gazebosim/sdformat", - get_sdformat_branch_name(version)) - sdformat_win_ci_job.with - { - triggers { - scm('@daily') - } - - steps { - batchFile("""\ - set USE_GZ_ZIP=FALSE - call "./scripts/jenkins-scripts/sdformat-default-devel-windows-amd64.bat" - """.stripIndent()) - } - } -} - -// Create the manual all-platforms jobs -def sdformat_ci_main = pipelineJob("sdformat-ci-manual_any") -OSRFCIWorkFlowMultiAnyGitHub.create(sdformat_ci_main, - [ci_build_any_job_name_win7]) diff --git a/jenkins-scripts/ign_cmake-default-devel-windows-amd64.bat b/jenkins-scripts/gz_cmake-default-devel-windows-amd64.bat similarity index 83% rename from jenkins-scripts/ign_cmake-default-devel-windows-amd64.bat rename to jenkins-scripts/gz_cmake-default-devel-windows-amd64.bat index 99e0e8373..3ee9322c4 100644 --- a/jenkins-scripts/ign_cmake-default-devel-windows-amd64.bat +++ b/jenkins-scripts/gz_cmake-default-devel-windows-amd64.bat @@ -1,6 +1,6 @@ set SCRIPT_DIR=%~dp0 -set VCS_DIRECTORY=gz-cmake +if not defined VCS_DIRECTORY set VCS_DIRECTORY=gz-cmake set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true diff --git a/jenkins-scripts/ign_common-default-devel-windows-amd64.bat b/jenkins-scripts/gz_common-default-devel-windows-amd64.bat similarity index 83% rename from jenkins-scripts/ign_common-default-devel-windows-amd64.bat rename to jenkins-scripts/gz_common-default-devel-windows-amd64.bat index 7d0937264..0c7f82614 100644 --- a/jenkins-scripts/ign_common-default-devel-windows-amd64.bat +++ b/jenkins-scripts/gz_common-default-devel-windows-amd64.bat @@ -1,6 +1,6 @@ set SCRIPT_DIR=%~dp0 -set VCS_DIRECTORY=gz-common +if not defined VCS_DIRECTORY set VCS_DIRECTORY=gz-common set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true diff --git a/jenkins-scripts/ign_fuel-tools-default-devel-windows-amd64.bat b/jenkins-scripts/gz_fuel_tools-default-devel-windows-amd64.bat similarity index 74% rename from jenkins-scripts/ign_fuel-tools-default-devel-windows-amd64.bat rename to jenkins-scripts/gz_fuel_tools-default-devel-windows-amd64.bat index 590f32e41..0401f1aa3 100644 --- a/jenkins-scripts/ign_fuel-tools-default-devel-windows-amd64.bat +++ b/jenkins-scripts/gz_fuel_tools-default-devel-windows-amd64.bat @@ -1,6 +1,6 @@ set SCRIPT_DIR=%~dp0 -set VCS_DIRECTORY=gz-fuel-tools +if not defined VCS_DIRECTORY set VCS_DIRECTORY=gz-fuel-tools set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true set COLCON_PACKAGE=gz-fuel_tools diff --git a/jenkins-scripts/ign_gui-default-devel-windows-amd64.bat b/jenkins-scripts/gz_gui-default-devel-windows-amd64.bat similarity index 76% rename from jenkins-scripts/ign_gui-default-devel-windows-amd64.bat rename to jenkins-scripts/gz_gui-default-devel-windows-amd64.bat index 3590abbac..1b1a95e9e 100644 --- a/jenkins-scripts/ign_gui-default-devel-windows-amd64.bat +++ b/jenkins-scripts/gz_gui-default-devel-windows-amd64.bat @@ -1,7 +1,7 @@ @echo on set SCRIPT_DIR=%~dp0 -set VCS_DIRECTORY=gz-gui +if not defined VCS_DIRECTORY set VCS_DIRECTORY=gz-gui set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true set COLCON_PACKAGE=gz-gui diff --git a/jenkins-scripts/ign_launch-default-devel-windows-amd64.bat b/jenkins-scripts/gz_launch-default-devel-windows-amd64.bat similarity index 75% rename from jenkins-scripts/ign_launch-default-devel-windows-amd64.bat rename to jenkins-scripts/gz_launch-default-devel-windows-amd64.bat index 0a7533df7..e78bb2e91 100644 --- a/jenkins-scripts/ign_launch-default-devel-windows-amd64.bat +++ b/jenkins-scripts/gz_launch-default-devel-windows-amd64.bat @@ -1,6 +1,6 @@ set SCRIPT_DIR=%~dp0 -set VCS_DIRECTORY=gz-launch +if not defined VCS_DIRECTORY set VCS_DIRECTORY=gz-launch set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true set COLCON_PACKAGE=gz-launch diff --git a/jenkins-scripts/ign_math-default-devel-windows-amd64.bat b/jenkins-scripts/gz_math-default-devel-windows-amd64.bat similarity index 76% rename from jenkins-scripts/ign_math-default-devel-windows-amd64.bat rename to jenkins-scripts/gz_math-default-devel-windows-amd64.bat index 0c383bf0a..4341c5e50 100644 --- a/jenkins-scripts/ign_math-default-devel-windows-amd64.bat +++ b/jenkins-scripts/gz_math-default-devel-windows-amd64.bat @@ -1,6 +1,6 @@ set SCRIPT_DIR=%~dp0 -set VCS_DIRECTORY=gz-math +if not defined VCS_DIRECTORY set VCS_DIRECTORY=gz-math set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true set COLCON_PACKAGE=gz-math diff --git a/jenkins-scripts/ign_msgs-default-devel-windows-amd64.bat b/jenkins-scripts/gz_msgs-default-devel-windows-amd64.bat similarity index 75% rename from jenkins-scripts/ign_msgs-default-devel-windows-amd64.bat rename to jenkins-scripts/gz_msgs-default-devel-windows-amd64.bat index 0d93f69db..7a94c6220 100644 --- a/jenkins-scripts/ign_msgs-default-devel-windows-amd64.bat +++ b/jenkins-scripts/gz_msgs-default-devel-windows-amd64.bat @@ -1,6 +1,6 @@ set SCRIPT_DIR=%~dp0 -set VCS_DIRECTORY=gz-msgs +if not defined VCS_DIRECTORY set VCS_DIRECTORY=gz-msgs set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true diff --git a/jenkins-scripts/ign_physics-default-devel-windows-amd64.bat b/jenkins-scripts/gz_physics-default-devel-windows-amd64.bat similarity index 75% rename from jenkins-scripts/ign_physics-default-devel-windows-amd64.bat rename to jenkins-scripts/gz_physics-default-devel-windows-amd64.bat index e07c16131..a4f509554 100644 --- a/jenkins-scripts/ign_physics-default-devel-windows-amd64.bat +++ b/jenkins-scripts/gz_physics-default-devel-windows-amd64.bat @@ -1,6 +1,6 @@ set SCRIPT_DIR=%~dp0 -set VCS_DIRECTORY=gz-physics +if not defined VCS_DIRECTORY set VCS_DIRECTORY=gz-physics set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true set COLCON_PACKAGE=gz-physics diff --git a/jenkins-scripts/ign_plugin-default-devel-windows-amd64.bat b/jenkins-scripts/gz_plugin-default-devel-windows-amd64.bat similarity index 75% rename from jenkins-scripts/ign_plugin-default-devel-windows-amd64.bat rename to jenkins-scripts/gz_plugin-default-devel-windows-amd64.bat index 63ad952f0..024c8f427 100644 --- a/jenkins-scripts/ign_plugin-default-devel-windows-amd64.bat +++ b/jenkins-scripts/gz_plugin-default-devel-windows-amd64.bat @@ -1,6 +1,6 @@ set SCRIPT_DIR=%~dp0 -set VCS_DIRECTORY=gz-plugin +if not defined VCS_DIRECTORY set VCS_DIRECTORY=gz-plugin set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true diff --git a/jenkins-scripts/ign_rendering-default-devel-windows-amd64.bat b/jenkins-scripts/gz_rendering-default-devel-windows-amd64.bat similarity index 75% rename from jenkins-scripts/ign_rendering-default-devel-windows-amd64.bat rename to jenkins-scripts/gz_rendering-default-devel-windows-amd64.bat index 00cfbb20e..994cb7095 100644 --- a/jenkins-scripts/ign_rendering-default-devel-windows-amd64.bat +++ b/jenkins-scripts/gz_rendering-default-devel-windows-amd64.bat @@ -1,7 +1,7 @@ @echo on set SCRIPT_DIR=%~dp0 -set VCS_DIRECTORY=gz-rendering +if not defined VCS_DIRECTORY set VCS_DIRECTORY=gz-rendering set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true set COLCON_PACKAGE=gz-rendering diff --git a/jenkins-scripts/ign_rndf-default-devel-windows-amd64.bat b/jenkins-scripts/gz_rndf-default-devel-windows-amd64.bat similarity index 69% rename from jenkins-scripts/ign_rndf-default-devel-windows-amd64.bat rename to jenkins-scripts/gz_rndf-default-devel-windows-amd64.bat index db6644516..c5a9874ba 100644 --- a/jenkins-scripts/ign_rndf-default-devel-windows-amd64.bat +++ b/jenkins-scripts/gz_rndf-default-devel-windows-amd64.bat @@ -1,6 +1,6 @@ set SCRIPT_DIR=%~dp0 -set VCS_DIRECTORY=ign-rndf +if not defined VCS_DIRECTORY set VCS_DIRECTORY=ign-rndf set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true diff --git a/jenkins-scripts/ign_sensors-default-devel-windows-amd64.bat b/jenkins-scripts/gz_sensors-default-devel-windows-amd64.bat similarity index 75% rename from jenkins-scripts/ign_sensors-default-devel-windows-amd64.bat rename to jenkins-scripts/gz_sensors-default-devel-windows-amd64.bat index 99528bf07..74cd1b98d 100644 --- a/jenkins-scripts/ign_sensors-default-devel-windows-amd64.bat +++ b/jenkins-scripts/gz_sensors-default-devel-windows-amd64.bat @@ -1,7 +1,7 @@ @echo on set SCRIPT_DIR=%~dp0 -set VCS_DIRECTORY=gz-sensors +if not defined VCS_DIRECTORY set VCS_DIRECTORY=gz-sensors set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true set COLCON_PACKAGE=gz-sensors diff --git a/jenkins-scripts/ign_gazebo-default-devel-windows-amd64.bat b/jenkins-scripts/gz_sim-default-devel-windows-amd64.bat similarity index 76% rename from jenkins-scripts/ign_gazebo-default-devel-windows-amd64.bat rename to jenkins-scripts/gz_sim-default-devel-windows-amd64.bat index 9a803131f..15c413b51 100644 --- a/jenkins-scripts/ign_gazebo-default-devel-windows-amd64.bat +++ b/jenkins-scripts/gz_sim-default-devel-windows-amd64.bat @@ -1,7 +1,7 @@ @echo on set SCRIPT_DIR=%~dp0 -set VCS_DIRECTORY=gz-sim +if not defined VCS_DIRECTORY set VCS_DIRECTORY=gz-sim set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true set COLCON_PACKAGE=gz-sim diff --git a/jenkins-scripts/ign_tools-default-devel-windows-amd64.bat b/jenkins-scripts/gz_tools-default-devel-windows-amd64.bat similarity index 92% rename from jenkins-scripts/ign_tools-default-devel-windows-amd64.bat rename to jenkins-scripts/gz_tools-default-devel-windows-amd64.bat index 89b4adff8..aab9a6f95 100644 --- a/jenkins-scripts/ign_tools-default-devel-windows-amd64.bat +++ b/jenkins-scripts/gz_tools-default-devel-windows-amd64.bat @@ -1,6 +1,6 @@ set SCRIPT_DIR=%~dp0 -set VCS_DIRECTORY=gz-tools +if not defined VCS_DIRECTORY set VCS_DIRECTORY=gz-tools set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true diff --git a/jenkins-scripts/ign_transport-default-devel-windows-amd64.bat b/jenkins-scripts/gz_transport-default-devel-windows-amd64.bat similarity index 74% rename from jenkins-scripts/ign_transport-default-devel-windows-amd64.bat rename to jenkins-scripts/gz_transport-default-devel-windows-amd64.bat index fd6baa68d..f1946bdbe 100644 --- a/jenkins-scripts/ign_transport-default-devel-windows-amd64.bat +++ b/jenkins-scripts/gz_transport-default-devel-windows-amd64.bat @@ -2,7 +2,7 @@ set SCRIPT_DIR=%~dp0 -set VCS_DIRECTORY=gz-transport +if not defined VCS_DIRECTORY set VCS_DIRECTORY=gz-transport set PLATFORM_TO_BUILD=amd64 set IGN_CLEAN_WORKSPACE=true diff --git a/jenkins-scripts/ign_utils-default-devel-windows-amd64.bat b/jenkins-scripts/gz_utils-default-devel-windows-amd64.bat similarity index 79% rename from jenkins-scripts/ign_utils-default-devel-windows-amd64.bat rename to jenkins-scripts/gz_utils-default-devel-windows-amd64.bat index a65bdb866..a1230317e 100644 --- a/jenkins-scripts/ign_utils-default-devel-windows-amd64.bat +++ b/jenkins-scripts/gz_utils-default-devel-windows-amd64.bat @@ -3,7 +3,7 @@ set SCRIPT_DIR=%~dp0 :: ign-utils shouldn't have external dependencies -set VCS_DIRECTORY=gz-utils +if not defined VCS_DIRECTORY set VCS_DIRECTORY=gz-utils set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true set COLCON_PACKAGE=gz-utils diff --git a/jenkins-scripts/sdformat-default-devel-windows-amd64.bat b/jenkins-scripts/sdformat-default-devel-windows-amd64.bat index e302ab7b1..1f57e581e 100644 --- a/jenkins-scripts/sdformat-default-devel-windows-amd64.bat +++ b/jenkins-scripts/sdformat-default-devel-windows-amd64.bat @@ -2,7 +2,7 @@ set SCRIPT_DIR=%~dp0 -set VCS_DIRECTORY=sdformat +if not defined VCS_DIRECTORY set VCS_DIRECTORY=sdformat set PLATFORM_TO_BUILD=x86_amd64 set IGN_CLEAN_WORKSPACE=true From badf2e953288360f7c3848e267b07a6b87f7aa9e Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Fri, 17 Nov 2023 10:47:17 -0800 Subject: [PATCH 533/540] homebrew_formula_pullrequest: unique branch name (#1062) Fixes #1020. Signed-off-by: Steve Peters --- jenkins-scripts/lib/homebrew_formula_pullrequest.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/lib/homebrew_formula_pullrequest.bash b/jenkins-scripts/lib/homebrew_formula_pullrequest.bash index 0256f8eed..fa345e62a 100644 --- a/jenkins-scripts/lib/homebrew_formula_pullrequest.bash +++ b/jenkins-scripts/lib/homebrew_formula_pullrequest.bash @@ -108,6 +108,6 @@ if [ "$FORMULA_REVISION" -gt 0 ]; then fi # create branch with name and sanitized version string -PULL_REQUEST_BRANCH="${PACKAGE_ALIAS}_`echo ${VERSION_SANITIZED} | tr ' ~:^?*[' '_'`" +PULL_REQUEST_BRANCH="${PACKAGE_ALIAS}_$(echo ${VERSION_SANITIZED} | tr ' ~:^?*[' '_')_$(date +%s)" . ${SCRIPT_LIBDIR}/_homebrew_github_commit.bash From 70a19a1b6ec4baff3816476728e73b5694044316 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 21 Nov 2023 18:08:48 +0100 Subject: [PATCH 534/540] Remove old windows pr_any generation on ignition.dsl (#1081) Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/ignition.dsl | 43 -------------------------------- 1 file changed, 43 deletions(-) diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index a0b6be5dd..4a0a324c3 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -606,49 +606,6 @@ gz_software.each { gz_sw -> // -------------------------------------------------------------- // WINDOWS: CI job -// 1. any -gz_software.each { gz_sw -> - - if (gz_sw == 'sim') - return - - if (is_a_colcon_package(gz_sw)) { - // colcon uses long paths and windows has a hard limit of 260 chars. Keep - // names minimal - gz_win_ci_any_job_name = "ign_${gz_sw}-pr-win" - Globals.gazebodistro_branch = true - } else { - gz_win_ci_any_job_name = "ignition_${gz_sw}-ci-pr_any-windows7-amd64" - Globals.gazebodistro_branch = false - } - - supported_branches = [] - - // ign-gazebo only support Windows from ign-gazebo5 - if (gz_sw == 'gazebo') { - supported_branches = [ 'ign-gazebo6', 'gz-sim7', 'main' ] - } - - // ign-launch only support Windows from ign-launch5 - if (gz_sw == 'launch') - supported_branches = [ 'ign-launch5', 'gz-launch6', 'main' ] - - def gz_win_ci_any_job = job(gz_win_ci_any_job_name) - OSRFWinCompilationAnyGitHub.create(gz_win_ci_any_job, - "gazebosim/gz-${gz_sw}", - enable_testing(gz_sw), - supported_branches, - ENABLE_GITHUB_PR_INTEGRATION, - enable_cmake_warnings(gz_sw)) - gz_win_ci_any_job.with - { - description 'Automatic generated job by DSL jenkins. Stub job for migration, not doing any check' - } - - // add ci-pr_any to the list for CIWorkflow - ci_pr_any_list[gz_sw] << gz_win_ci_any_job_name -} - // Main CI workflow gz_software.each { gz_sw -> if (gz_sw == 'sim') From a41ab11bacac6c0c0017de074e3c6676ef90ddd2 Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Mon, 27 Nov 2023 08:16:34 -0800 Subject: [PATCH 535/540] Jenkins views: fix some job names (#1079) --------- Signed-off-by: Steve Peters --- jenkins-scripts/dsl/ignition_collection.dsl | 465 ++++++++++---------- 1 file changed, 232 insertions(+), 233 deletions(-) diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index d5096bf3e..660bb8b58 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -27,167 +27,166 @@ String get_debbuilder_name(parsed_yaml_lib, parsed_yaml_packaging) gz_collection_jobs = [ 'citadel' : [ - 'ign_cmake-ign-2-win', - 'ign_common-ign-3-win', - 'ign_fuel-tools-ign-4-win', - 'ign_gazebo-ign-3-win', - 'ign_gui-ign-3-win', - 'ign_math-ign-6-win', - 'ign_msgs-ign-5-win', - 'ign_physics-ign-2-win', - 'ign_plugin-ign-1-win', - 'ign_rendering-ign-3-win', - 'ign_sensors-ign-3-win', - 'ign_tools-ign-1-win', - 'ign_transport-ign-8-win', + 'ign_cmake-2-win', + 'ign_common-3-win', + 'ign_fuel-tools-4-win', + 'ign_gazebo-3-win', + 'ign_gui-3-win', + 'ign_math-6-win', + 'ign_msgs-5-win', + 'ign_physics-2-win', + 'ign_plugin-1-win', + 'ign_rendering-3-win', + 'ign_sensors-3-win', + 'ign_tools-1-win', + 'ign_transport-8-win', 'ignition_citadel-ci-main-homebrew-amd64', 'ignition_citadel-install-pkg-bionic-amd64', 'ignition_citadel-install_bottle-homebrew-amd64', - 'ignition_cmake-ci-ign-cmake2-focal-amd64', - 'ignition_cmake-ci-ign-cmake2-homebrew-amd64', + 'ign_cmake-ci-ign-cmake2-bionic-amd64', + 'ign_cmake-ci-ign-cmake2-homebrew-amd64', 'ignition_cmake2-install-pkg-focal-amd64', 'ignition_cmake2-install_bottle-homebrew-amd64', - 'ignition_common-ci-ign-common3-focal-amd64', - 'ignition_common-ci-ign-common3-homebrew-amd64', + 'ign_common-ci-ign-common3-bionic-amd64', + 'ign_common-ci-ign-common3-homebrew-amd64', 'ignition_common3-install-pkg-focal-amd64', 'ignition_common3-install_bottle-homebrew-amd64', - 'ignition_fuel-tools-ci-ign-fuel-tools4-focal-amd64', - 'ignition_fuel-tools-ci-ign-fuel-tools4-homebrew-amd64', + 'ign_fuel-tools-ci-ign-fuel-tools4-bionic-amd64', + 'ign_fuel-tools-ci-ign-fuel-tools4-homebrew-amd64', 'ignition_fuel-tools4-install-pkg-focal-amd64', 'ignition_fuel-tools4-install_bottle-homebrew-amd64', - 'ignition_gazebo-ci-ign-gazebo3-focal-amd64', - 'ignition_gazebo-ci-ign-gazebo3-homebrew-amd64', + 'ign_gazebo-ci-ign-gazebo3-bionic-amd64', + 'ign_gazebo-ci-ign-gazebo3-homebrew-amd64', 'ignition_gazebo3-install-pkg-focal-amd64', 'ignition_gazebo3-install_bottle-homebrew-amd64', - 'ignition_gui-ci-ign-gui3-focal-amd64', - 'ignition_gui-ci-ign-gui3-homebrew-amd64', + 'ign_gui-ci-ign-gui3-bionic-amd64', + 'ign_gui-ci-ign-gui3-homebrew-amd64', 'ignition_gui3-install-pkg-focal-amd64', 'ignition_gui3-install_bottle-homebrew-amd64', - 'ignition_launch-ci-ign-launch2-focal-amd64', - 'ignition_launch-ci-ign-launch2-homebrew-amd64', + 'ign_launch-ci-ign-launch2-bionic-amd64', + 'ign_launch-ci-ign-launch2-homebrew-amd64', 'ignition_launch2-install-pkg-focal-amd64', 'ignition_launch2-install_bottle-homebrew-amd64', - 'ignition_math-ci-ign-math6-focal-amd64', - 'ignition_math-ci-ign-math6-homebrew-amd64', + 'ign_math-ci-ign-math6-bionic-amd64', + 'ign_math-ci-ign-math6-homebrew-amd64', 'ignition_math6-install-pkg-focal-amd64', 'ignition_math6-install_bottle-homebrew-amd64', - 'ignition_msgs-ci-ign-msgs5-focal-amd64', - 'ignition_msgs-ci-ign-msgs5-homebrew-amd64', + 'ign_msgs-ci-ign-msgs5-bionic-amd64', + 'ign_msgs-ci-ign-msgs5-homebrew-amd64', 'ignition_msgs5-install-pkg-focal-amd64', 'ignition_msgs5-install_bottle-homebrew-amd64', - 'ignition_physics-ci-ign-physics2-focal-amd64', - 'ignition_physics-ci-ign-physics2-homebrew-amd64', + 'ign_physics-ci-ign-physics2-bionic-amd64', + 'ign_physics-ci-ign-physics2-homebrew-amd64', 'ignition_physics2-install-pkg-focal-amd64', 'ignition_physics2-install_bottle-homebrew-amd64', - 'ignition_plugin-ci-ign-plugin1-focal-amd64', - 'ignition_plugin-ci-ign-plugin1-homebrew-amd64', + 'ign_plugin-ci-ign-plugin1-bionic-amd64', + 'ign_plugin-ci-ign-plugin1-homebrew-amd64', 'ignition_plugin-install-pkg-focal-amd64', 'ignition_plugin1-install_bottle-homebrew-amd64', - 'ignition_rendering-ci-ign-rendering3-focal-amd64', - 'ignition_rendering-ci-ign-rendering3-homebrew-amd64', + 'ign_rendering-ci-ign-rendering3-bionic-amd64', + 'ign_rendering-ci-ign-rendering3-homebrew-amd64', 'ignition_rendering3-install-pkg-focal-amd64', 'ignition_rendering3-install_bottle-homebrew-amd64', - 'ignition_sensors-ci-ign-sensors3-focal-amd64', - 'ignition_sensors-ci-ign-sensors3-homebrew-amd64', + 'ign_sensors-ci-ign-sensors3-bionic-amd64', + 'ign_sensors-ci-ign-sensors3-homebrew-amd64', 'ignition_sensors3-install-pkg-focal-amd64', 'ignition_sensors3-install_bottle-homebrew-amd64', - 'ignition_tools-ci-ign-tools1-focal-amd64', - 'ignition_tools-ci-ign-tools1-homebrew-amd64', + 'ign_tools-ci-ign-tools1-bionic-amd64', + 'ign_tools-ci-ign-tools1-homebrew-amd64', 'ignition_tools-install-pkg-focal-amd64', 'ignition_tools1-install_bottle-homebrew-amd64', - 'ignition_transport-ci-ign-transport8-focal-amd64', - 'ignition_transport-ci-ign-transport8-homebrew-amd64', + 'ign_transport-ci-ign-transport8-bionic-amd64', + 'ign_transport-ci-ign-transport8-homebrew-amd64', 'ignition_transport8-install-pkg-focal-amd64', 'ignition_transport8-install_bottle-homebrew-amd64', - 'sdformat-ci-sdformat9-focal-amd64', - 'sdformat-ci-sdformat9-homebrew-amd64', - 'sdformat-ci-sdformat9-windows7-amd64', + 'sdformat-ci-sdf9-bionic-amd64', + 'sdformat-ci-sdf9-homebrew-amd64', 'sdformat-install-sdformat9_pkg-focal-amd64', - 'sdformat-sdf-9-win' + 'sdformat-sdf9-win' ], 'fortress' : [ - 'ign_cmake-ign-2-win', - 'ign_common-ign-4-win', - 'ign_fuel-tools-ign-7-win', - 'ign_gazebo-ign-6-win', - 'ign_gui-ign-6-win', - 'ign_launch-ign-5-win', - 'ign_math-ign-6-win', - 'ign_msgs-ign-8-win', - 'ign_physics-ign-5-win', - 'ign_plugin-ign-1-win', - 'ign_rendering-ign-6-win', - 'ign_sensors-ign-6-win', - 'ign_tools-ign-1-win', - 'ign_transport-ign-11-win', - 'ign_utils-ign-1-win', - 'ignition_cmake-ci-ign-cmake2-focal-amd64', - 'ignition_cmake-ci-ign-cmake2-homebrew-amd64', - 'ignition_cmake2-install-pkg-focal-amd64', + 'gz_cmake-ign-cmake2-win', + 'gz_common-ign-common4-win', + 'gz_fuel_tools-ign-fuel-tools7-win', + 'gz_sim-ign-gazebo6-win', + 'gz_gui-ign-gui6-win', + 'gz_launch-ign-launch5-win', + 'gz_math-ign-math6-win', + 'gz_msgs-ign-msgs8-win', + 'gz_physics-ign-physics5-win', + 'gz_plugin-ign-plugin1-win', + 'gz_rendering-ign-rendering6-win', + 'gz_sensors-ign-sensors6-win', + 'gz_tools-ign-tools1-win', + 'gz_transport-ign-transport11-win', + 'gz_utils-ign-utils1-win', + 'gz_cmake-ci-ign-cmake2-focal-amd64', + 'gz_cmake-ci-ign-cmake2-homebrew-amd64', + 'gz_cmake2-install-pkg-focal-amd64', 'ignition_cmake2-install_bottle-homebrew-amd64', - 'ignition_common-ci-ign-common4-focal-amd64', - 'ignition_common-ci-ign-common4-homebrew-amd64', - 'ignition_common4-install-pkg-focal-amd64', + 'gz_common-ci-ign-common4-focal-amd64', + 'gz_common-ci-ign-common4-homebrew-amd64', + 'gz_common4-install-pkg-focal-amd64', 'ignition_common4-install_bottle-homebrew-amd64', 'ignition_fortress-ci-main-homebrew-amd64', 'ignition_fortress-install-pkg-focal-amd64', 'ignition_fortress-install_bottle-homebrew-amd64', - 'ignition_fuel-tools-ci-ign-fuel-tools7-focal-amd64', - 'ignition_fuel-tools-ci-ign-fuel-tools7-homebrew-amd64', - 'ignition_fuel-tools7-install-pkg-focal-amd64', + 'gz_fuel-tools-ci-ign-fuel-tools7-focal-amd64', + 'gz_fuel-tools-ci-ign-fuel-tools7-homebrew-amd64', + 'gz_fuel-tools7-install-pkg-focal-amd64', 'ignition_fuel-tools7-install_bottle-homebrew-amd64', - 'ignition_gazebo-ci-ign-gazebo6-focal-amd64', - 'ignition_gazebo-ci-ign-gazebo6-homebrew-amd64', + 'gz_sim-ci-ign-gazebo6-focal-amd64', + 'gz_sim-ci-ign-gazebo6-homebrew-amd64', 'ignition_gazebo6-install-pkg-focal-amd64', 'ignition_gazebo6-install_bottle-homebrew-amd64', - 'ignition_gui-ci-ign-gui6-focal-amd64', - 'ignition_gui-ci-ign-gui6-homebrew-amd64', - 'ignition_gui6-install-pkg-focal-amd64', + 'gz_gui-ci-ign-gui6-focal-amd64', + 'gz_gui-ci-ign-gui6-homebrew-amd64', + 'gz_gui6-install-pkg-focal-amd64', 'ignition_gui6-install_bottle-homebrew-amd64', - 'ignition_launch-ci-ign-launch5-focal-amd64', - 'ignition_launch-ci-ign-launch5-homebrew-amd64', - 'ignition_launch5-install-pkg-focal-amd64', + 'gz_launch-ci-ign-launch5-focal-amd64', + 'gz_launch-ci-ign-launch5-homebrew-amd64', + 'gz_launch5-install-pkg-focal-amd64', 'ignition_launch5-install_bottle-homebrew-amd64', - 'ignition_math-ci-ign-math6-focal-amd64', - 'ignition_math-ci-ign-math6-homebrew-amd64', - 'ignition_math6-install-pkg-focal-amd64', + 'gz_math-ci-ign-math6-focal-amd64', + 'gz_math-ci-ign-math6-homebrew-amd64', + 'gz_math6-install-pkg-focal-amd64', 'ignition_math6-install_bottle-homebrew-amd64', - 'ignition_msgs-ci-ign-msgs8-focal-amd64', - 'ignition_msgs-ci-ign-msgs8-homebrew-amd64', - 'ignition_msgs8-install-pkg-focal-amd64', + 'gz_msgs-ci-ign-msgs8-focal-amd64', + 'gz_msgs-ci-ign-msgs8-homebrew-amd64', + 'gz_msgs8-install-pkg-focal-amd64', 'ignition_msgs8-install_bottle-homebrew-amd64', - 'ignition_physics-ci-ign-physics5-focal-amd64', - 'ignition_physics-ci-ign-physics5-homebrew-amd64', - 'ignition_physics5-install-pkg-focal-amd64', + 'gz_physics-ci-ign-physics5-focal-amd64', + 'gz_physics-ci-ign-physics5-homebrew-amd64', + 'gz_physics5-install-pkg-focal-amd64', 'ignition_physics5-install_bottle-homebrew-amd64', - 'ignition_plugin-ci-ign-plugin1-focal-amd64', - 'ignition_plugin-ci-ign-plugin1-homebrew-amd64', + 'gz_plugin-ci-ign-plugin1-focal-amd64', + 'gz_plugin-ci-ign-plugin1-homebrew-amd64', 'ignition_plugin-install-pkg-focal-amd64', 'ignition_plugin1-install_bottle-homebrew-amd64', - 'ignition_rendering-ci-ign-rendering6-focal-amd64', - 'ignition_rendering-ci-ign-rendering6-homebrew-amd64', - 'ignition_rendering6-install-pkg-focal-amd64', + 'gz_rendering-ci-ign-rendering6-focal-amd64', + 'gz_rendering-ci-ign-rendering6-homebrew-amd64', + 'gz_rendering6-install-pkg-focal-amd64', 'ignition_rendering6-install_bottle-homebrew-amd64', - 'ignition_sensors-ci-ign-sensors6-focal-amd64', - 'ignition_sensors-ci-ign-sensors6-homebrew-amd64', - 'ignition_sensors6-install-pkg-focal-amd64', + 'gz_sensors-ci-ign-sensors6-focal-amd64', + 'gz_sensors-ci-ign-sensors6-homebrew-amd64', + 'gz_sensors6-install-pkg-focal-amd64', 'ignition_sensors6-install_bottle-homebrew-amd64', - 'ignition_tools-ci-ign-tools1-focal-amd64', - 'ignition_tools-ci-ign-tools1-homebrew-amd64', + 'gz_tools-ci-ign-tools1-focal-amd64', + 'gz_tools-ci-ign-tools1-homebrew-amd64', 'ignition_tools-install-pkg-focal-amd64', 'ignition_tools1-install_bottle-homebrew-amd64', - 'ignition_transport-ci-ign-transport11-focal-amd64', - 'ignition_transport-ci-ign-transport11-homebrew-amd64', - 'ignition_transport11-install-pkg-focal-amd64', + 'gz_transport-ci-ign-transport11-focal-amd64', + 'gz_transport-ci-ign-transport11-homebrew-amd64', + 'gz_transport11-install-pkg-focal-amd64', 'ignition_transport11-install_bottle-homebrew-amd64', - 'ignition_utils-ci-ign-utils1-focal-amd64', - 'ignition_utils-ci-ign-utils1-homebrew-amd64', + 'gz_utils-ci-ign-utils1-focal-amd64', + 'gz_utils-ci-ign-utils1-homebrew-amd64', 'ignition_utils-install-pkg-focal-amd64', 'ignition_utils1-install_bottle-homebrew-amd64', - 'sdformat-ci-sdformat12-focal-amd64', - 'sdformat-ci-sdformat12-homebrew-amd64', + 'sdformat-ci-sdf12-focal-amd64', + 'sdformat-ci-sdf12-homebrew-amd64', 'sdformat-install-sdformat12_pkg-focal-amd64', - 'sdformat-sdf-12-win.xml' + 'sdformat-sdf12-win' ], 'garden' : [ 'gz_cmake3-install-pkg-focal-amd64', @@ -207,89 +206,89 @@ gz_collection_jobs = 'gz_transport12-install-pkg-focal-amd64', 'gz_utils2-install-pkg-focal-amd64', 'sdformat-install-sdformat13_pkg-focal-amd64', - 'ign_cmake-gz-3-win', - 'ign_common-gz-5-win', - 'ign_fuel-tools-gz-8-win', - 'ign_gazebo-gz-7-win', - 'ign_gui-gz-7-win', - 'ign_launch-gz-6-win', - 'ign_math-gz-7-win', - 'ign_msgs-gz-9-win', - 'ign_physics-gz-6-win', - 'ign_plugin-gz-2-win', - 'ign_rendering-gz-7-win', - 'ign_sensors-gz-7-win', - 'ign_tools-gz-2-win', - 'ign_transport-gz-12-win', - 'ign_utils-gz-2-win', - 'sdformat-sdf-13-win', - 'ignition_cmake-ci-gz-cmake3-focal-amd64', - 'ignition_cmake-ci-gz-cmake3-homebrew-amd64', + 'gz_cmake-3-win', + 'gz_common-5-win', + 'gz_fuel-tools-8-win', + 'gz_sim-7-win', + 'gz_gui-7-win', + 'gz_launch-6-win', + 'gz_math-7-win', + 'gz_msgs-9-win', + 'gz_physics-6-win', + 'gz_plugin-2-win', + 'gz_rendering-7-win', + 'gz_sensors-7-win', + 'gz_tools-2-win', + 'gz_transport-12-win', + 'gz_utils-2-win', + 'sdformat-sdf13-win', + 'gz_cmake-ci-gz-cmake3-focal-amd64', + 'gz_cmake-ci-gz-cmake3-homebrew-amd64', 'ignition_cmake3-install_bottle-homebrew-amd64', - 'ignition_common-ci-gz-common5-focal-amd64', - 'ignition_common-ci-gz-common5-homebrew-amd64', + 'gz_common-ci-gz-common5-focal-amd64', + 'gz_common-ci-gz-common5-homebrew-amd64', 'ignition_common5-install_bottle-homebrew-amd64', - 'ignition_fuel-tools-ci-gz-fuel-tools8-focal-amd64', - 'ignition_fuel-tools-ci-gz-fuel-tools8-homebrew-amd64', + 'gz_fuel-tools-ci-gz-fuel-tools8-focal-amd64', + 'gz_fuel-tools-ci-gz-fuel-tools8-homebrew-amd64', 'ignition_fuel-tools8-install_bottle-homebrew-amd64', 'ignition_garden-ci-main-homebrew-amd64', 'ignition_garden-install-pkg-focal-amd64', 'ignition_garden-install_bottle-homebrew-amd64', - 'ignition_gazebo-ci-gz-sim7-focal-amd64', - 'ignition_gazebo-ci-gz-sim7-homebrew-amd64', + 'gz_sim-ci-gz-sim7-focal-amd64', + 'gz_sim-ci-gz-sim7-homebrew-amd64', 'ignition_sim7-install_bottle-homebrew-amd64', - 'ignition_gui-ci-gz-gui7-focal-amd64', - 'ignition_gui-ci-gz-gui7-homebrew-amd64', + 'gz_gui-ci-gz-gui7-focal-amd64', + 'gz_gui-ci-gz-gui7-homebrew-amd64', 'ignition_gui6-install_bottle-homebrew-amd64', - 'ignition_launch-ci-gz-launch6-focal-amd64', - 'ignition_launch-ci-gz-launch6-homebrew-amd64', + 'gz_launch-ci-gz-launch6-focal-amd64', + 'gz_launch-ci-gz-launch6-homebrew-amd64', 'ignition_launch6-install_bottle-homebrew-amd64', - 'ignition_math-ci-gz-math7-focal-amd64', - 'ignition_math-ci-gz-math7-homebrew-amd64', + 'gz_math-ci-gz-math7-focal-amd64', + 'gz_math-ci-gz-math7-homebrew-amd64', 'ignition_math7-install_bottle-homebrew-amd64', - 'ignition_msgs-ci-gz-msgs9-focal-amd64', - 'ignition_msgs-ci-gz-msgs9-homebrew-amd64', + 'gz_msgs-ci-gz-msgs9-focal-amd64', + 'gz_msgs-ci-gz-msgs9-homebrew-amd64', 'ignition_msgs9-install_bottle-homebrew-amd64', - 'ignition_physics-ci-gz-physics6-focal-amd64', - 'ignition_physics-ci-gz-physics6-homebrew-amd64', + 'gz_physics-ci-gz-physics6-focal-amd64', + 'gz_physics-ci-gz-physics6-homebrew-amd64', 'ignition_physics6-install_bottle-homebrew-amd64', - 'ignition_plugin-ci-gz-plugin2-focal-amd64', - 'ignition_plugin-ci-gz-plugin2-homebrew-amd64', + 'gz_plugin-ci-gz-plugin2-focal-amd64', + 'gz_plugin-ci-gz-plugin2-homebrew-amd64', 'ignition_plugin2-install_bottle-homebrew-amd64', - 'ignition_rendering-ci-gz-rendering7-focal-amd64', - 'ignition_rendering-ci-gz-rendering7-homebrew-amd64', + 'gz_rendering-ci-gz-rendering7-focal-amd64', + 'gz_rendering-ci-gz-rendering7-homebrew-amd64', 'ignition_rendering7-install_bottle-homebrew-amd64', - 'ignition_sensors-ci-gz-sensors7-focal-amd64', - 'ignition_sensors-ci-gz-sensors7-homebrew-amd64', + 'gz_sensors-ci-gz-sensors7-focal-amd64', + 'gz_sensors-ci-gz-sensors7-homebrew-amd64', 'ignition_sensors7-install_bottle-homebrew-amd64', - 'ignition_tools-ci-gz-tools2-focal-amd64', - 'ignition_tools-ci-gz-tools2-homebrew-amd64', + 'gz_tools-ci-gz-tools2-focal-amd64', + 'gz_tools-ci-gz-tools2-homebrew-amd64', 'ignition_tools2-install_bottle-homebrew-amd64', - 'ignition_transport-ci-gz-transport12-focal-amd64', - 'ignition_transport-ci-gz-transport12-homebrew-amd64', + 'gz_transport-ci-gz-transport12-focal-amd64', + 'gz_transport-ci-gz-transport12-homebrew-amd64', 'ignition_transport12-install_bottle-homebrew-amd64', - 'ignition_utils-ci-gz-utils2-focal-amd64', - 'ignition_utils-ci-gz-utils2-homebrew-amd64', + 'gz_utils-ci-gz-utils2-focal-amd64', + 'gz_utils-ci-gz-utils2-homebrew-amd64', 'ignition_utils2-install_bottle-homebrew-amd64', - 'sdformat-ci-sdformat13-focal-amd64', - 'sdformat-ci-sdformat13-homebrew-amd64' + 'sdformat-ci-sdf13-focal-amd64', + 'sdformat-ci-sdf13-homebrew-amd64' ], 'harmonic' : [ - 'ign_cmake-gz-3-win', - 'ign_common-gz-5-win', - 'ign_fuel-tools-gz-9-win', - 'ign_gazebo-gz-8-win', - 'ign_gui-gz-8-win', - 'ign_launch-gz-7-win', - 'ign_math-gz-7-win', - 'ign_msgs-gz-10-win', - 'ign_physics-gz-7-win', - 'ign_plugin-gz-2-win', - 'ign_rendering-gz-8-win', - 'ign_sensors-gz-8-win', - 'ign_tools-gz-2-win', - 'ign_transport-gz-13-win', - 'ign_utils-gz-2-win', + 'gz_cmake-3-win', + 'gz_common-5-win', + 'gz_fuel-tools-9-win', + 'gz_sim-8-win', + 'gz_gui-8-win', + 'gz_launch-7-win', + 'gz_math-7-win', + 'gz_msgs-10-win', + 'gz_physics-7-win', + 'gz_plugin-2-win', + 'gz_rendering-8-win', + 'gz_sensors-8-win', + 'gz_tools-2-win', + 'gz_transport-13-win', + 'gz_utils-2-win', 'gz_cmake-ci-gz-cmake3-jammy-amd64', 'gz_common-ci-gz-common5-jammy-amd64', 'gz_fuel_tools-ci-gz-fuel-tools9-jammy-amd64', @@ -305,93 +304,93 @@ gz_collection_jobs = 'gz_tools-ci-gz-tools2-jammy-amd64', 'gz_transport-ci-gz-transport13-jammy-amd64', 'gz_utils-ci-gz-utils2-jammy-amd64', - 'ignition_cmake-ci-gz-cmake3-homebrew-amd64', + 'gz_cmake-ci-gz-cmake3-homebrew-amd64', 'ignition_cmake3-install_bottle-homebrew-amd64', - 'ignition_common-ci-gz-common5-homebrew-amd64', + 'gz_common-ci-gz-common5-homebrew-amd64', 'ignition_common5-install_bottle-homebrew-amd64', - 'ignition_fuel-tools-ci-gz-fuel-tools9-homebrew-amd64', + 'gz_fuel-tools-ci-gz-fuel-tools9-homebrew-amd64', 'ignition_fuel-tools9-install_bottle-homebrew-amd64', 'ignition_harmonic-ci-main-homebrew-amd64', 'ignition_harmonic-install_bottle-homebrew-amd64', - 'ignition_gazebo-ci-gz-sim8-homebrew-amd64', + 'gz_sim-ci-gz-sim8-homebrew-amd64', 'ignition_sim8-install_bottle-homebrew-amd64', - 'ignition_gui-ci-gz-gui8-homebrew-amd64', + 'gz_gui-ci-gz-gui8-homebrew-amd64', 'ignition_gui8-install_bottle-homebrew-amd64', - 'ignition_launch-gz-launch7-homebrew-amd64', + 'gz_launch-gz-launch7-homebrew-amd64', 'ignition_launch7-install_bottle-homebrew-amd64', - 'ignition_math-ci-gz-math7-homebrew-amd64', + 'gz_math-ci-gz-math7-homebrew-amd64', 'ignition_math7-install_bottle-homebrew-amd64', - 'ignition_msgs-ci-gz-msgs10-homebrew-amd64', + 'gz_msgs-ci-gz-msgs10-homebrew-amd64', 'ignition_msgs10-install_bottle-homebrew-amd64', - 'ignition_physics-ci-gz-physics7-homebrew-amd64', + 'gz_physics-ci-gz-physics7-homebrew-amd64', 'ignition_physics7-install_bottle-homebrew-amd64', - 'ignition_plugin-ci-gz-plugin2-homebrew-amd64', + 'gz_plugin-ci-gz-plugin2-homebrew-amd64', 'ignition_plugin2-install_bottle-homebrew-amd64', - 'ignition_rendering-ci-gz-rendering8-homebrew-amd64', + 'gz_rendering-ci-gz-rendering8-homebrew-amd64', 'ignition_rendering8-install_bottle-homebrew-amd64', - 'ignition_sensors-ci-gz-sensors8-homebrew-amd64', + 'gz_sensors-ci-gz-sensors8-homebrew-amd64', 'ignition_sensors8-install_bottle-homebrew-amd64', - 'ignition_tools-ci-gz-tools2-homebrew-amd64', + 'gz_tools-ci-gz-tools2-homebrew-amd64', 'ignition_tools2-install_bottle-homebrew-amd64', - 'ignition_transport-ci-gz-transport13-homebrew-amd64', + 'gz_transport-ci-gz-transport13-homebrew-amd64', 'ignition_transport13-install_bottle-homebrew-amd64', - 'ignition_utils-ci-gz-utils2-homebrew-amd64', + 'gz_utils-ci-gz-utils2-homebrew-amd64', 'ignition_utils2-install_bottle-homebrew-amd64', - 'sdformat-ci-sdformat14-jammy-amd64', - 'sdformat-ci-sdformat14-homebrew-amd64', + 'sdformat-ci-sdf14-jammy-amd64', + 'sdformat-ci-sdf14-homebrew-amd64', 'sdformat14-install_bottle-homebrew-amd64', - 'sdformat-sdf-14-win' + 'sdformat-sdf14-win' ], 'ionic' : [ - 'ign_cmake-gz-4-win', - 'ign_common-gz-6-win', - 'ign_fuel-tools-gz-10-win', - 'ign_gazebo-gz-9-win', - 'ign_gui-gz-9-win', - 'ign_launch-gz-8-win', - 'ign_math-gz-8-win', - 'ign_msgs-gz-11-win', - 'ign_physics-gz-8-win', - 'ign_plugin-gz-3-win', - 'ign_rendering-gz-9-win', - 'ign_sensors-gz-9-win', - 'ign_tools-gz-2-win', - 'ign_transport-gz-14-win', - 'ign_utils-gz-3-win', - 'gz_cmake-ci-gz-cmake4-jammy-amd64', - 'gz_common-ci-gz-common6-jammy-amd64', - 'gz_fuel_tools-ci-gz-fuel-tools10-jammy-amd64', - 'gz_gui-ci-gz-gui9-jammy-amd64', - 'gz_launch-ci-gz-launch8-jammy-amd64', - 'gz_math-ci-gz-math8-jammy-amd64', - 'gz_msgs-ci-gz-msgs11-jammy-amd64', - 'gz_physics-ci-gz-physics8-jammy-amd64', - 'gz_plugin-ci-gz-plugin3-jammy-amd64', - 'gz_rendering-ci-gz-rendering9-jammy-amd64', - 'gz_sensors-ci-gz-sensors9-jammy-amd64', - 'gz_sim-ci-gz-sim9-jammy-amd64', + 'gz_cmake-main-win', + 'gz_common-main-win', + 'gz_fuel-tools-main-win', + 'gz_gazebo-main-win', + 'gz_gui-main-win', + 'gz_launch-main-win', + 'gz_math-main-win', + 'gz_msgs-main-win', + 'gz_physics-main-win', + 'gz_plugin-main-win', + 'gz_rendering-main-win', + 'gz_sensors-main-win', + 'gz_tools-2-win', + 'gz_transport-main-win', + 'gz_utils-main-win', + 'gz_cmake-ci-main-jammy-amd64', + 'gz_common-ci-main-jammy-amd64', + 'gz_fuel_tools-ci-main-jammy-amd64', + 'gz_gui-ci-main-jammy-amd64', + 'gz_launch-ci-main-jammy-amd64', + 'gz_math-ci-main-jammy-amd64', + 'gz_msgs-ci-main-jammy-amd64', + 'gz_physics-ci-main-jammy-amd64', + 'gz_plugin-ci-main-jammy-amd64', + 'gz_rendering-ci-main-jammy-amd64', + 'gz_sensors-ci-main-jammy-amd64', + 'gz_sim-ci-main-jammy-amd64', 'gz_tools-ci-gz-tools2-jammy-amd64', - 'gz_transport-ci-gz-transport14-jammy-amd64', - 'gz_utils-ci-gz-utils3-jammy-amd64', - 'ignition_cmake-ci-gz-cmake4-homebrew-amd64', - 'ignition_common-ci-gz-common6-homebrew-amd64', - 'ignition_fuel-tools-ci-gz-fuel-tools10-homebrew-amd64', - 'ignition_ionic-ci-main-homebrew-amd64', - 'ignition_gazebo-ci-gz-sim9-homebrew-amd64', - 'ignition_gui-ci-gz-gui9-homebrew-amd64', - 'ignition_launch-gz-launch8-homebrew-amd64', - 'ignition_math-ci-gz-math8-homebrew-amd64', - 'ignition_msgs-ci-gz-msgs11-homebrew-amd64', - 'ignition_physics-ci-gz-physics8-homebrew-amd64', - 'ignition_plugin-ci-gz-plugin3-homebrew-amd64', - 'ignition_rendering-ci-gz-rendering9-homebrew-amd64', - 'ignition_sensors-ci-gz-sensors9-homebrew-amd64', - 'ignition_tools-ci-gz-tools2-homebrew-amd64', - 'ignition_transport-ci-gz-transport14-homebrew-amd64', - 'ignition_utils-ci-gz-utils3-homebrew-amd64', - 'sdformat-ci-sdformat15-jammy-amd64', - 'sdformat-ci-sdformat15-homebrew-amd64', - 'sdformat-sdf-15-win' + 'gz_transport-ci-main-jammy-amd64', + 'gz_utils-ci-main-jammy-amd64', + 'gz_cmake-ci-main-homebrew-amd64', + 'gz_common-ci-main-homebrew-amd64', + 'gz_fuel-tools-ci-main-homebrew-amd64', + 'gz_ionic-ci-main-homebrew-amd64', + 'gz_sim-ci-main-homebrew-amd64', + 'gz_gui-ci-main-homebrew-amd64', + 'gz_launch-ci-main-homebrew-amd64', + 'gz_math-ci-main-homebrew-amd64', + 'gz_msgs-ci-main-homebrew-amd64', + 'gz_physics-ci-main-homebrew-amd64', + 'gz_plugin-ci-main-homebrew-amd64', + 'gz_rendering-ci-main-homebrew-amd64', + 'gz_sensors-ci-main-homebrew-amd64', + 'gz_tools-ci-gz-tools2-homebrew-amd64', + 'gz_transport-ci-main-homebrew-amd64', + 'gz_utils-ci-main-homebrew-amd64', + 'sdformat-ci-main-jammy-amd64', + 'sdformat-ci-main-homebrew-amd64', + 'sdformat-main-win' ], ] From 22a5fd68aa2c55b27f41f52d4b28e0dc18462308 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 28 Nov 2023 16:24:28 +0100 Subject: [PATCH 536/540] Increase priority for all PR jobs (#1077) Overwrite it to 200 going after release builders but before other CI runs. Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/_configs_/GenericAnyJobGitHub.groovy | 9 +++++++++ jenkins-scripts/dsl/_configs_/OSRFLinuxABIGitHub.groovy | 4 ++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/jenkins-scripts/dsl/_configs_/GenericAnyJobGitHub.groovy b/jenkins-scripts/dsl/_configs_/GenericAnyJobGitHub.groovy index e6d96d143..2939a3b92 100644 --- a/jenkins-scripts/dsl/_configs_/GenericAnyJobGitHub.groovy +++ b/jenkins-scripts/dsl/_configs_/GenericAnyJobGitHub.groovy @@ -4,6 +4,7 @@ import javaposse.jobdsl.dsl.Job /* Implements: + - priority 200 - parameters: SRC_REPO, SRC_BRANCH, JOB_DESCRIPTION - job.Description - scm check with SRC_REPO + SRC_BRANCH @@ -33,6 +34,14 @@ class GenericAnyJobGitHub job.with { + // Overrwrite the priority set by other classes + configure { project -> + project / 'properties' / 'jenkins.advancedqueue.priority.strategy.PriorityJobProperty' { + 'useJobPriority'(true) + 'priority'(200) + } + } + parameters { stringParam('sha1', 'main', 'commit or refname to build. To manually use a branch: origin/$branch_name') diff --git a/jenkins-scripts/dsl/_configs_/OSRFLinuxABIGitHub.groovy b/jenkins-scripts/dsl/_configs_/OSRFLinuxABIGitHub.groovy index d21fd655e..b0bba9ed6 100644 --- a/jenkins-scripts/dsl/_configs_/OSRFLinuxABIGitHub.groovy +++ b/jenkins-scripts/dsl/_configs_/OSRFLinuxABIGitHub.groovy @@ -6,7 +6,7 @@ import javaposse.jobdsl.dsl.Job -> OSRFLinuxBase Implements: - - priority 300 + - priority 200 - logrotator - concurrent builds - parameter: DEST_BRANCH, SRC_BRANCH @@ -35,7 +35,7 @@ class OSRFLinuxABIGitHub job.with { properties { - priority 300 + priority 200 } logRotator { From 460240b58cc3963a951fb9e490f8e49567c593c0 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 28 Nov 2023 17:40:28 +0100 Subject: [PATCH 537/540] Change output in release.py to be more user friendly (#1074) * Introduce print_dbg and use it to hide main debug command * Improve output and added a header and downloading * Use debug info in check_releasepy * Display help in source calls for the user * Use search params for brew and builders --------- Signed-off-by: Jose Luis Rivero --- check_releasepy.bash | 1 + release.py | 84 +++++++++++++++++++++++++++++++++++--------- 2 files changed, 68 insertions(+), 17 deletions(-) diff --git a/check_releasepy.bash b/check_releasepy.bash index 5e71fb168..a7c07a222 100755 --- a/check_releasepy.bash +++ b/check_releasepy.bash @@ -3,6 +3,7 @@ test_dir=$(mktemp -d) mkdir -p ${test_dir}/{focal,jammy,ubuntu}/debian export _RELEASEPY_TEST_RELEASE_REPO=${test_dir} +export _RELEASEPY_DEBUG=1 exec_releasepy_test() { diff --git a/release.py b/release.py index db1bd2c35..56ca9c5b5 100755 --- a/release.py +++ b/release.py @@ -73,6 +73,11 @@ def print_success(msg): print(" + OK " + msg) +def print_only_dbg(msg): + if '_RELEASEPY_DEBUG' in os.environ: + print(msg) + + # Remove the last character if it is a number. # That should leave just the package name instead of packageVersion # I.E gazebo5 -> gazebo @@ -370,19 +375,19 @@ def discover_distros(repo_dir): arches_supported = [x for x in SUPPORTED_ARCHS if x not in excluded_arches] distro_arch_list[d] = arches_supported - print('Distributions in release-repo:') - for k in distro_arch_list: - print("- " + k + " (" + ', '.join(distro_arch_list[k]) + ")") + print('Linux distributions in the -release repository:') + for distro in distro_arch_list: + print(f" + {distro} {*distro_arch_list[distro],}") return distro_arch_list def check_call(cmd, ignore_dry_run=False): if DRY_RUN and not ignore_dry_run: - print('Dry-run running:\n %s\n' % (' '.join(cmd))) + print_only_dbg('Dry-run running:\n %s\n' % (' '.join(cmd))) return b'', b'' else: - print('Running:\n %s' % (' '.join(cmd))) + print_only_dbg('Running:\n %s' % (' '.join(cmd))) po = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = po.communicate() if po.returncode != 0: @@ -433,7 +438,7 @@ def generate_source_repository_uri(args): out, err = check_call(['git', 'ls-remote', '--get-url', 'origin'], IGNORE_DRY_RUN) if err: - print(f"An error happened running git ls-remote: ${err}") + print(f"An error happened running git ls-remote: {err}") sys.exit(1) git_remote = out.decode().split('\n')[0] @@ -469,16 +474,46 @@ def generate_source_params(args): return params -def call_jenkins_build(job_name, params, output_string): +def call_jenkins_build(job_name, params, output_string, + search_description_help): + # Only to help user feedback this block + help_url = f'{JENKINS_URL}/job/{job_name}' + if search_description_help: + search_param = urllib.parse.urlencode( + {'search': search_description_help}) + help_url += f'?{search_param}' + print(f" + Releasing {output_string} in {help_url}") + # Real action happen here params_query = urllib.parse.urlencode(params) url = '%s/job/%s/buildWithParameters?%s' % (JENKINS_URL, job_name, params_query) - print(f"- {output_string}: {url}") + print_only_dbg(f" -- {output_string}: {url}") if not DRY_RUN: urllib.request.urlopen(url) +def display_help_job_chain_for_source_calls(args): + # Encode the different ways using in the job descriptions to filter builds + # - "package version" in repository_uploader_packages + # - "packages/version-rev" in _releasepy + url_search_params = urllib.parse.urlencode( + {'search': + f'{args.package_alias} {args.version}'}) + pkgs_upload_check_url = \ + f'{JENKINS_URL}/job/repository_uploader_packages/?{url_search_params}' + rel_search_params = urllib.parse.urlencode( + {'search': + f'{args.package_alias}/{args.version}-{args.release_version}'}) + releasepy_check_url = \ + f'{JENKINS_URL}/job/_releasepy/?{rel_search_params}' + print('\tINFO: After the source job finished, the release process will trigger:\n' + '\t * Source upload:' + f'{pkgs_upload_check_url}\n' + '\t * Builders using release.py --source-tarball-uri:' + f'{releasepy_check_url}') + + def go(argv): args = parse_args(argv) @@ -486,6 +521,7 @@ def go(argv): if not args.release_version: args.release_version = 1 + print(f"Downloading releasing info for {args.package}") # Sanity checks and dicover supported distributions before proceed. repo_dir, args.release_repo_branch = download_release_repository(args.package, args.release_repo_branch) # The supported distros are the ones in the top level of -release repo @@ -507,13 +543,17 @@ def go(argv): if args.extra_repo: params['OSRF_REPOS_TO_USE'] += " " + args.extra_repo - + if args.dry_run: + print("Simulation of jobs to be called if not dry-run:") + else: + print("Triggering release jobs:") # a) Mode nightly or builders: if NIGHTLY or args.source_tarball_uri: # RELEASING FOR BREW if not NIGHTLY and not args.bump_rev_linux_only: call_jenkins_build(GENERIC_BREW_PULLREQUEST_JOB, - params, 'Brew') + params, 'Brew', + f'{args.package_alias}-{args.version}') # RELEASING FOR LINUX for l in LINUX_DISTROS: if (l == 'ubuntu'): @@ -553,15 +593,21 @@ def go(argv): # control nightly generation using a single machine to process # all distribution builds to avoid race conditions. Note: this - # assumes that large-memory nodes are beind used for nightly + # assumes that large-memory nodes are being used for nightly # tags. # https://github.com/gazebo-tooling/release-tools/issues/644 if (NIGHTLY): assert a == 'amd64', f'Nightly tag assumed amd64 but arch is {a}' linux_platform_params['JENKINS_NODE_TAG'] = 'linux-nightly-' + d - + # TODO: last parameter of providing help for -debbuilders + # does not currently work. Somehow the string composed by + # "-()" do not work even in the web UI directly. Real + # string should be: + # f"{args.version}-{args.release_version}({l}/{d}::{a})") call_jenkins_build(f"{args.package_alias}-debbuilder", - linux_platform_params, f"{l} {d}/{a}") + linux_platform_params, + f"{l} {d}/{a}", + f"{args.version}-{args.release_version}") else: # b) Mode generate source # Choose platform to run gz-source on. It will need to install gz-cmake @@ -579,14 +625,18 @@ def go(argv): # Tag should not go before any method or step that can fail and just # before the calls to the servers. if not args.source_repo_ref: - print('INFO: no --source-repo-existing-ref used, tag the local' - 'repository as the reference for the source code of the' - 'release') + print(' * INFO: no --source-repo-existing-ref used, tag the local' + ' repository as the reference for the source code of the' + ' release') params['SOURCE_REPO_REF'] = tag_repo(args) \ if not args.source_repo_ref else args.source_repo_ref - call_jenkins_build(f"{args.package_alias}-source", params, 'Source') + call_jenkins_build(f"{args.package_alias}-source", + params, + 'Source', + args.version) + display_help_job_chain_for_source_calls(args) if __name__ == '__main__': From de8ac8a786b69fca03fc7a1cae5407fa85731eba Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 29 Nov 2023 19:05:22 +0100 Subject: [PATCH 538/540] Split CI: releasepy checks (#1085) Signed-off-by: Jose Luis Rivero --- .github/workflows/ci-releasepy.yaml | 18 ++++++++++++++++++ .github/workflows/ci.yaml | 2 -- 2 files changed, 18 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/ci-releasepy.yaml diff --git a/.github/workflows/ci-releasepy.yaml b/.github/workflows/ci-releasepy.yaml new file mode 100644 index 000000000..fa9f0c9c7 --- /dev/null +++ b/.github/workflows/ci-releasepy.yaml @@ -0,0 +1,18 @@ +name: Releasing tests + +on: + push: + paths: + - release.py + +jobs: + dsl_ci: + runs-on: ubuntu-latest + name: release.py checks + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Run release.py script tests + run: ./check_releasepy.bash diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 524dd498f..caa9d6c1d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -40,8 +40,6 @@ jobs: with: distribution: 'temurin' java-version: '11' - - name: Run release.py script tests - run: ./check_releasepy.bash - name: Download and setup job dsl jar if: steps.dsl_check.outputs.run_job == 'true' run: ./jenkins-scripts/dsl/tools/setup_local_generation.bash From 2b92f844a4b1c3331f0c6056963ed63e4e936456 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 30 Nov 2023 12:22:09 +0100 Subject: [PATCH 539/540] Change Citadel CI: from Bionic to Focal and change name from ign_ to use gz_ (#1083) * Move Citadel to use gz and Focal * Remove bionic from packaging Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/gz-collections.yaml | 92 +++++++------------------ 1 file changed, 23 insertions(+), 69 deletions(-) diff --git a/jenkins-scripts/dsl/gz-collections.yaml b/jenkins-scripts/dsl/gz-collections.yaml index 300235db5..50addd304 100644 --- a/jenkins-scripts/dsl/gz-collections.yaml +++ b/jenkins-scripts/dsl/gz-collections.yaml @@ -28,31 +28,31 @@ spec_version: 0 collections: - name: 'citadel' libs: - - name: ign-cmake + - name: gz-cmake major_version: 2 repo: current_branch: ign-cmake2 - - name: ign-tools + - name: gz-tools major_version: 1 repo: current_branch: ign-tools1 - - name: ign-math + - name: gz-math major_version: 6 repo: current_branch: ign-math6 - - name: ign-plugin + - name: gz-plugin major_version: 1 repo: current_branch: ign-plugin1 - - name: ign-common + - name: gz-common major_version: 3 repo: current_branch: ign-common3 - - name: ign-msgs + - name: gz-msgs major_version: 5 repo: current_branch: ign-msgs5 - - name: ign-rendering + - name: gz-rendering major_version: 3 repo: current_branch: ign-rendering3 @@ -60,49 +60,49 @@ collections: major_version: 9 repo: current_branch: sdf9 - - name: ign-fuel-tools + - name: gz-fuel-tools major_version: 4 repo: current_branch: ign-fuel-tools4 - - name: ign-transport + - name: gz-transport major_version: 8 repo: current_branch: ign-transport8 - - name: ign-gui + - name: gz-gui major_version: 3 repo: current_branch: ign-gui3 - - name: ign-sensors + - name: gz-sensors major_version: 3 repo: current_branch: ign-sensors3 - - name: ign-physics + - name: gz-physics major_version: 2 repo: current_branch: ign-physics2 - - name: ign-gazebo - major_version: 7 + - name: gz-sim + major_version: 3 repo: current_branch: ign-gazebo3 - - name: ign-launch + - name: gz-launch major_version: 2 repo: current_branch: ign-launch2 - - name: ign-citadel + - name: gz-citadel major_version: 1 repo: current_branch: main ci: configs: - - bionic + - focal - brew - win packaging: configs: - - bionic + - focal linux: ignore_major_version: - - ign-citadel + - gz-citadel - name: 'fortress' libs: - name: gz-cmake @@ -443,30 +443,6 @@ collections: exclude: - __upcoming__ ci_configs: - - name: bionic - system: - so: linux - distribution: ubuntu - version: bionic - arch: amd64 - exclude: - all: - - ign-citadel - abichecker: - - ign-cmake - - ign-tools - requirements: - large_memory: - - ign-physics - nvidia_gpu: - - ign-gazebo - - ign-gui - - ign-rendering - - ign-sensors - pre_setup_script_hook: - gz-physics: - - "export MAKE_JOBS=1" - tests_disabled: - name: focal system: so: linux @@ -483,6 +459,7 @@ ci_configs: - gz-sensors exclude: all: + - gz-citadel - gz-fortress - gz-garden abichecker: @@ -494,7 +471,7 @@ ci_configs: - gz-plugin - gz-utils pre_setup_script_hook: - ign-physics: + gz-physics: - "export MAKE_JOBS=1" tests_disabled: - name: jammy @@ -531,7 +508,7 @@ ci_configs: requirements: exclude: all: - - ign-citadel + - gz-citadel - gz-fortress - gz-garden - gz-harmonic @@ -562,7 +539,7 @@ ci_configs: requirements: exclude: all: - - ign-citadel + - gz-citadel - gz-fortress - gz-garden - gz-harmonic @@ -583,31 +560,8 @@ ci_configs: - gz-tools - gz-transport - gz-utils - - ign-cmake - - ign-common - - ign-fuel-tools - - ign-sim - - ign-gui - - ign-launch - - ign-math - - ign-msgs - - ign-physics - - ign-rendering - - ign-sensors - - ign-tools - - ign-transport - - ign-utils - sdformat packaging_configs: - - name: bionic - system: - so: linux - distribution: ubuntu - version: bionic - arch: - - i386 - - amd64 - - arm64 - name: focal system: so: linux From d468bd268b8de64268f107f453bc39df5b66c99f Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 30 Nov 2023 16:55:20 +0100 Subject: [PATCH 540/540] Clean up DSL CI by using GitHub actions paths (#1086) Signed-off-by: Jose Luis Rivero --- .github/workflows/ci.yaml | 37 +++++-------------------------------- 1 file changed, 5 insertions(+), 32 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index caa9d6c1d..8539f9ba4 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,7 +1,9 @@ -name: CI +name: DSL logs and checks -# base_ref / head_reaf are only available in PRs -on: [pull_request] +on: + pull_request: + paths: + - 'jenkins-scripts/dsl/**' jobs: dsl_ci: @@ -12,39 +14,17 @@ jobs: uses: actions/checkout@v3 with: fetch-depth: 0 - - name: Idenfify files changed in this PR - id: files - run: | - git fetch origin ${{ github.ref }} - git diff --name-only origin/${{ github.base_ref }}...FETCH_HEAD - echo "changed-files=$(git diff --name-only origin/${{ github.base_ref }}...FETCH_HEAD| tr '\n' ' ')" >> $GITHUB_OUTPUT - - name: Run testing on changed config files - id: dsl_check - run: | - for changed_file in ${{ steps.files.outputs.changed-files }}; do - if [[ ${changed_file} != ${changed_file/dsl\/*} ]]; then - echo "+ Detected at leat one config file: ${changed_file}." - echo "run_job=true" >> $GITHUB_OUTPUT - break - else - echo "run_job=false" >> $GITHUB_OUTPUT - fi - done - name: Checkout - if: steps.dsl_check.outputs.run_job == 'true' uses: actions/checkout@v3 with: fetch-depth: 2 - uses: actions/setup-java@v3 - if: steps.dsl_check.outputs.run_job == 'true' with: distribution: 'temurin' java-version: '11' - name: Download and setup job dsl jar - if: steps.dsl_check.outputs.run_job == 'true' run: ./jenkins-scripts/dsl/tools/setup_local_generation.bash - name: Generate all DSL files - if: steps.dsl_check.outputs.run_job == 'true' run: | # simulate token for brew_release sudo mkdir -p /var/lib/jenkins/ && sudo touch /var/lib/jenkins/remote_token @@ -52,12 +32,10 @@ jobs: cd jenkins-scripts/dsl WRITE_JOB_LOG=1 java -jar tools/jobdsl.jar *.dsl - name: Checks for DSL Code - if: steps.dsl_check.outputs.run_job == 'true' run: | cd jenkins-scripts/dsl ./dsl_checks.bash - name: Export XML generated configuration for diff - if: steps.dsl_check.outputs.run_job == 'true' run: | cd jenkins-scripts/dsl # export files for later diff @@ -67,7 +45,6 @@ jobs: mv *.xml /tmp/pr_xml_configuration/ mv *.txt /tmp/pr_log_generated/ - name: Generate master DSL files - if: steps.dsl_check.outputs.run_job == 'true' run: | git clean -f -e jobdsl.jar git checkout master @@ -79,26 +56,22 @@ jobs: mv *.xml /tmp/current_xml_configuration/ mv *.txt /tmp/current_log_generated/ || true - name: Generating diffs - if: steps.dsl_check.outputs.run_job == 'true' run: | # somehow the Jenkins views changed the portlet_ id on every run. diff -qr -I '.*dashboard_portlet_.*.*' /tmp/current_xml_configuration /tmp/pr_xml_configuration | sort > /tmp/xml_config_files_changed.diff || true diff -ur -I '.*dashboard_portlet_.*.*' /tmp/current_xml_configuration /tmp/pr_xml_configuration > /tmp/xml_config_content_changed.diff || true diff -ur /tmp/current_log_generated /tmp/pr_log_generated > /tmp/log_content_changed.diff || true - name: Archive files changes - if: steps.dsl_check.outputs.run_job == 'true' uses: actions/upload-artifact@v3 with: name: xml_config_files_changed path: /tmp/xml_config_files_changed.diff - name: Archive content changes - if: steps.dsl_check.outputs.run_job == 'true' uses: actions/upload-artifact@v3 with: name: xml_config_content_changed path: /tmp/xml_config_content_changed.diff - name: Archive log changes - if: steps.dsl_check.outputs.run_job == 'true' uses: actions/upload-artifact@v3 with: name: log_content_changed