diff --git a/jenkins-scripts/dsl/dsl_checks.bash b/jenkins-scripts/dsl/dsl_checks.bash index e4dfe8f25..876750389 100755 --- a/jenkins-scripts/dsl/dsl_checks.bash +++ b/jenkins-scripts/dsl/dsl_checks.bash @@ -6,7 +6,9 @@ if [[ -z $(ls -- *.xml) ]]; then exit 1 fi -not_null=$(grep -3 'null' -- *.xml || true) +# Some XML generated files include valid Groovy code using null +# like the _outdated_jobs job. Do not search for null unconditionally +not_null=$(grep -3 '>.*null.*<' -- *.xml || true) if [[ -n ${not_null} ]]; then echo "Found a null value in a configuration file:" echo "${not_null}" @@ -50,16 +52,27 @@ if [[ -n ${non_github_orgs} ]]; then exit 1 fi +# Check that whiteListedTargetBranches are non empty in all generated gazebo_libs +# see https://github.com/gazebo-tooling/release-tools/pull/1144 +# For other jobs the use case is valid since pr can be enabled on all branches +empty_branches_on_github_triggered=$(grep '' \ + -- {gz_,sdformat}*{-abichecker-,-pr_any-}*.xml || true) +if [[ -n ${empty_branches_on_github_triggered} ]]; then + echo "Unexpected whiteListTargetBranches without values. It will trigger all branches:" + echo "${empty_branches_on_github_triggered}" + exit 1 +fi + # re-enable after https://github.com/gazebo-tooling/release-tools/issues/1095 # Filter out the previous auto jobs -# filtered_dir=$(mktemp -d) -# cp -- *-abichecker-*.xml "${filtered_dir}" -# rm -f "${filtered_dir}"/*-ubuntu_auto*.xml -# repeated=$(grep '\' "${filtered_dir}"/*-abichecker-*.xml | awk '{ print $2 }' | sort | uniq -d) -# if [[ -n ${repeated} ]]; then -# echo "Found a duplicate in an abichecker branch:" -# echo "${repeated}" -# echo "please exclude one of the versions in the yaml file to reduce the server workload" -# exit 1 -# fi +filtered_dir=$(mktemp -d) +cp -- *-abichecker-*.xml "${filtered_dir}" +rm -f "${filtered_dir}"/*-ubuntu_auto*.xml +repeated=$(grep '\' "${filtered_dir}"/*-abichecker-*.xml | awk '{ print $2 }' | sort | uniq -d) +if [[ -n ${repeated} ]]; then + echo "Found a duplicate in an abichecker branch:" + echo "${repeated}" + echo "please exclude one of the versions in the yaml file to reduce the server workload" + exit 1 +fi diff --git a/jenkins-scripts/dsl/gazebo_libs.dsl b/jenkins-scripts/dsl/gazebo_libs.dsl index 3b3b18a0b..b56f6b99c 100644 --- a/jenkins-scripts/dsl/gazebo_libs.dsl +++ b/jenkins-scripts/dsl/gazebo_libs.dsl @@ -345,6 +345,7 @@ gz_collections_yaml.collections.each { collection -> branch_index[lib_name][platform]['pr'] << [branch: branch_name, ci_name: config_name] } if (categories_enabled.contains('pr_abichecker') && + (branch_name != 'main') && (! ci_config.exclude.abichecker?.contains(lib_name))) { branch_index[lib_name][platform]['pr_abichecker'].contains(branch_name) ?: @@ -482,7 +483,7 @@ branch_index.each { lib_name, distro_configs -> OSRFLinuxABIGitHub.create(abi_job) GenericAnyJobGitHub.create(abi_job, "gazebosim/${lib_name}", - branch_names - [ 'main']) + branch_names) generate_label_by_requirements(abi_job, lib_name, ci_config.requirements) abi_job.with { diff --git a/jenkins-scripts/dsl/gz-collections.yaml b/jenkins-scripts/dsl/gz-collections.yaml index fdf0ba837..7dc3f4f3c 100644 --- a/jenkins-scripts/dsl/gz-collections.yaml +++ b/jenkins-scripts/dsl/gz-collections.yaml @@ -1,4 +1,4 @@ -# + # SPEC: not yet approved, used as draft # #--- @@ -256,7 +256,7 @@ collections: current_branch: main ci: configs: - - focal + - focal_exclude_harmonic_abi_dups - brew - win packaging: @@ -413,6 +413,10 @@ collections: major_version: 8 repo: current_branch: main + - name: gz-ionic + major_version: 1 + repo: + current_branch: main ci: configs: - noble @@ -503,6 +507,43 @@ ci_configs: - pr_abichecker - stable_branches - stable_branches_asan + - name: focal_exclude_harmonic_abi_dups + system: + so: linux + distribution: ubuntu + version: focal + arch: amd64 + requirements: + large_memory: + - gz-physics + nvidia_gpu: + - gz-sim + - gz-gui + - gz-rendering + - gz-sensors + exclude: + all: + - gz-citadel + - gz-fortress + - gz-garden + abichecker: + - gz-cmake + - gz-tools + # Exclude the abichecker that are duplicate with Harmonic + # The ones on Harmonic run on jammy + - gz-common + - gz-math + - gz-plugin + - gz-utils + pre_setup_script_hook: + gz-physics: + - "export MAKE_JOBS=1" + tests_disabled: + ci_categories_enabled: + - pr + - pr_abichecker + - stable_branches + - stable_branches_asan - name: noble system: so: linux @@ -520,6 +561,7 @@ ci_configs: exclude: all: - gz-harmonic + - gz-ionic - __upcoming__ abichecker: - gz-cmake diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index 43018c8df..e00232093 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -62,25 +62,6 @@ void generate_install_job(prefix, gz_collection_name, distro, arch) // Testing compilation from source gz_collections_yaml.collections.each { collection -> gz_collection_name = collection.name - - if (! collection.packaging.exclude?.contains(gz_collection_name)) { - // DEBBUILD: linux package builder - // -------------------------------------------------------------- - def build_pkg_job = job("gz-${gz_collection_name}-debbuilder") - OSRFLinuxBuildPkg.create(build_pkg_job) - build_pkg_job.with - { - steps { - shell("""\ - #!/bin/bash -xe - - /bin/bash -x ./scripts/jenkins-scripts/docker/multidistribution-ignition-debbuild.bash - """.stripIndent()) - } - } - } - - collection.ci.configs.each { ci_config_name -> ci_config = gz_collections_yaml.ci_configs.find { it.name == ci_config_name } distro = ci_config.system.version @@ -153,74 +134,6 @@ gz_collections_yaml.collections.each { collection -> """.stripIndent()) } } - - // COLCON - Windows - Globals.gazebodistro_branch = true - def gz_win_ci_job = job("ign_${gz_collection_name}-ci-win") - OSRFWinCompilation.create(gz_win_ci_job, false) - gz_win_ci_job.with - { - steps { - batchFile("""\ - set IGNITION_COLLECTION=${gz_collection_name} - call "./scripts/jenkins-scripts/lib/ign_collection-base.bat" - """.stripIndent()) - } - } - Globals.gazebodistro_branch = false - - // MAC Brew CI job - // -------------------------------------------------------------- - def gz_brew_ci_job = job("ignition_${gz_collection_name}-ci-main-homebrew-amd64") - OSRFBrewCompilation.create(gz_brew_ci_job, DISABLE_TESTS) - OSRFGitHub.create(gz_brew_ci_job, - "gazebosim/gz-${gz_collection_name}", - "main", - "ign-${gz_collection_name}") - gz_brew_ci_job.with - { - steps { - shell("""\ - #!/bin/bash -xe - - /bin/bash -xe - "./scripts/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash" "gz-${gz_collection_name}" - """.stripIndent()) - } - } - - // MAC Brew bottle install job - // -------------------------------------------------------------- - def gz_brew_install_bottle_job = job("ignition_${gz_collection_name}-install_bottle-homebrew-amd64") - OSRFBrewInstall.create(gz_brew_install_bottle_job) - - gz_brew_install_bottle_job.with - { - triggers { - cron('@daily') - } - - def bottle_name = "ignition-${gz_collection_name}" - - steps { - shell("""\ - #!/bin/bash -xe - - /bin/bash -x ./scripts/jenkins-scripts/lib/project-install-homebrew.bash ${bottle_name} - """.stripIndent()) - } - - publishers - { - configure { project -> - project / publishers << 'hudson.plugins.logparser.LogParserPublisher' { - unstableOnWarning true - failBuildOnError false - parsingRulesPath('/var/lib/jenkins/logparser_warn_on_mark_unstable') - } - } - } - } } } diff --git a/jenkins-scripts/dsl/logs/generated_jobs.txt b/jenkins-scripts/dsl/logs/generated_jobs.txt index bfbf662a7..a41368dcf 100644 --- a/jenkins-scripts/dsl/logs/generated_jobs.txt +++ b/jenkins-scripts/dsl/logs/generated_jobs.txt @@ -476,6 +476,8 @@ install_ci ionic gz_fuel_tools10-install-pkg-jammy-amd64 install_ci ionic gz_fuel_tools10-install_bottle-homebrew-amd64 install_ci ionic gz_gui9-install-pkg-jammy-amd64 install_ci ionic gz_gui9-install_bottle-homebrew-amd64 +install_ci ionic gz_ionic-install-pkg-jammy-amd64 +install_ci ionic gz_ionic-install_bottle-homebrew-amd64 install_ci ionic gz_launch8-install-pkg-jammy-amd64 install_ci ionic gz_launch8-install_bottle-homebrew-amd64 install_ci ionic gz_math8-install-pkg-jammy-amd64 diff --git a/jenkins-scripts/dsl/sdformat.dsl b/jenkins-scripts/dsl/sdformat.dsl deleted file mode 100644 index 01f9803b9..000000000 --- a/jenkins-scripts/dsl/sdformat.dsl +++ /dev/null @@ -1,200 +0,0 @@ -import _configs_.* -import javaposse.jobdsl.dsl.Job - -def sdformat_supported_versions = [ 'sdformat6' , 'sdformat9', 'sdformat12', 'sdformat13', 'sdformat14' ] -def sdformat_gz11_versions = [ 'sdformat9', 'sdformat12', 'sdformat13', 'sdformat14', 'main' ] -// nightly and prereleases -def extra_sdformat_debbuilder = ['sdformat15'] - -// Main platform using for quick CI -def ci_distro = Globals.get_ci_distro() -def abi_distro = Globals.get_abi_distro() -// Other supported platform to be checked but no for quick -// CI integration. -def other_supported_distros = Globals.get_other_supported_distros() -def all_supported_distros = Globals.get_all_supported_distros() -def supported_arches = Globals.get_supported_arches() -def experimental_arches = Globals.get_experimental_arches() - -String ci_distro_str = ci_distro[0] - -// Need to be used in ci_pr -String abi_job_name = '' - -// Helper function -String get_sdformat_branch_name(String full_branch_name) -{ - String sdf_branch = full_branch_name.replace("ormat",'') - - return sdf_branch -} - -// ABI Checker job -// Need to be the before ci-pr_any so the abi job name is defined -abi_branches = sdformat_supported_versions.collect { it -> get_sdformat_branch_name(it) } -abi_distro.each { distro -> - supported_arches.each { arch -> - abi_job_name = "sdformat-abichecker-any_to_any-ubuntu_auto-${arch}" - def abi_job = job(abi_job_name) - OSRFLinuxABIGitHub.create(abi_job) - GenericAnyJobGitHub.create(abi_job, 'gazebosim/sdformat', abi_branches) - abi_job.with - { - steps { - shell("""\ - #!/bin/bash -xe - - export DISTRO=${distro} - - export ARCH=${arch} - export DEST_BRANCH=\${DEST_BRANCH:-\$ghprbTargetBranch} - export SRC_BRANCH=\${SRC_BRANCH:-\$ghprbSourceBranch} - export SRC_REPO=\${SRC_REPO:-\$ghprbAuthorRepoGitUrl} - - /bin/bash -xe ./scripts/jenkins-scripts/docker/sdformat-abichecker.bash - """.stripIndent()) - } // end of steps - } // end of with - } // end of arch -} // end of distro - -void generate_asan_ci_job(sdformat_ci_job, version, distro, arch) -{ - generate_ci_job(sdformat_ci_job, version, distro, arch, - '-DGZ_SANITIZER=Address', - Globals.MAKETEST_SKIP_GZ, - 'export ASAN_OPTIONS=check_initialization_order=true:strict_init_order=true') -} - - -void generate_ci_job(sdformat_ci_job, version, distro, arch, - extra_cmake = '', extra_test = '', extra_cmd = '') -{ - OSRFLinuxCompilation.create(sdformat_ci_job) - OSRFGitHub.create(sdformat_ci_job, "gazebosim/sdformat", - get_sdformat_branch_name(version)) - sdformat_ci_job.with - { - steps { - shell("""\ - #!/bin/bash -xe - - ${extra_cmd} - export BUILDING_EXTRA_CMAKE_PARAMS="${extra_cmake}" - export BUILDING_EXTRA_MAKETEST_PARAMS="${extra_test}" - export DISTRO=${distro} - export ARCH=${arch} - /bin/bash -xe ./scripts/jenkins-scripts/docker/sdformat-compilation.bash - """.stripIndent()) - } - } -} - -// BRANCHES CI JOB @ SCM -sdformat_supported_versions.each { version -> - ci_distro.each { distro -> - supported_arches.each { arch -> - // ci_asan job for the rest of arches / scm@weekend - def sdformat_ci_asan_job = job("sdformat-ci_asan-${version}-${distro}-${arch}") - generate_asan_ci_job(sdformat_ci_asan_job, version, distro, arch) - sdformat_ci_asan_job.with - { - triggers { - scm(Globals.CRON_ON_WEEKEND) - } - } - } - } -} - -// INSTALL LINUX -DEV PACKAGES ALL PLATFORMS @ CRON/DAILY -sdformat_supported_versions.each { version -> - ci_distro.each { distro -> - supported_arches.each { arch -> - // -------------------------------------------------------------- - def install_default_job = job("sdformat-install-${version}_pkg-${distro}-${arch}") - OSRFLinuxInstall.create(install_default_job) - install_default_job.with - { - triggers { - cron(Globals.CRON_EVERY_THREE_DAYS) - } - - def dev_package = "lib${version}-dev" - - steps { - shell("""\ - #!/bin/bash -xe - - export DISTRO=${distro} - export ARCH=${arch} - export INSTALL_JOB_PKG=${dev_package} - export GZDEV_PROJECT_NAME="${version}" - /bin/bash -x ./scripts/jenkins-scripts/docker/generic-install-test-job.bash - """.stripIndent()) - } - } // end of with - } // end of arch - } // end of distro -} // end of version - -// -------------------------------------------------------------- -// DEBBUILD: linux package builder - -all_debbuild_versions = sdformat_supported_versions + extra_sdformat_debbuilder -all_debbuild_versions.each { version -> - def build_pkg_job = job("${version}-debbuilder") - OSRFLinuxBuildPkg.create(build_pkg_job) - - extra_cmd_str = "" - if (version in sdformat_gz11_versions) - extra_cmd_str = "export NEED_C17_COMPILER=true" - - build_pkg_job.with - { - steps { - shell("""\ - #!/bin/bash -xe - - ${extra_cmd_str} - export ENABLE_ROS=false - /bin/bash -x ./scripts/jenkins-scripts/docker/multidistribution-debbuild.bash - """.stripIndent()) - } - } -} - -// -------------------------------------------------------------- -// BREW: CI jobs - -// install jobs to test bottles -sdformat_supported_versions.each { version -> - def install_default_job = job("${version}-install_bottle-homebrew-amd64") - OSRFBrewInstall.create(install_default_job) - - install_default_job.with - { - triggers { - cron('@daily') - } - - steps { - shell("""\ - #!/bin/bash -xe - - /bin/bash -x ./scripts/jenkins-scripts/lib/project-install-homebrew.bash ${version} - """.stripIndent()) - } - - publishers - { - configure { project -> - project / publishers << 'hudson.plugins.logparser.LogParserPublisher' { - unstableOnWarning true - failBuildOnError false - parsingRulesPath('/var/lib/jenkins/logparser_warn_on_mark_unstable') - } - } - } - } -} diff --git a/jenkins-scripts/tools/outdated-job-runner.groovy b/jenkins-scripts/tools/outdated-job-runner.groovy index 7165da8d6..69e2ea469 100644 --- a/jenkins-scripts/tools/outdated-job-runner.groovy +++ b/jenkins-scripts/tools/outdated-job-runner.groovy @@ -88,8 +88,8 @@ long fourDaysAgoMillis = System.currentTimeMillis() - 4 * 24 * 60 * 60 * 1000; / Date fourDaysAgoDate = new Date(fourDaysAgoMillis); jenkinsJobs.getItems(Project).each { project -> - // Filter jobs that have not been updated in the last 8 days - if (!project.disabled && trackedJobs.contains(project.displayName) && project.lastBuild.getTime().before(fourDaysAgoDate)) { + // Filter jobs that have not been run in 4 days (or that have not yet run) + if (!project.disabled && trackedJobs.contains(project.displayName) && (project.lastBuild == null || project.lastBuild.getTime().before(fourDaysAgoDate))) { if (project.displayName.contains('homebrew')) { jobsToRun.osx << project.displayName } else if (project.displayName.contains('win')) {