Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Categories support for declarative CI configs in yaml #1123

Merged
merged 4 commits into from
Apr 4, 2024
Merged

Categories support for declarative CI configs in yaml #1123

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
2183ad8
Categories support for declarative CI configs in yaml
j-rivero Apr 3, 2024
4007ffa
Automated change: update logs [skip ci]
j-rivero Apr 3, 2024
739aaaf
Fix typo in categories
j-rivero Apr 3, 2024
141e96c
Automated change: update logs [skip ci]
j-rivero Apr 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
226 changes: 120 additions & 106 deletions jenkins-scripts/dsl/gazebo_libs.dsl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -335,143 +335,157 @@ ciconf_per_lib_index.each { lib_name, lib_configs ->
def gz_job_name_prefix = lib_name.replaceAll('-','_')
def distro = ci_config.system.version
def arch = ci_config.system.arch
def categories_enabled = ci_config.ci_categories_enabled
def ws_checkout_dir = lib_name
if (ci_config.exclude.all?.contains(lib_name))
return
assert(lib_name)
assert(branch_names)
assert(ci_config)
assert(categories_enabled)

// CI branch jobs (-ci-$branch-) (pulling check every 5 minutes)
branches_with_collections.each { branch_and_collection ->
def gz_ci_job
branch_name = branch_and_collection.branch
if (ci_config.system.so == 'linux') {
gz_ci_job = job("${gz_job_name_prefix}-ci-${branch_name}-${distro}-${arch}")
generate_ci_job(gz_ci_job, lib_name, branch_name, ci_config)
// Generate asan jobs on Linux
def gz_ci_asan_job = job("${gz_job_name_prefix}-ci_asan-${branch_name}-${distro}-${arch}")
generate_asan_ci_job(gz_ci_asan_job, lib_name, branch_name, ci_config)
gz_ci_asan_job.with
if (categories_enabled.contains('stable_branches'))
{
if (ci_config.system.so == 'linux') {
gz_ci_job = job("${gz_job_name_prefix}-ci-${branch_name}-${distro}-${arch}")
generate_ci_job(gz_ci_job, lib_name, branch_name, ci_config)
if (categories_enabled.contains('stable_branches_asan'))
{
// Generate asan jobs on Linux
def gz_ci_asan_job = job("${gz_job_name_prefix}-ci_asan-${branch_name}-${distro}-${arch}")
generate_asan_ci_job(gz_ci_asan_job, lib_name, branch_name, ci_config)
gz_ci_asan_job.with
{
triggers {
scm(Globals.CRON_ON_WEEKEND)
}
}
logging_list['asan_ci'].add(
[collection: branch_and_collection.collection,
job_name: gz_ci_asan_job.name])
}
} else if (ci_config.system.so == 'darwin') {
gz_ci_job = job("${gz_job_name_prefix}-ci-${branch_name}-homebrew-${arch}")
generate_brew_ci_job(gz_ci_job, lib_name, branch_name, ci_config)
} else if (ci_config.system.so == 'windows') {
branch_number = branch_name - lib_name
Globals.gazebodistro_branch = true
gz_ci_job = job("${gz_job_name_prefix}-${branch_number}-win")
generate_win_ci_job(gz_ci_job, lib_name, branch_name, ci_config)
Globals.gazebodistro_branch = false
} else {
assert false : "Unexpected config.system.so type: ${ci_config.system.so}"
}

gz_ci_job.with
{
triggers {
scm(Globals.CRON_ON_WEEKEND)
scm('@daily')
}
}
logging_list['asan_ci'].add(
[collection: branch_and_collection.collection,
job_name: gz_ci_asan_job.name])
} else if (ci_config.system.so == 'darwin') {
gz_ci_job = job("${gz_job_name_prefix}-ci-${branch_name}-homebrew-${arch}")
generate_brew_ci_job(gz_ci_job, lib_name, branch_name, ci_config)
} else if (ci_config.system.so == 'windows') {
branch_number = branch_name - lib_name
Globals.gazebodistro_branch = true
gz_ci_job = job("${gz_job_name_prefix}-${branch_number}-win")
generate_win_ci_job(gz_ci_job, lib_name, branch_name, ci_config)
Globals.gazebodistro_branch = false
} else {
assert false : "Unexpected config.system.so type: ${ci_config.system.so}"
}

gz_ci_job.with
{
triggers {
scm('@daily')
}
}

logging_list['branch_ci'].add(
[collection: branch_and_collection.collection,
job_name: gz_ci_job.name])
logging_list['branch_ci'].add(
[collection: branch_and_collection.collection,
job_name: gz_ci_job.name])
} // end of daily category enabled
} // end_of_branch

if (ci_config.system.so == 'linux') {
def pre_setup_script = ci_config.pre_setup_script_hook?.get(lib_name)?.join('\n')
def extra_cmd = pre_setup_script ?: ""

def gz_ci_job_name = "${gz_job_name_prefix}-ci-pr_any-${distro}-${arch}"
def gz_ci_any_job = job(gz_ci_job_name)
OSRFLinuxCompilationAnyGitHub.create(gz_ci_any_job,
"gazebosim/${lib_name}",
is_testing_enabled(lib_name, ci_config),
ENABLE_CPPCHECK,
branch_names)
generate_label_by_requirements(gz_ci_any_job, lib_name, ci_config.requirements)
gz_ci_any_job.with
if (categories_enabled.contains('pr'))
{
if (ci_config.system.so == 'linux')
{
steps
def pre_setup_script = ci_config.pre_setup_script_hook?.get(lib_name)?.join('\n')
def extra_cmd = pre_setup_script ?: ""

def gz_ci_job_name = "${gz_job_name_prefix}-ci-pr_any-${distro}-${arch}"
def gz_ci_any_job = job(gz_ci_job_name)
OSRFLinuxCompilationAnyGitHub.create(gz_ci_any_job,
"gazebosim/${lib_name}",
is_testing_enabled(lib_name, ci_config),
ENABLE_CPPCHECK,
branch_names)
generate_label_by_requirements(gz_ci_any_job, lib_name, ci_config.requirements)
gz_ci_any_job.with
{
shell("""\
#!/bin/bash -xe

export DISTRO=${distro}

${GLOBAL_SHELL_CMD}
${extra_cmd}

export BUILDING_SOFTWARE_DIRECTORY=${lib_name}
export ARCH=${arch}
/bin/bash -xe ./scripts/jenkins-scripts/docker/${script_name_prefix}-compilation.bash
""".stripIndent())
} // end of steps
} // end of ci_any_job

if (! ci_config.exclude.abichecker?.contains(lib_name)) {
// ABI branch jobs (-ci-abichecker-) for non main branches
def abi_job_name = "${gz_job_name_prefix}-abichecker-any_to_any-ubuntu-${distro}-${arch}"
def abi_job = job(abi_job_name)
OSRFLinuxABIGitHub.create(abi_job)
GenericAnyJobGitHub.create(abi_job,
"gazebosim/${lib_name}",
branch_names - [ 'main'])
generate_label_by_requirements(abi_job, lib_name, ci_config.requirements)
abi_job.with
{
steps {
shell("""\
steps
{
shell("""\
#!/bin/bash -xe

export DISTRO=${distro}

${GLOBAL_SHELL_CMD}
${extra_cmd}

export BUILDING_SOFTWARE_DIRECTORY=${lib_name}
export ARCH=${arch}
export DEST_BRANCH=\${DEST_BRANCH:-\$ghprbTargetBranch}
export SRC_BRANCH=\${SRC_BRANCH:-\$ghprbSourceBranch}
export SRC_REPO=\${SRC_REPO:-\$ghprbAuthorRepoGitUrl}
export ABI_JOB_SOFTWARE_NAME=${lib_name}
/bin/bash -xe ./scripts/jenkins-scripts/docker/gz-abichecker.bash
/bin/bash -xe ./scripts/jenkins-scripts/docker/${script_name_prefix}-compilation.bash
""".stripIndent())
} // end of steps
} // end of with
} // end of ci_any_job

if (categories_enabled.contains('stable_branches') && \
(! ci_config.exclude.abichecker?.contains(lib_name)))
{
// ABI branch jobs (-ci-abichecker-) for non main branches
def abi_job_name = "${gz_job_name_prefix}-abichecker-any_to_any-ubuntu-${distro}-${arch}"
def abi_job = job(abi_job_name)
OSRFLinuxABIGitHub.create(abi_job)
GenericAnyJobGitHub.create(abi_job,
"gazebosim/${lib_name}",
branch_names - [ 'main'])
generate_label_by_requirements(abi_job, lib_name, ci_config.requirements)
abi_job.with
{
steps {
shell("""\
#!/bin/bash -xe

export DISTRO=${distro}

${GLOBAL_SHELL_CMD}
${extra_cmd}

export ARCH=${arch}
export DEST_BRANCH=\${DEST_BRANCH:-\$ghprbTargetBranch}
export SRC_BRANCH=\${SRC_BRANCH:-\$ghprbSourceBranch}
export SRC_REPO=\${SRC_REPO:-\$ghprbAuthorRepoGitUrl}
export ABI_JOB_SOFTWARE_NAME=${lib_name}
/bin/bash -xe ./scripts/jenkins-scripts/docker/gz-abichecker.bash
""".stripIndent())
} // end of steps
} // end of with
}
} else if (ci_config.system.so == 'darwin') {
// --------------------------------------------------------------
def gz_brew_ci_any_job_name = "${gz_job_name_prefix}-ci-pr_any-homebrew-amd64"
def gz_brew_ci_any_job = job(gz_brew_ci_any_job_name)
OSRFBrewCompilationAnyGitHub.create(gz_brew_ci_any_job,
"gazebosim/${lib_name}",
is_testing_enabled(lib_name, ci_config),
branch_names,
ENABLE_GITHUB_PR_INTEGRATION,
are_cmake_warnings_enabled(lib_name, ci_config))
add_brew_shell_build_step(gz_brew_ci_any_job, lib_name, ws_checkout_dir)
} else if (ci_config.system.so == 'windows') {
def gz_win_ci_any_job_name = "${gz_job_name_prefix}-pr-win"
def gz_win_ci_any_job = job(gz_win_ci_any_job_name)
Globals.gazebodistro_branch = true
OSRFWinCompilationAnyGitHub.create(gz_win_ci_any_job,
"gazebosim/${lib_name}",
is_testing_enabled(lib_name, ci_config),
branch_names,
ENABLE_GITHUB_PR_INTEGRATION,
are_cmake_warnings_enabled(lib_name, ci_config))
add_win_devel_bat_call(gz_win_ci_any_job, lib_name, ws_checkout_dir)
Globals.gazebodistro_branch = false
}
} else if (ci_config.system.so == 'darwin') {
// --------------------------------------------------------------
def gz_brew_ci_any_job_name = "${gz_job_name_prefix}-ci-pr_any-homebrew-amd64"
def gz_brew_ci_any_job = job(gz_brew_ci_any_job_name)
OSRFBrewCompilationAnyGitHub.create(gz_brew_ci_any_job,
"gazebosim/${lib_name}",
is_testing_enabled(lib_name, ci_config),
branch_names,
ENABLE_GITHUB_PR_INTEGRATION,
are_cmake_warnings_enabled(lib_name, ci_config))
add_brew_shell_build_step(gz_brew_ci_any_job, lib_name, ws_checkout_dir)
} else if (ci_config.system.so == 'windows') {
def gz_win_ci_any_job_name = "${gz_job_name_prefix}-pr-win"
def gz_win_ci_any_job = job(gz_win_ci_any_job_name)
Globals.gazebodistro_branch = true
OSRFWinCompilationAnyGitHub.create(gz_win_ci_any_job,
"gazebosim/${lib_name}",
is_testing_enabled(lib_name, ci_config),
branch_names,
ENABLE_GITHUB_PR_INTEGRATION,
are_cmake_warnings_enabled(lib_name, ci_config))
add_win_devel_bat_call(gz_win_ci_any_job, lib_name, ws_checkout_dir)
Globals.gazebodistro_branch = false
}
} //en of lib_configs
} //end of pr enabled
} //end of lib_configs
} // end of lib

pkgconf_per_src_index.each { pkg_src, pkg_src_configs ->
Expand Down
19 changes: 19 additions & 0 deletions jenkins-scripts/dsl/gz-collections.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -467,6 +467,11 @@ ci_configs:
gz-physics:
- "export MAKE_JOBS=1"
tests_disabled:
ci_categories_enabled:
- pr
- pr_abichecker
- stable_branches
- stable_branches_asan
- name: jammy
system:
so: linux
Expand All @@ -492,6 +497,11 @@ ci_configs:
gz-physics:
- "export MAKE_JOBS=1"
tests_disabled:
ci_categories_enabled:
- pr
- pr_abichecker
- stable_branches
- stable_branches_asan
- name: brew
system:
so: darwin
Expand Down Expand Up @@ -523,6 +533,9 @@ ci_configs:
- gz-transport
- gz-utils
- sdformat
ci_categories_enabled:
- pr
- stable_branches
- name: win
system:
so: windows
Expand Down Expand Up @@ -553,6 +566,9 @@ ci_configs:
- gz-transport
- gz-utils
- sdformat
ci_categories_enabled:
- pr
- stable_branches
# No gz-sim and gz-launch on Citadel
- name: win_citadel
system:
Expand Down Expand Up @@ -580,6 +596,9 @@ ci_configs:
- gz-transport
- gz-utils
- sdformat
ci_categories_enabled:
- pr
- stable_branches
packaging_configs:
- name: focal
system:
Expand Down