diff --git a/check_releasepy.bash b/check_releasepy.bash index 9d5c100c4..814924955 100755 --- a/check_releasepy.bash +++ b/check_releasepy.bash @@ -1,6 +1,7 @@ #!/bin/bash -e export _RELEASEPY_DEBUG=1 + test_dir=$(mktemp -d) export _RELEASEPY_TEST_RELEASE_REPO="${test_dir}/test-release" mkdir -p ${_RELEASEPY_TEST_RELEASE_REPO}/{focal,jammy,ubuntu}/debian @@ -25,7 +26,8 @@ exec_releasepy_test() ./release.py \ --dry-run \ --no-sanity-checks \ - gz-foo 1.2.3 token ${test_params} + --auth user:fake \ + gz-foo 1.2.3 ${test_params} } exec_ignition_releasepy_test() @@ -35,7 +37,8 @@ exec_ignition_releasepy_test() ./release.py \ --dry-run \ --no-sanity-checks \ - ign-foo 1.2.3 token ${test_params} + --auth user:fake \ + ign-foo 1.2.3 ${test_params} } exec_ignition_gazebo_releasepy_test() @@ -45,7 +48,8 @@ exec_ignition_gazebo_releasepy_test() ./release.py \ --dry-run \ --no-sanity-checks \ - ign-gazebo 1.2.3 token ${test_params} + --auth user:fake \ + ign-gazebo 1.2.3 ${test_params} } exec_releasepy_with_real_gz() @@ -54,9 +58,10 @@ exec_releasepy_with_real_gz() ./release.py \ --dry-run \ --no-sanity-checks \ + --auth user:fake \ --source-repo-uri http://github.com/gazebosim/gz-common \ --source-repo-existing-ref http://github.com/gazebosim/gz-common/foo-tag \ - "${gz_pkg}" "${major_version}.x.y" token + "${gz_pkg}" "${major_version}.x.y" } expect_job_run() diff --git a/jenkins-scripts/dsl/_configs_/GenericRemoteToken.groovy b/jenkins-scripts/dsl/_configs_/GenericRemoteToken.groovy deleted file mode 100644 index 07e29d7c2..000000000 --- a/jenkins-scripts/dsl/_configs_/GenericRemoteToken.groovy +++ /dev/null @@ -1,34 +0,0 @@ -package _configs_ - -import javaposse.jobdsl.dsl.Job - -class GenericRemoteToken -{ - // FIXME getEnvVars can not be called in a static scope. Hardcoded by now. - // static File token_file = new File(build.getEnvVars()['HOME'] + '/remote_token') - static File token_file = new File('/var/lib/jenkins/remote_token') - - static void create(Job job) - { - if (! token_file.exists()) { - println("!!! token file was not found for setting the remote password") - println("check your filesystem in the jenkins node for: ") - println(token_file) - // We can not use exit here, DSL job hangs - job.with - { - steps { - setBuildResult('UNSTABLE') - } - } - } - - job.with - { - // remote calls don't have DSL implementation - configure { project -> - project / authToken(token_file.text.replaceAll("[\n\r]", "")) - } - } - } -} diff --git a/jenkins-scripts/dsl/_configs_/OSRFCredentials.groovy b/jenkins-scripts/dsl/_configs_/OSRFCredentials.groovy index 943ab345c..961344aed 100644 --- a/jenkins-scripts/dsl/_configs_/OSRFCredentials.groovy +++ b/jenkins-scripts/dsl/_configs_/OSRFCredentials.groovy @@ -13,8 +13,8 @@ class OSRFCredentials credentialsBinding { crendentials_list.each { credential_keyword -> if (credential_keyword == 'OSRFBUILD_GITHUB_TOKEN') { - usernamePassword('OSRFBUILD_USER', - 'OSRFBUILD_TOKEN', + usernamePassword('OSRFBUILD_GITHUB_USER', + 'OSRFBUILD_GITHUB_TOKEN', 'github-osrfbuild-apitoken') } else if (credential_keyword == 'OSRFBUILD_JENKINS_TOKEN') { usernamePassword('OSRFBUILD_JENKINS_USER', diff --git a/jenkins-scripts/dsl/_configs_/OSRFLinuxBackportPkg.groovy b/jenkins-scripts/dsl/_configs_/OSRFLinuxBackportPkg.groovy index 077e57e56..d3b28888d 100644 --- a/jenkins-scripts/dsl/_configs_/OSRFLinuxBackportPkg.groovy +++ b/jenkins-scripts/dsl/_configs_/OSRFLinuxBackportPkg.groovy @@ -4,7 +4,6 @@ import javaposse.jobdsl.dsl.Job /* -> OSRFLinuxBase - -> GenericRemoteToken Implements: - priorioty 300 @@ -24,7 +23,6 @@ class OSRFLinuxBackportPkg static void create(Job job) { OSRFLinuxBase.create(job) - GenericRemoteToken.create(job) job.with { diff --git a/jenkins-scripts/dsl/_configs_/OSRFLinuxBuildPkg.groovy b/jenkins-scripts/dsl/_configs_/OSRFLinuxBuildPkg.groovy index 41d8aac15..fa100e6d9 100644 --- a/jenkins-scripts/dsl/_configs_/OSRFLinuxBuildPkg.groovy +++ b/jenkins-scripts/dsl/_configs_/OSRFLinuxBuildPkg.groovy @@ -2,10 +2,10 @@ package _configs_ import javaposse.jobdsl.dsl.Job import _configs_.Globals +import _configs_.OSRFCredentials /* -> OSRFLinuxBuildPkgBase - -> GenericRemoteToken Implements: - priority 100 @@ -28,7 +28,7 @@ class OSRFLinuxBuildPkg static void create(Job job, Map default_params = [:]) { OSRFLinuxBuildPkgBase.create(job) - GenericRemoteToken.create(job) + OSRFCredentials.allowOsrfbuildToRunTheBuild(job) job.with { diff --git a/jenkins-scripts/dsl/_configs_/OSRFReleasepy.groovy b/jenkins-scripts/dsl/_configs_/OSRFReleasepy.groovy index 2dac2bce2..081099d5d 100644 --- a/jenkins-scripts/dsl/_configs_/OSRFReleasepy.groovy +++ b/jenkins-scripts/dsl/_configs_/OSRFReleasepy.groovy @@ -9,6 +9,7 @@ class OSRFReleasepy { // Base class for the job OSRFUNIXBase.create(job) + OSRFCredentials.setOSRFCrendentials(job, ['OSRFBUILD_JENKINS_TOKEN']) job.with { @@ -58,8 +59,6 @@ class OSRFReleasepy shell("""\ #!/bin/bash -xe - set +x # keep password secret - PASS=\$(cat \$HOME/build_pass) dry_run_str="" if \$DRY_RUN; then @@ -72,10 +71,11 @@ class OSRFReleasepy fi echo "releasing \${n} (from branch \${src_branch})" - python3 ./scripts/release.py \${dry_run_str} "\${PACKAGE}" "\${VERSION}" "\${PASS}" \${extra_osrf_repo} \ + python3 ./scripts/release.py \${dry_run_str} "\${PACKAGE}" "\${VERSION}" \${extra_osrf_repo} \ + --auth "\${OSRFBUILD_JENKINS_USER}:\${OSRFBUILD_JENKINS_TOKEN}" \ --source-tarball-uri \${SOURCE_TARBALL_URI} \ --release-repo-branch \${RELEASE_REPO_BRANCH} \ - --upload-to-repo \${UPLOAD_TO_REPO} > log + --upload-to-repo \${UPLOAD_TO_REPO} echo " - done" """.stripIndent()) } diff --git a/jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy b/jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy index 2f8f490bb..827d93238 100644 --- a/jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy +++ b/jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy @@ -57,7 +57,6 @@ class OSRFSourceCreation static void create(Job job, Map default_params = [:], Map default_hidden_params = [:]) { OSRFLinuxBuildPkgBase.create(job) - GenericRemoteToken.create(job) OSRFSourceCreation.addParameters(job, default_params) def pkg_sources_dir="pkgs" diff --git a/jenkins-scripts/dsl/brew_release.dsl b/jenkins-scripts/dsl/brew_release.dsl index 2265b156f..fd001c0e8 100644 --- a/jenkins-scripts/dsl/brew_release.dsl +++ b/jenkins-scripts/dsl/brew_release.dsl @@ -51,7 +51,7 @@ void include_common_params(Job job) // 1. BREW pull request SHA updater def release_job = job("generic-release-homebrew_pull_request_updater") OSRFUNIXBase.create(release_job) -GenericRemoteToken.create(release_job) +OSRFCredentials.allowOsrfbuildToRunTheBuild(release_job) include_common_params(release_job) release_job.with @@ -131,8 +131,6 @@ OSRFBrewCompilationAnyGitHub.create(bottle_job_builder, DISABLE_TESTS, NO_SUPPORTED_BRANCHES, DISABLE_GITHUB_INTEGRATION) -GenericRemoteToken.create(bottle_job_builder) - bottle_job_builder.with { wrappers { @@ -247,7 +245,6 @@ bottle_job_builder.with // 4. BREW bottle hash update def bottle_job_hash_updater = job(bottle_hash_updater_job_name) OSRFUNIXBase.create(bottle_job_hash_updater) -GenericRemoteToken.create(bottle_job_hash_updater) include_common_params(bottle_job_hash_updater) bottle_job_hash_updater.with diff --git a/jenkins-scripts/dsl/gazebo_ros_pkgs.dsl b/jenkins-scripts/dsl/gazebo_ros_pkgs.dsl index 0d795f954..34342e25e 100644 --- a/jenkins-scripts/dsl/gazebo_ros_pkgs.dsl +++ b/jenkins-scripts/dsl/gazebo_ros_pkgs.dsl @@ -260,7 +260,6 @@ bloom_debbuild_jobs.each { bloom_pkg -> // Use the linux install as base OSRFLinuxBuildPkgBase.create(build_pkg_job) - GenericRemoteToken.create(build_pkg_job) build_pkg_job.with { diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index f214e5a9b..dff60bc90 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -148,6 +148,7 @@ nightly_collection = gz_collections_yaml.collections def nightly_scheduler_job = job("ignition-${gz_nightly}-nightly-scheduler") OSRFUNIXBase.create(nightly_scheduler_job) +OSRFCredentials.setOSRFCrendentials(nightly_scheduler_job, ['OSRFBUILD_JENKINS_TOKEN']) nightly_scheduler_job.with { @@ -190,8 +191,6 @@ nightly_scheduler_job.with steps { shell("""\ #!/bin/bash -xe - set +x # keep password secret - PASS=\$(cat \$HOME/build_pass) dry_run_str="" if \$DRY_RUN; then @@ -240,7 +239,11 @@ nightly_scheduler_job.with fi echo "releasing \${n} (from branch \${src_branch})" - python3 ./scripts/release.py \${dry_run_str} "\${n}" nightly "\${PASS}" --release-repo-branch main --nightly-src-branch \${src_branch} --upload-to-repo nightly > log || echo "MARK_AS_UNSTABLE" + python3 ./scripts/release.py \${dry_run_str} "\${n}" nightly \ + --auth "\${OSRFBUILD_JENKINS_USER}:\${OSRFBUILD_JENKINS_TOKEN}" \ + --release-repo-branch main \ + --nightly-src-branch \${src_branch} \ + --upload-to-repo nightly echo " - done" done diff --git a/jenkins-scripts/dsl/ros_gz_bridge.dsl b/jenkins-scripts/dsl/ros_gz_bridge.dsl index b25e9d791..b8f110882 100644 --- a/jenkins-scripts/dsl/ros_gz_bridge.dsl +++ b/jenkins-scripts/dsl/ros_gz_bridge.dsl @@ -28,7 +28,6 @@ bridge_packages.each { pkg -> // Use the linux install as base OSRFLinuxBuildPkgBase.create(build_pkg_job) - GenericRemoteToken.create(build_pkg_job) build_pkg_job.with { diff --git a/jenkins-scripts/dsl/test.dsl b/jenkins-scripts/dsl/test.dsl index deecda1c2..7d6e7a546 100644 --- a/jenkins-scripts/dsl/test.dsl +++ b/jenkins-scripts/dsl/test.dsl @@ -153,12 +153,12 @@ test_credentials_token_job.with echo " + Testing OSRFBUILD_GITHUB_TOKEN ability to push into the fork osrfbuild/homebrew-simulation" echo " (out of the test is the ability to create pull requests into osrf/homebrew-simulation)" rm -fr homebrew-simulation - git clone https://github.com/\${OSRFBUILD_USER}/homebrew-simulation.git + git clone https://github.com/\${OSRFBUILD_GITHUB_USER}/homebrew-simulation.git cd homebrew-simulation - git config user.name \${OSRFBUILD_USER} --replace-all - git config user.email "\${OSRFBUILD_USER}@openrobotics.org" --replace-all + git config user.name \${OSRFBUILD_GITHUB_USER} --replace-all + git config user.email "\${OSRFBUILD_GITHUB_USER}@openrobotics.org" --replace-all set +x - git config url."https://osrfbuild:\${OSRFBUILD_TOKEN}@github.com/osrfbuild/homebrew-simulation.git".InsteadOf https://github.com/osrfbuild/homebrew-simulation.git + git config url."https://osrfbuild:\${OSRFBUILD_GITHUB_TOKEN}@github.com/osrfbuild/homebrew-simulation.git".InsteadOf https://github.com/osrfbuild/homebrew-simulation.git set -x GIT_TERMINAL_PROMPT=0 git push -u origin master --dry-run """.stripIndent()) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..03a665708 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,5 @@ +[build-system] +build-backend = 'setuptools.build_meta' +requires = [ + 'setuptools', +] diff --git a/release.py b/release.py index 88708c84b..f69038eae 100755 --- a/release.py +++ b/release.py @@ -2,22 +2,25 @@ from __future__ import print_function from argparse import RawTextHelpFormatter +from configparser import ConfigParser from typing import Tuple +from urllib3.exceptions import RequestError +from urllib3.util import make_headers import subprocess import sys import tempfile import os import urllib.parse -import urllib.request +import urllib3 import argparse import shutil import venv -USAGE = 'release.py ' +USAGE = 'release.py ' try: JENKINS_URL = os.environ['JENKINS_URL'] except KeyError: - JENKINS_URL = 'http://build.osrfoundation.org' + JENKINS_URL = 'https://build.osrfoundation.org' JOB_NAME_PATTERN = '%s-debbuilder' GENERIC_BREW_PULLREQUEST_JOB = 'generic-release-homebrew_pull_request_updater' @@ -111,21 +114,27 @@ def parse_args(argv): Script to handle the release process for the Gazebo devs. Examples: A) Generate source: local repository tag + call source job: - $ release.py + $ release.py (auto calculate source-repo-uri from local directory) B) Call builders: reuse existing tarball version + call build jobs: - $ release.py --source-tarball-uri + $ release.py --source-tarball-uri (no call to source job, directly build jobs with tarball URL) C) Nightly builds (linux) - $ release.py --source-repo-existing-ref --upload-to-repo nightly + $ release.py --source-repo-existing-ref --upload-to-repo nightly """) parser.add_argument('package', help='which package to release') parser.add_argument('version', help='which version to release') - parser.add_argument('jenkins_token', help='secret token to allow access to Jenkins to start builds') + parser.add_argument('deprecated_jenkins_token', + default=None, + nargs="?", + help=argparse.SUPPRESS) parser.add_argument('--dry-run', dest='dry_run', action='store_true', default=False, help='dry-run; i.e., do actually run any of the commands') + parser.add_argument('--auth', dest='auth_input_arg', + default=None, + help='Explicit jenkins user:token string overriding the jenkins.ini credentials file.') parser.add_argument('-a', '--package-alias', dest='package_alias', default=None, help='different name that we are releasing under') @@ -176,6 +185,44 @@ def parse_args(argv): return args +# +# BEGIN: Credentials code copied from ros_buildfarm +# +def get_credentials(jenkins_url=None): + config = ConfigParser() + config_file = get_credential_path() + if not os.path.exists(config_file): + print("Could not find credential file '%s'" % config_file, + file=sys.stderr) + return None, None + + config.read(config_file) + section_name = None + if jenkins_url is not None and jenkins_url in config: + section_name = jenkins_url + if section_name is None and 'DEFAULT' in config: + section_name = 'DEFAULT' + + if section_name is None or 'username' not in config[section_name] or \ + 'password' not in config[section_name]: + print( + "Could not find credentials for '%s' in file '%s'" % + (jenkins_url, config_file), file=sys.stderr) + return None, None + return config[section_name]['username'], config[section_name]['password'] + + +def get_credential_path(): + return os.path.join( + os.path.expanduser('~'), get_relative_credential_path()) + + +def get_relative_credential_path(): + return os.path.join('.buildfarm', 'jenkins.ini') + +# +# END: Credentials code copied from ros_buildfarm +# def get_release_repository_info(package): github_url = "https://github.com/gazebo-release/" + package + "-release" @@ -336,6 +383,8 @@ def sanity_checks(args, repo_dir): sanity_check_sdformat_versions(args.package, args.version) sanity_project_package_in_stable(args.version, args.upload_to_repository) + check_credentials(args.auth_input_arg) + print_success("Jenkins credentials are good") shutil.rmtree(repo_dir) @@ -486,9 +535,34 @@ def generate_source_params(args): return params - -def call_jenkins_build(job_name, params, output_string, - search_description_help): +def build_credentials_header(auth_input_arg = None): + if auth_input_arg: + if len(auth_input_arg.split(':')) != 2: + error("Auth string is not in the form of 'user:token' ") + username, api_token = auth_input_arg.split(':') + else: + username, api_token = get_credentials(JENKINS_URL) + if not username: + exit(1) + + return make_headers(basic_auth=f'{username}:{api_token}') + +def check_credentials(auth_input_arg = None): + http = urllib3.PoolManager() + response = http.request('GET', + JENKINS_URL, + headers=build_credentials_header(auth_input_arg)) + if response.status != 200: + print(f"Crendentials error: {response.status}: {response.reason}") + http.clear() + exit(1) + + +def call_jenkins_build(job_name, + params, + output_string, + search_description_help, + auth_input_arg = None): # Only to help user feedback this block help_url = f'{JENKINS_URL}/job/{job_name}' if search_description_help: @@ -502,9 +576,23 @@ def call_jenkins_build(job_name, params, output_string, job_name, params_query) print_only_dbg(f" -- {output_string}: {url}") - if not DRY_RUN: - urllib.request.urlopen(url) + if not DRY_RUN: + http = urllib3.PoolManager() + try : + response = http.request('POST', + url , + headers=build_credentials_header(auth_input_arg)) + # 201 code is "created", it is the expected return of POST + if response.status != 201: + print(f"Error {response.status}: {response.reason}") + exit(1) + except RequestError as e: + print(f"An error occurred in the http request: {e}") + except Exception as e: + print(f"An unexpected error occurred: {e}") + finally: + http.clear() def display_help_job_chain_for_source_calls(args): # Encode the different ways using in the job descriptions to filter builds @@ -686,6 +774,10 @@ def process_ros_vendor_package(args): def go(argv): args = parse_args(argv) + if args.deprecated_jenkins_token: + error('Build token has been removed. Please generate a user token:\n' + ' - https://gazebosim.org/docs/latest/releases-instructions/#access-and-credentials') + # If only the process of ROS vendor package is set, just do it if args.bump_ros_vendor_only: process_ros_vendor_package(args) @@ -707,7 +799,6 @@ def go(argv): sanity_checks(args, repo_dir) params = generate_source_params(args) - params['token'] = args.jenkins_token params['PACKAGE'] = args.package params['VERSION'] = args.version if not NIGHTLY else 'nightly' params['RELEASE_REPO_BRANCH'] = args.release_repo_branch diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 000000000..b7c6b864f --- /dev/null +++ b/setup.cfg @@ -0,0 +1,10 @@ +[metadata] +name = releasepy +version = 0.0.1 + +[options] +install_requires = + urllib3 >= 1.26.0 + argcomplete >= 1.8.0 + +packages = find: