Add support to create PR against ROS gz_*_vendor repositories in release.py

check_releasepy.bash

@@ -1,9 +1,22 @@
 #!/bin/bash -e
 
-test_dir=$(mktemp -d)
-mkdir -p ${test_dir}/{focal,jammy,ubuntu}/debian
-export _RELEASEPY_TEST_RELEASE_REPO=${test_dir}
 export _RELEASEPY_DEBUG=1
+test_dir=$(mktemp -d)
+export _RELEASEPY_TEST_RELEASE_REPO="${test_dir}/test-release"
+mkdir -p ${_RELEASEPY_TEST_RELEASE_REPO}/{focal,jammy,ubuntu}/debian
+export _RELEASEPY_TEST_SOURCE_REPO="${test_dir}/src"
+mkdir -p ${_RELEASEPY_TEST_SOURCE_REPO}
+# Fake packages.xml to make the vendor package script happy
+cat > "${_RELEASEPY_TEST_SOURCE_REPO}/package.xml" <<-EOF
+<?xml version="1.0"?>
+<package format="2">
+  <name>gz-foo</name>
+  <version>0.0.0</version>
+  <description>test</description>
+  <maintainer email="[email protected]">Testing maintainer</maintainer>
+  <license>Foo License</license>
+</package>
+EOF
 
 exec_releasepy_test()
 {
@@ -12,7 +25,7 @@ exec_releasepy_test()
     ./release.py \
       --dry-run \
       --no-sanity-checks \
-    gz-foo 1.2.3 token ${test_params}""
+    gz-foo 1.2.3 token ${test_params}
 }
 
 exec_ignition_releasepy_test()
@@ -22,7 +35,7 @@ exec_ignition_releasepy_test()
     ./release.py \
       --dry-run \
       --no-sanity-checks \
-    ign-foo 1.2.3 token ${test_params}""
+    ign-foo 1.2.3 token ${test_params}
 }
 
 exec_ignition_gazebo_releasepy_test()
@@ -32,7 +45,18 @@ exec_ignition_gazebo_releasepy_test()
     ./release.py \
       --dry-run \
       --no-sanity-checks \
-    ign-gazebo 1.2.3 token ${test_params}""
+    ign-gazebo 1.2.3 token ${test_params}
+}
+
+exec_releasepy_with_real_gz()
+{
+  gz_pkg=${1} major_version=${2}
+    ./release.py \
+      --dry-run \
+      --no-sanity-checks \
+      --source-repo-uri http://github.com/gazebosim/gz-common \
+      --source-repo-existing-ref http://github.com/gazebosim/gz-common/foo-tag \
+    "${gz_pkg}" "${major_version}.x.y" token
 }
 
 expect_job_run()
@@ -73,35 +97,58 @@ expect_param()
     echo "${param} not found in test output"
     exit 1
   fi
+}
+
+expect_vendor_repo()
+{
+  output="${1}" repo="${2}"
+
+  if ! grep -q "Github ${repo}" <<< "${output}"; then
+    echo "${repo} not found in test output"
+    exit 1
+  fi
+}
+
+expect_no_vendor()
+{
+  output="${1}"
 
+  if grep -q 'in ROS 2' <<< "${output}"; then
+    echo "ROS 2 string found in output"
+    exit 1
+  fi
 }
 
 source_repo_uri_test=$(exec_releasepy_test "--source-repo-uri https://github.com/gazebosim/gz-foo.git")
 expect_job_run "${source_repo_uri_test}" "gz-foo-source"
 expect_job_not_run "${source_repo_uri_test}" "gz-foo-debbuilder"
 expect_number_of_jobs "${source_repo_uri_test}" "1"
 expect_param "${source_repo_uri_test}" "SOURCE_REPO_URI=https%3A%2F%2Fgithub.com%2Fgazebosim%2Fgz-foo.git"
+expect_no_vendor "${source_repo_uri_test}"  # non existing package
 
 source_tarball_uri_test=$(exec_releasepy_test "--source-tarball-uri https://gazebosim/gz-foo-1.2.3.tar.gz")
 expect_job_run "${source_tarball_uri_test}" "gz-foo-debbuilder"
 expect_job_run "${source_tarball_uri_test}" "generic-release-homebrew_pull_request_updater"
 expect_job_not_run "${source_tarball_uri_test}" "gz-foo-source"
 expect_number_of_jobs "${source_tarball_uri_test}" "7"
 expect_param "${source_tarball_uri_test}" "SOURCE_TARBALL_URI=https%3A%2F%2Fgazebosim%2Fgz-foo-1.2.3.tar.gz"
+expect_no_vendor "${source_tarball_uri_test}"
 
 nightly_test=$(exec_releasepy_test "--nightly-src-branch my-nightly-branch3 --upload-to-repo nightly")
 expect_job_run "${nightly_test}" "gz-foo-debbuilder"
 expect_job_not_run "${nightly_test}" "generic-release-homebrew_pull_request_updater"
 expect_job_not_run "${nightly_test}" "gz-foo-source"
 expect_number_of_jobs "${nightly_test}" "2"
 expect_param "${nightly_test}" "SOURCE_TARBALL_URI=my-nightly-branch3"
+expect_no_vendor "${nightly_test}"
 
 bump_linux_test=$(exec_releasepy_test "--source-tarball-uri https://gazebosim/gz-foo-1.2.3.tar.gz --only-bump-revision-linux -r 2")
 expect_job_run "${bump_linux_test}" "gz-foo-debbuilder"
 expect_job_not_run "${bump_linux_test}" "generic-release-homebrew_pull_request_updater"
 expect_job_not_run "${bump_linux_test}" "gz-foo-source"
 expect_number_of_jobs "${bump_linux_test}" "6"
 expect_param "${bump_linux_test}" "RELEASE_VERSION=2"
+expect_no_vendor "${bump_linux_test}"
 
 ignition_test=$(exec_ignition_releasepy_test "--source-repo-uri https://github.com/gazebosim/gz-foo.git")
 expect_job_run "${ignition_test}" "gz-foo-source"
@@ -128,3 +175,12 @@ expect_number_of_jobs "${ign_gazebo_source_tarball_uri_test}" "7"
 expect_param "${ign_gazebo_source_tarball_uri_test}" "SOURCE_TARBALL_URI=https%3A%2F%2Fgazebosim%2Fign-gazebo-1.2.3.tar.gz"
 expect_param "${ign_gazebo_source_tarball_uri_test}" "PACKAGE=ign-gazebo"
 expect_param "${ign_gazebo_source_tarball_uri_test}" "PACKAGE_ALIAS=ignition-gazebo"
+
+ros_vendor_test=$(exec_releasepy_with_real_gz gz-fuel-tools 9)
+expect_vendor_repo "${ros_vendor_test}" gazebo-release/gz_fuel_tools_vendor
+
+ros_vendor_test=$(exec_releasepy_with_real_gz gz-cmake 2)
+expect_no_vendor "${ros_vendor_test}"
+
+ros_vendor_test=$(exec_releasepy_with_real_gz gz-ionic 3)
+expect_no_vendor "${ros_vendor_test}"

release.py

@@ -2,6 +2,7 @@
 
 from __future__ import print_function
 from argparse import RawTextHelpFormatter
+from typing import Tuple
 import subprocess
 import sys
 import tempfile
@@ -10,6 +11,7 @@
 import urllib.request
 import argparse
 import shutil
+import venv
 
 USAGE = 'release.py <package> <version> <jenkinstoken>'
 try:
@@ -22,6 +24,7 @@
 LINUX_DISTROS = ['ubuntu', 'debian']
 SUPPORTED_ARCHS = ['amd64', 'armhf', 'arm64']
 RELEASEPY_NO_ARCH_PREFIX = '.releasepy_NO_ARCH_'
+ROS_VENDOR = {'harmonic': 'jazzy', 'ionic': 'rolling'}
 
 OSRF_REPOS_SUPPORTED = "stable prerelease nightly testing none"
 
@@ -48,6 +51,10 @@ class ErrorNoOutput(Exception):
     pass
 
 
+class ErrorAlreadyExists(Exception):
+    pass
+
+
 def error(msg):
     print("\n !! " + msg + "\n")
     sys.exit(1)
@@ -151,6 +158,9 @@ def parse_args(argv):
     parser.add_argument('--only-bump-revision-linux', dest='bump_rev_linux_only',
                         action='store_true', default=False,
                         help='Bump only revision number. Do not upload new tarball.')
+    parser.add_argument('--only-bump-ros-vendor-package', dest='bump_ros_vendor_only',
+                        action='store_true', default=False,
+                        help='Only process the ROS vendor package (if any).')
 
     args = parser.parse_args()
 
@@ -375,13 +385,13 @@ def discover_distros(repo_dir):
     return distro_arch_list
 
 
-def check_call(cmd, ignore_dry_run=False):
+def check_call(cmd, ignore_dry_run=False, cwd=None):
     if DRY_RUN and not ignore_dry_run:
         print_only_dbg('Dry-run running:\n  %s\n' % (' '.join(cmd)))
         return b'', b''
     else:
         print_only_dbg('Running:\n  %s' % (' '.join(cmd)))
-        po = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        po = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd)
         out, err = po.communicate()
         if po.returncode != 0:
             # bitbucket for the first one, github for the second
@@ -391,6 +401,8 @@ def check_call(cmd, ignore_dry_run=False):
                 raise ErrorNoPermsRepo()
             if b"abort: no username supplied" in err:
                 raise ErrorNoUsernameSupplied()
+            if b"already exists:" in err:
+                raise ErrorAlreadyExists()
             if not out and not err:
                 # assume that call is only for getting return code
                 raise ErrorNoOutput()
@@ -515,9 +527,170 @@ def display_help_job_chain_for_source_calls(args):
           f'{releasepy_check_url}')
 
 
+def get_collections_for_package(package_name, version) -> list:
+    script_directory = os.path.dirname(os.path.abspath(sys.argv[0]))
+    helper_script = f'{script_directory}/jenkins-scripts/dsl/tools/get_collections_from_package_and_version.py'
+    collection_yaml = f'{script_directory}/jenkins-scripts/dsl/gz-collections.yaml'
+    cmd = [helper_script,
+           get_canonical_package_name(package_name),
+           version,
+           collection_yaml]
+    try:
+        _out, _err = check_call(cmd, IGNORE_DRY_RUN)
+    except ErrorNoOutput:
+        # no output is a valid result
+        _out = b""
+        _err = ""
+    else:
+        if _err:
+            print(f"An error happened running get_collections_from_package_and_version: {_err}")
+            sys.exit(1)
+
+    collection_list = _out.decode().strip().split(' ')
+    return collection_list
+
+
+def get_vendor_github_repo(package_name) -> str:
+    canonical_name = get_canonical_package_name(package_name)
+    return f"gazebo-release/{canonical_name.replace('-', '_')}_vendor"
+
+
+def get_vendor_repo_url(package_name) -> str:
+    # Clone needs ssh for real pushing operations. In simulation prefer https to avoid
+    # unexpected pushes and facilitate testing
+    protocol = 'https://github.com/' if DRY_RUN else 'ssh://[email protected]:'
+    return f"{protocol}{get_vendor_github_repo(package_name)}"
+
+
+def prepare_vendor_pr_temp_workspace(package_name, ws_dir) -> Tuple[str, str, str]:
+    gz_vendor_tool = os.path.join(ws_dir, "gz_vendor")
+    # Create virtualenv for vendor dependencies
+    venv_dir = os.path.join(ws_dir, "venv")
+    venv.create(venv_dir, system_site_packages=True, with_pip=True)
+    subprocess.run([os.path.join(venv_dir, 'bin', 'pip3'), 'install', '-q',
+                    'jinja2==3.1.2',
+                    'catkin_pkg==1.0.0'])
+    cmd = ['git', 'clone', '-q',
+           'https://github.com/gazebo-tooling/gz_vendor/',
+           gz_vendor_tool]
+    _, _err_tool = check_call(cmd, IGNORE_DRY_RUN)
+    gz_vendor_repo = os.path.join(ws_dir, 'gz_vendor_repo')
+    cmd = ['git', 'clone', '-q',
+           get_vendor_repo_url(package_name),
+           gz_vendor_repo]
+    _, _err_repo = check_call(cmd, IGNORE_DRY_RUN)
+    if _err_tool or _err_repo:
+        print("Problems with cloning vendor and tool repos:")
+        print(f"{_err_tool} {_err_repo}")
+        sys.exit(1)
+
+    return gz_vendor_tool, gz_vendor_repo, venv_dir
+
+
+def execute_update_vendor_package_tool(vendor_tool_path,
+                                       vendor_repo_path,
+                                       vendor_venv) -> None:
+    # The source repository when releasing matches the
+    src_repo = os.getcwd()
+    try:
+        src_repo = os.environ['_RELEASEPY_TEST_SOURCE_REPO']
+    except KeyError:
+        pass
+
+    run_cmd = [os.path.join(vendor_venv, 'bin', 'python3'),
+               f"{vendor_tool_path}/create_gz_vendor_pkg/create_vendor_package.py",
+               f"{os.path.join(src_repo, 'package.xml')}",
+               '--output_dir', vendor_repo_path]
+    _, _err_run = check_call(run_cmd, IGNORE_DRY_RUN)
+    if _err_run:
+        print("Problems running the create_vendor_package.py script:")
+        print(_err_run.decode())
+        sys.exit(1)
+
+
+def create_pr_for_vendor_package(args, repo_path, base_branch) -> str:
+    cmd_diff = ['git', "-C", repo_path, 'diff']
+    _out, _ = check_call(cmd_diff, IGNORE_DRY_RUN)
+    if not _out.decode():
+        return 'vendor tool did not produce any change, avoid the PR'
+
+    branch_name = f'releasepy/{args.version}'
+    vendor_repo = get_vendor_repo_url(args.package)
+    branch_cmd = ['git', "-C", repo_path,
+                  'checkout', '-b',  branch_name]
+    _, _ = check_call(branch_cmd, IGNORE_DRY_RUN)
+    commit_cmd = ['git', "-C", repo_path,
+                  'commit',
+                  '-m', f'Bump version to {args.version}',
+                  '--all']
+    _, _ = check_call(commit_cmd)
+    push_cmd = ['git', "-C", repo_path,
+                'push', '--force',
+                vendor_repo, branch_name]
+    _, _ = check_call(push_cmd)
+    pr_cmd = ['gh', 'pr', 'create',
+              '--base', base_branch,
+              '--head', branch_name,
+              '--title', f'Bump version to {args.version}',
+              '--body', 'PR automatically created by release.py']
+    try:
+        _out, _err = check_call(pr_cmd, cwd=repo_path)
+    except ErrorAlreadyExists:
+        return f'there is already a PR for the branch: {branch_name} .'\
+                'Please check it out manuallly.'
+
+    if _err:
+        print("Problems creating the PR for the vendor package:")
+        print(_err.decode())
+        sys.exit(1)
+
+    if DRY_RUN:
+        return ' (skipped the creation on --dry-run)'
+
+    return _out.decode()
+
+
+def create_pr_in_gz_vendor_repo(args, ros_distro) -> str:
+    pr_msg = ''
+    with tempfile.TemporaryDirectory() as ws_dir:
+        ws_dir = tempfile.mkdtemp()
+        # Prepare the temporary workspace
+        vendor_tool_path, vendor_repo_path, venv_dir = \
+            prepare_vendor_pr_temp_workspace(args.package, ws_dir)
+        # Run updating script on the temporary workspace
+        execute_update_vendor_package_tool(
+            vendor_tool_path, vendor_repo_path, venv_dir)
+        # Commits and PR creation
+        pr_msg = create_pr_for_vendor_package(
+            args, vendor_repo_path, ros_distro)
+
+    return pr_msg
+
+
+def process_ros_vendor_package(args):
+    print("ROS vendor packages that can be updated:")
+    if  args.package.replace('gz-','') in ROS_VENDOR:
+        print(" - There are no gz metapackages in ROS")
+        return
+    for collection in get_collections_for_package(args.package,
+                                                  args.version):
+        if collection in ROS_VENDOR:
+            ros_distro = ROS_VENDOR[collection]
+            print(f" * Github {get_vendor_github_repo(args.package)} "
+                  f"part of {collection} in ROS 2 {ros_distro}")
+            print("   + Preparing a PR: ", end='', flush=True)
+            pr_url = create_pr_in_gz_vendor_repo(args, ros_distro)
+            print(pr_url)
+
+
 def go(argv):
     args = parse_args(argv)
 
+    # If only the process of ROS vendor package is set, just do it
+    if args.bump_ros_vendor_only:
+        process_ros_vendor_package(args)
+        sys.exit(0)
+
     # Default to release 1 if not present
     if not args.release_version:
         args.release_version = 1
@@ -640,7 +813,8 @@ def go(argv):
                            'Source',
                            args.version)
         display_help_job_chain_for_source_calls(args)
-
+        # Process the possible update of an associated ROS vendor package
+        process_ros_vendor_package(args)
 
 if __name__ == '__main__':
     go(sys.argv)