From 80b401da012ced7cc35ae474eafe46a7f591fe7c Mon Sep 17 00:00:00 2001 From: Matthias Dellweg Date: Tue, 3 Dec 2024 15:41:23 +0100 Subject: [PATCH] WIP Update CI Try to keep the workflows a bit closer to how we setup things in the plugin repositories. --- .ci/scripts/validate_commit_message.py | 91 +++++++++++---------- .github/workflows/ci.yml | 78 ++++++++++++------ .github/workflows/{docs-ci.yml => docs.yml} | 5 +- .github/workflows/scripts/before_install.sh | 13 ++- .github/workflows/scripts/check_commit.sh | 20 +---- .github/workflows/scripts/install.sh | 4 +- .github/workflows/scripts/test_bindings.py | 3 +- .github/workflows/scripts/test_bindings.rb | 15 ++-- 8 files changed, 124 insertions(+), 105 deletions(-) rename .github/workflows/{docs-ci.yml => docs.yml} (84%) diff --git a/.ci/scripts/validate_commit_message.py b/.ci/scripts/validate_commit_message.py index 181729c..7930faf 100644 --- a/.ci/scripts/validate_commit_message.py +++ b/.ci/scripts/validate_commit_message.py @@ -1,46 +1,54 @@ -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_file' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - +import os import re -import requests import subprocess import sys +import tomllib from pathlib import Path -KEYWORDS = ["fixes", "closes", "re", "ref"] -NO_ISSUE = "[noissue]" -STATUSES = ["NEW", "ASSIGNED", "POST", "MODIFIED"] -REDMINE_URL = "https://pulp.plan.io" -CHANGELOG_EXTS = [".feature", ".bugfix", ".doc", ".removal", ".misc", ".deprecation"] +from github import Github + +with open("pyproject.toml", "rb") as fp: + PYPROJECT_TOML = tomllib.load(fp) +KEYWORDS = ["fixes", "closes"] +BLOCKING_REGEX = [ + r"^DRAFT", + r"^WIP", + r"^NOMERGE", + r"^DO\s*NOT\s*MERGE", + r"^EXPERIMENT", + r"^FIXUP", + r"Apply suggestions from code review", +] +try: + CHANGELOG_EXTS = [ + f".{item['directory']}" for item in PYPROJECT_TOML["tool"]["towncrier"]["type"] + ] +except KeyError: + CHANGELOG_EXTS = [".feature", ".bugfix", ".doc", ".removal", ".misc"] +NOISSUE_MARKER = "[noissue]" sha = sys.argv[1] -project = "" message = subprocess.check_output(["git", "log", "--format=%B", "-n 1", sha]).decode("utf-8") +if NOISSUE_MARKER in message: + sys.exit(f"Do not add '{NOISSUE_MARKER}' in the commit message.") + +if any((re.match(pattern, message, re.IGNORECASE) for pattern in BLOCKING_REGEX)): + sys.exit("This PR is not ready for consumption.") -def __check_status(issue): - response = requests.get(f"{REDMINE_URL}/issues/{issue}.json") - response.raise_for_status() - bug_json = response.json() - status = bug_json["issue"]["status"]["name"] - if status not in STATUSES: - sys.exit( - "Error: issue #{issue} has invalid status of {status}. Status must be one of " - "{statuses}.".format(issue=issue, status=status, statuses=", ".join(STATUSES)) - ) +g = Github(os.environ.get("GITHUB_TOKEN")) +repo = g.get_repo("pulp/pulp-openapi-generator") - if project: - project_id = bug_json["issue"]["project"]["id"] - project_json = requests.get(f"{REDMINE_URL}/projects/{project_id}.json").json() - if project_json["project"]["identifier"] != project: - sys.exit(f"Error: issue {issue} is not in the {project} project.") +def check_status(issue): + gi = repo.get_issue(int(issue)) + if gi.pull_request: + sys.exit(f"Error: issue #{issue} is a pull request.") + if gi.closed_at: + sys.exit(f"Error: issue #{issue} is closed.") -def __check_changelog(issue): + +def check_changelog(issue): matches = list(Path("CHANGES").rglob(f"{issue}.*")) if len(matches) < 1: @@ -53,22 +61,15 @@ def __check_changelog(issue): print("Checking commit message for {sha}.".format(sha=sha[0:7])) # validate the issue attached to the commit -regex = r"(?:{keywords})[\s:]+#(\d+)".format(keywords=("|").join(KEYWORDS)) -pattern = re.compile(regex, re.IGNORECASE) - -issues = pattern.findall(message) +issue_regex = r"(?:{keywords})[\s:]+#(\d+)".format(keywords=("|").join(KEYWORDS)) +issues = re.findall(issue_regex, message, re.IGNORECASE) +cherry_pick_regex = r"^\s*\(cherry picked from commit [0-9a-f]*\)\s*$" +cherry_pick = re.search(cherry_pick_regex, message, re.MULTILINE) if issues: - for issue in pattern.findall(message): - __check_status(issue) - # __check_changelog(issue) -else: - if NO_ISSUE in message: - print("Commit {sha} has no issues but is tagged {tag}.".format(sha=sha[0:7], tag=NO_ISSUE)) - else: - sys.exit( - "Error: no attached issues found for {sha}. If this was intentional, add " - " '{tag}' to the commit message.".format(sha=sha[0:7], tag=NO_ISSUE) - ) + for issue in issues: + if not cherry_pick: + check_status(issue) + check_changelog(issue) print("Commit message for {sha} passed.".format(sha=sha[0:7])) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0c9cc2b..2ec700d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,56 +1,63 @@ --- -name: pulp-openapi-generator PR CI -on: pull_request +name: "pulp-openapi-generator PR CI" +on: + pull_request: -jobs: +concurrency: + group: ${{ github.ref_name }}-${{ github.workflow }} + cancel-in-progress: true - lint: - runs-on: ubuntu-latest +defaults: + run: + working-directory: "pulp-openapi-generator" +jobs: + check-commits: + runs-on: "ubuntu-latest" steps: - - uses: actions/checkout@v2 + - uses: "actions/checkout@v4" with: - # by default, it uses a depth of 1 - # this fetches all history so that we can read each commit fetch-depth: 0 - - - uses: actions/setup-python@v2 + path: "pulp-openapi-generator" + - uses: "actions/setup-python@v5" with: - python-version: "3.8" - - - name: Check commit message + python-version: "3.11" + - name: "Install python dependencies" + run: | + pip install requests pygithub + - name: "Check commit message" if: github.event_name == 'pull_request' env: - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} + PY_COLORS: "1" + ANSIBLE_FORCE_COLOR: "1" + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" run: | - pip install requests - sh .github/workflows/scripts/check_commit.sh + .github/workflows/scripts/check_commit.sh docs: - uses: "./.github/workflows/docs-ci.yml" + uses: "./.github/workflows/docs.yml" - pulp: - runs-on: ubuntu-latest + test: + runs-on: "ubuntu-latest" # run only after lint and docs finishes needs: - - lint - - docs + - "docs" strategy: fail-fast: false matrix: env: - - TEST: pulp + - TEST: "pulp" steps: - - uses: actions/checkout@v2 + - uses: "actions/checkout@v4" with: - # by default, it uses a depth of 1 - # this fetches all history so that we can read each commit fetch-depth: 0 + path: "pulp-openapi-generator" - - uses: actions/setup-python@v2 + - uses: "actions/setup-python@v5" with: - python-version: "3.8" + python-version: "3.11" - uses: ruby/setup-ruby@v1 with: @@ -93,3 +100,20 @@ jobs: docker logs pulp || true docker exec pulp ls -latr /etc/yum.repos.d/ || true docker exec pulp cat /etc/yum.repos.d/* || true + + ready-to-ship: + # This is a dummy dependent task to have a single entry for the branch protection rules. + runs-on: "ubuntu-latest" + needs: + - "check-commits" + - "test" + - "docs" + if: "always()" + steps: + - name: "Collect needed jobs results" + working-directory: "." + run: | + echo '${{toJson(needs)}}' | jq -r 'to_entries[]|select(.value.result!="success")|.key + ": " + .value.result' + echo '${{toJson(needs)}}' | jq -e 'to_entries|map(select(.value.result!="success"))|length == 0' + echo "CI says: Looks good!" +... diff --git a/.github/workflows/docs-ci.yml b/.github/workflows/docs.yml similarity index 84% rename from .github/workflows/docs-ci.yml rename to .github/workflows/docs.yml index ccd6215..3f466b2 100644 --- a/.github/workflows/docs-ci.yml +++ b/.github/workflows/docs.yml @@ -15,5 +15,6 @@ jobs: - name: "Install Test Dependencies" run: | pip install -r doc_requirements.txt - - name: Build docs - run: make docs + - name: "Build docs" + run: | + make docs diff --git a/.github/workflows/scripts/before_install.sh b/.github/workflows/scripts/before_install.sh index 45adf6a..c360e1d 100755 --- a/.github/workflows/scripts/before_install.sh +++ b/.github/workflows/scripts/before_install.sh @@ -7,9 +7,20 @@ # # For more info visit https://github.com/pulp/plugin_template +# make sure this script runs at the repo root +cd "$(dirname "$(realpath -e "$0")")"/../../.. + set -mveuo pipefail # Intall requirements for ansible playbooks pip install docker netaddr boto3 ansible -ansible-galaxy collection install amazon.aws +for i in {1..3} +do + ansible-galaxy collection install "amazon.aws:8.1.0" && s=0 && break || s=$? && sleep 3 +done +if [[ $s -gt 0 ]] +then + echo "Failed to install amazon.aws" + exit $s +fi diff --git a/.github/workflows/scripts/check_commit.sh b/.github/workflows/scripts/check_commit.sh index 3d36c59..62fe94b 100755 --- a/.github/workflows/scripts/check_commit.sh +++ b/.github/workflows/scripts/check_commit.sh @@ -1,23 +1,11 @@ #!/bin/bash -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_file' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template +# make sure this script runs at the repo root +cd "$(dirname "$(realpath -e "$0")")/../../.." set -euv -echo ::group::REQUESTS -pip3 install requests -echo ::endgroup:: - -for sha in $(curl $GITHUB_CONTEXT | jq '.[].sha' | sed 's/"//g') +for SHA in $(curl -H "Authorization: token $GITHUB_TOKEN" "$GITHUB_CONTEXT" | jq -r '.[].sha') do - python3 .ci/scripts/validate_commit_message.py $sha - VALUE=$? - if [ "$VALUE" -gt 0 ]; then - exit $VALUE - fi + python3 .ci/scripts/validate_commit_message.py "$SHA" done diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh index 8b49505..ec8d4bc 100755 --- a/.github/workflows/scripts/install.sh +++ b/.github/workflows/scripts/install.sh @@ -7,11 +7,11 @@ # # For more info visit https://github.com/pulp/plugin_template -set -euv - # make sure this script runs at the repo root cd "$(dirname "$(realpath -e "$0")")"/../../.. +set -euv + TAG="${TAG:-latest}" mkdir -p .ci/ansible/vars diff --git a/.github/workflows/scripts/test_bindings.py b/.github/workflows/scripts/test_bindings.py index 621eebc..2bb0697 100644 --- a/.github/workflows/scripts/test_bindings.py +++ b/.github/workflows/scripts/test_bindings.py @@ -152,8 +152,7 @@ def upload_file_in_chunks(file_path): pprint(repository_version_1) # Create an artifact from a local file -file_path = os.path.join(os.environ["GITHUB_WORKSPACE"], ".github/workflows/scripts/test_bindings.py") -artifact = artifacts.create(file=file_path) +artifact = artifacts.create(file=__file__) pprint(artifact) # Create a FileContent from the artifact diff --git a/.github/workflows/scripts/test_bindings.rb b/.github/workflows/scripts/test_bindings.rb index c69fd68..d0b9b74 100644 --- a/.github/workflows/scripts/test_bindings.rb +++ b/.github/workflows/scripts/test_bindings.rb @@ -40,7 +40,6 @@ def monitor_task(task_href) # # Returns: # list[str]: List of hrefs that identify resource created by the task - completed = [] task = @tasks_api.read(task_href) until ["completed", "failed", "canceled"].include? task.state sleep(2) @@ -86,8 +85,8 @@ def upload_file_in_chunks(file_path) filechunk.write(chunk) filechunk.flush() actual_chunk_size = File.size(filechunk) - response = @uploads_api.update(content_range(offset, offset + actual_chunk_size -1, total_size), upload_href, filechunk) - offset += actual_chunk_size -1 + response = @uploads_api.update(content_range(offset, offset + actual_chunk_size - 1, total_size), upload_href, filechunk) + offset += actual_chunk_size - 1 ensure filechunk.close filechunk.unlink @@ -100,7 +99,7 @@ def upload_file_in_chunks(file_path) end -artifact = upload_file_in_chunks(File.join(ENV['GITHUB_WORKSPACE'], 'README.rst')) +artifact = upload_file_in_chunks(File.new(File.expand_path(__FILE__))) # Create a File Remote remote_url = 'https://fixtures.pulpproject.org/file/PULP_MANIFEST' @@ -120,12 +119,8 @@ def upload_file_in_chunks(file_path) repository_version_1 = @repoversions_api.read(created_resources[0]) -# Create an artifact from a local file -file_path = File.join(ENV['GITHUB_WORKSPACE'], '.github/workflows/scripts/test_bindings.rb') -artifact = @artifacts_api.create(File.new(file_path)) - -# Create a FileContent from the artifact -filecontent_response = @filecontent_api.create('foo.tar.gz', {artifact: artifact.pulp_href}) +# Create a FileContent from a local file +filecontent_response = @filecontent_api.create('foo.tar.gz', {file: File.new(File.expand_path(__FILE__))}) created_resources = monitor_task(filecontent_response.task)