Skip to content

Commit

Permalink
WIP Update CI
Browse files Browse the repository at this point in the history
Try to keep the workflows a bit closer to how we setup things in the
plugin repositories.
  • Loading branch information
mdellweg committed Dec 13, 2024
1 parent f1a4448 commit 80b401d
Show file tree
Hide file tree
Showing 8 changed files with 124 additions and 105 deletions.
91 changes: 46 additions & 45 deletions .ci/scripts/validate_commit_message.py
Original file line number Diff line number Diff line change
@@ -1,46 +1,54 @@
# WARNING: DO NOT EDIT!
#
# This file was generated by plugin_template, and is managed by it. Please use
# './plugin-template --github pulp_file' to update this file.
#
# For more info visit https://github.com/pulp/plugin_template

import os
import re
import requests
import subprocess
import sys
import tomllib
from pathlib import Path

KEYWORDS = ["fixes", "closes", "re", "ref"]
NO_ISSUE = "[noissue]"
STATUSES = ["NEW", "ASSIGNED", "POST", "MODIFIED"]
REDMINE_URL = "https://pulp.plan.io"
CHANGELOG_EXTS = [".feature", ".bugfix", ".doc", ".removal", ".misc", ".deprecation"]
from github import Github

with open("pyproject.toml", "rb") as fp:
PYPROJECT_TOML = tomllib.load(fp)
KEYWORDS = ["fixes", "closes"]
BLOCKING_REGEX = [
r"^DRAFT",
r"^WIP",
r"^NOMERGE",
r"^DO\s*NOT\s*MERGE",
r"^EXPERIMENT",
r"^FIXUP",
r"Apply suggestions from code review",
]
try:
CHANGELOG_EXTS = [
f".{item['directory']}" for item in PYPROJECT_TOML["tool"]["towncrier"]["type"]
]
except KeyError:
CHANGELOG_EXTS = [".feature", ".bugfix", ".doc", ".removal", ".misc"]
NOISSUE_MARKER = "[noissue]"

sha = sys.argv[1]
project = ""
message = subprocess.check_output(["git", "log", "--format=%B", "-n 1", sha]).decode("utf-8")

if NOISSUE_MARKER in message:
sys.exit(f"Do not add '{NOISSUE_MARKER}' in the commit message.")

if any((re.match(pattern, message, re.IGNORECASE) for pattern in BLOCKING_REGEX)):
sys.exit("This PR is not ready for consumption.")

def __check_status(issue):
response = requests.get(f"{REDMINE_URL}/issues/{issue}.json")
response.raise_for_status()
bug_json = response.json()
status = bug_json["issue"]["status"]["name"]
if status not in STATUSES:
sys.exit(
"Error: issue #{issue} has invalid status of {status}. Status must be one of "
"{statuses}.".format(issue=issue, status=status, statuses=", ".join(STATUSES))
)
g = Github(os.environ.get("GITHUB_TOKEN"))
repo = g.get_repo("pulp/pulp-openapi-generator")

if project:
project_id = bug_json["issue"]["project"]["id"]
project_json = requests.get(f"{REDMINE_URL}/projects/{project_id}.json").json()
if project_json["project"]["identifier"] != project:
sys.exit(f"Error: issue {issue} is not in the {project} project.")

def check_status(issue):
gi = repo.get_issue(int(issue))
if gi.pull_request:
sys.exit(f"Error: issue #{issue} is a pull request.")
if gi.closed_at:
sys.exit(f"Error: issue #{issue} is closed.")

def __check_changelog(issue):

def check_changelog(issue):
matches = list(Path("CHANGES").rglob(f"{issue}.*"))

if len(matches) < 1:
Expand All @@ -53,22 +61,15 @@ def __check_changelog(issue):
print("Checking commit message for {sha}.".format(sha=sha[0:7]))

# validate the issue attached to the commit
regex = r"(?:{keywords})[\s:]+#(\d+)".format(keywords=("|").join(KEYWORDS))
pattern = re.compile(regex, re.IGNORECASE)

issues = pattern.findall(message)
issue_regex = r"(?:{keywords})[\s:]+#(\d+)".format(keywords=("|").join(KEYWORDS))
issues = re.findall(issue_regex, message, re.IGNORECASE)
cherry_pick_regex = r"^\s*\(cherry picked from commit [0-9a-f]*\)\s*$"
cherry_pick = re.search(cherry_pick_regex, message, re.MULTILINE)

if issues:
for issue in pattern.findall(message):
__check_status(issue)
# __check_changelog(issue)
else:
if NO_ISSUE in message:
print("Commit {sha} has no issues but is tagged {tag}.".format(sha=sha[0:7], tag=NO_ISSUE))
else:
sys.exit(
"Error: no attached issues found for {sha}. If this was intentional, add "
" '{tag}' to the commit message.".format(sha=sha[0:7], tag=NO_ISSUE)
)
for issue in issues:
if not cherry_pick:
check_status(issue)
check_changelog(issue)

print("Commit message for {sha} passed.".format(sha=sha[0:7]))
78 changes: 51 additions & 27 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
@@ -1,56 +1,63 @@
---
name: pulp-openapi-generator PR CI
on: pull_request
name: "pulp-openapi-generator PR CI"
on:
pull_request:

jobs:
concurrency:
group: ${{ github.ref_name }}-${{ github.workflow }}
cancel-in-progress: true

lint:
runs-on: ubuntu-latest
defaults:
run:
working-directory: "pulp-openapi-generator"

jobs:
check-commits:
runs-on: "ubuntu-latest"
steps:
- uses: actions/checkout@v2
- uses: "actions/checkout@v4"
with:
# by default, it uses a depth of 1
# this fetches all history so that we can read each commit
fetch-depth: 0

- uses: actions/setup-python@v2
path: "pulp-openapi-generator"
- uses: "actions/setup-python@v5"
with:
python-version: "3.8"

- name: Check commit message
python-version: "3.11"
- name: "Install python dependencies"
run: |
pip install requests pygithub
- name: "Check commit message"
if: github.event_name == 'pull_request'
env:
GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }}
PY_COLORS: "1"
ANSIBLE_FORCE_COLOR: "1"
GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}"
run: |
pip install requests
sh .github/workflows/scripts/check_commit.sh
.github/workflows/scripts/check_commit.sh
docs:
uses: "./.github/workflows/docs-ci.yml"
uses: "./.github/workflows/docs.yml"

pulp:
runs-on: ubuntu-latest
test:
runs-on: "ubuntu-latest"
# run only after lint and docs finishes
needs:
- lint
- docs
- "docs"
strategy:
fail-fast: false
matrix:
env:
- TEST: pulp
- TEST: "pulp"

steps:
- uses: actions/checkout@v2
- uses: "actions/checkout@v4"
with:
# by default, it uses a depth of 1
# this fetches all history so that we can read each commit
fetch-depth: 0
path: "pulp-openapi-generator"

- uses: actions/setup-python@v2
- uses: "actions/setup-python@v5"
with:
python-version: "3.8"
python-version: "3.11"

- uses: ruby/setup-ruby@v1
with:
Expand Down Expand Up @@ -93,3 +100,20 @@ jobs:
docker logs pulp || true
docker exec pulp ls -latr /etc/yum.repos.d/ || true
docker exec pulp cat /etc/yum.repos.d/* || true
ready-to-ship:
# This is a dummy dependent task to have a single entry for the branch protection rules.
runs-on: "ubuntu-latest"
needs:
- "check-commits"
- "test"
- "docs"
if: "always()"
steps:
- name: "Collect needed jobs results"
working-directory: "."
run: |
echo '${{toJson(needs)}}' | jq -r 'to_entries[]|select(.value.result!="success")|.key + ": " + .value.result'
echo '${{toJson(needs)}}' | jq -e 'to_entries|map(select(.value.result!="success"))|length == 0'
echo "CI says: Looks good!"
...
5 changes: 3 additions & 2 deletions .github/workflows/docs-ci.yml → .github/workflows/docs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,5 +15,6 @@ jobs:
- name: "Install Test Dependencies"
run: |
pip install -r doc_requirements.txt
- name: Build docs
run: make docs
- name: "Build docs"
run: |
make docs
13 changes: 12 additions & 1 deletion .github/workflows/scripts/before_install.sh
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,20 @@
#
# For more info visit https://github.com/pulp/plugin_template

# make sure this script runs at the repo root
cd "$(dirname "$(realpath -e "$0")")"/../../..

set -mveuo pipefail

# Intall requirements for ansible playbooks
pip install docker netaddr boto3 ansible

ansible-galaxy collection install amazon.aws
for i in {1..3}
do
ansible-galaxy collection install "amazon.aws:8.1.0" && s=0 && break || s=$? && sleep 3
done
if [[ $s -gt 0 ]]
then
echo "Failed to install amazon.aws"
exit $s
fi
20 changes: 4 additions & 16 deletions .github/workflows/scripts/check_commit.sh
Original file line number Diff line number Diff line change
@@ -1,23 +1,11 @@
#!/bin/bash

# WARNING: DO NOT EDIT!
#
# This file was generated by plugin_template, and is managed by it. Please use
# './plugin-template --github pulp_file' to update this file.
#
# For more info visit https://github.com/pulp/plugin_template
# make sure this script runs at the repo root
cd "$(dirname "$(realpath -e "$0")")/../../.."

set -euv

echo ::group::REQUESTS
pip3 install requests
echo ::endgroup::

for sha in $(curl $GITHUB_CONTEXT | jq '.[].sha' | sed 's/"//g')
for SHA in $(curl -H "Authorization: token $GITHUB_TOKEN" "$GITHUB_CONTEXT" | jq -r '.[].sha')
do
python3 .ci/scripts/validate_commit_message.py $sha
VALUE=$?
if [ "$VALUE" -gt 0 ]; then
exit $VALUE
fi
python3 .ci/scripts/validate_commit_message.py "$SHA"
done
4 changes: 2 additions & 2 deletions .github/workflows/scripts/install.sh
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,11 @@
#
# For more info visit https://github.com/pulp/plugin_template

set -euv

# make sure this script runs at the repo root
cd "$(dirname "$(realpath -e "$0")")"/../../..

set -euv

TAG="${TAG:-latest}"

mkdir -p .ci/ansible/vars
Expand Down
3 changes: 1 addition & 2 deletions .github/workflows/scripts/test_bindings.py
Original file line number Diff line number Diff line change
Expand Up @@ -152,8 +152,7 @@ def upload_file_in_chunks(file_path):
pprint(repository_version_1)

# Create an artifact from a local file
file_path = os.path.join(os.environ["GITHUB_WORKSPACE"], ".github/workflows/scripts/test_bindings.py")
artifact = artifacts.create(file=file_path)
artifact = artifacts.create(file=__file__)
pprint(artifact)

# Create a FileContent from the artifact
Expand Down
15 changes: 5 additions & 10 deletions .github/workflows/scripts/test_bindings.rb
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,6 @@ def monitor_task(task_href)
#
# Returns:
# list[str]: List of hrefs that identify resource created by the task
completed = []
task = @tasks_api.read(task_href)
until ["completed", "failed", "canceled"].include? task.state
sleep(2)
Expand Down Expand Up @@ -86,8 +85,8 @@ def upload_file_in_chunks(file_path)
filechunk.write(chunk)
filechunk.flush()
actual_chunk_size = File.size(filechunk)
response = @uploads_api.update(content_range(offset, offset + actual_chunk_size -1, total_size), upload_href, filechunk)
offset += actual_chunk_size -1
response = @uploads_api.update(content_range(offset, offset + actual_chunk_size - 1, total_size), upload_href, filechunk)
offset += actual_chunk_size - 1
ensure
filechunk.close
filechunk.unlink
Expand All @@ -100,7 +99,7 @@ def upload_file_in_chunks(file_path)
end


artifact = upload_file_in_chunks(File.join(ENV['GITHUB_WORKSPACE'], 'README.rst'))
artifact = upload_file_in_chunks(File.new(File.expand_path(__FILE__)))

# Create a File Remote
remote_url = 'https://fixtures.pulpproject.org/file/PULP_MANIFEST'
Expand All @@ -120,12 +119,8 @@ def upload_file_in_chunks(file_path)

repository_version_1 = @repoversions_api.read(created_resources[0])

# Create an artifact from a local file
file_path = File.join(ENV['GITHUB_WORKSPACE'], '.github/workflows/scripts/test_bindings.rb')
artifact = @artifacts_api.create(File.new(file_path))

# Create a FileContent from the artifact
filecontent_response = @filecontent_api.create('foo.tar.gz', {artifact: artifact.pulp_href})
# Create a FileContent from a local file
filecontent_response = @filecontent_api.create('foo.tar.gz', {file: File.new(File.expand_path(__FILE__))})

created_resources = monitor_task(filecontent_response.task)

Expand Down

0 comments on commit 80b401d

Please sign in to comment.