From 15ec753b513db4ca9e388a66c23e8b1d16253e58 Mon Sep 17 00:00:00 2001 From: Agnes Garoux Date: Fri, 25 Oct 2024 14:23:49 +0100 Subject: [PATCH] format and release: out with BK, in with GHA --- .buildkite/pipeline.yml | 24 -- .github/workflows/release.yml | 81 +++++++ .github/workflows/tf_formatting.yml | 39 ++++ Dockerfile | 31 --- docker-compose.yml | 14 -- gha_scripts/create_release.py | 99 ++++++++ scripts/tooling.py | 338 ---------------------------- 7 files changed, 219 insertions(+), 407 deletions(-) delete mode 100644 .buildkite/pipeline.yml create mode 100644 .github/workflows/release.yml create mode 100644 .github/workflows/tf_formatting.yml delete mode 100644 Dockerfile delete mode 100644 docker-compose.yml create mode 100644 gha_scripts/create_release.py delete mode 100755 scripts/tooling.py diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml deleted file mode 100644 index 62c04e2..0000000 --- a/.buildkite/pipeline.yml +++ /dev/null @@ -1,24 +0,0 @@ -- label: "autoformat" - plugins: - - docker-compose#v3.5.0: - run: tooling - command: ["autoformat"] - env: - - BUILDKITE_BRANCH - - BUILDKITE_REPO - agents: - queue: nano - -- wait - -- label: "release" - if: build.branch == "main" - plugins: - - docker-compose#v3.5.0: - run: tooling - command: ["release"] - env: - - BUILDKITE_BRANCH - - BUILDKITE_REPO - agents: - queue: nano diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..6995338 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,81 @@ +name: "Release new version of the module" + +on: + push: + branches: + - main + +permissions: + id-token: write + contents: write + +jobs: + autoformat: + uses: ./.github/workflows/tf_formatting.yml + + # First, check if there is a RELEASE.md file in the root of the repository. + # If not, no release will be created and subsequent steps and jobs will be skipped. + check-for-release-file: + runs-on: ubuntu-latest + outputs: + has-release: ${{ steps.check-for-release-file.outputs.has-release }} + needs: autoformat + steps: + - uses: actions/checkout@v4 + - name: Check for RELEASE.md file + id: check-for-release-file + run: | + if [ ! -f ./RELEASE.md ]; then + echo "has-release=false" >> $GITHUB_OUTPUT + echo "No release detected. Exiting." + exit 0 + fi + echo "has-release=true" >> $GITHUB_OUTPUT + + create-release: + runs-on: ubuntu-latest + outputs: + new-version: ${{ steps.create-release.outputs.new-version }} + needs: check-for-release-file + if: needs.check-for-release-file.outputs.has-release == 'true' + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Update CHANGELOG.md and version + id: create-release + run: | + git fetch --tags + LATEST_TAG=$(git describe --tags $(git rev-list --tags --max-count=1)) + python3 gha_scripts/create_release.py ${LATEST_TAG} $(pwd) + + VERSION_TAG="$(cat CHANGELOG.md | grep -m1 -o 'v[0-9]\+\.[0-9]\+\.[0-9]\+')" + echo "new-version=${VERSION_TAG:1}" >> $GITHUB_OUTPUT + + - name: Generate a token + id: generate-token + uses: actions/create-github-app-token@v1 + with: + app-id: 129326 # App ID of the Wellcome Collection app + private-key: ${{ secrets.WELLCOME_COLLECTION_APP_PRIVATE_KEY }} + + # We need to give the GitHub action full repo privileges so that it can push the release directly into main + - name: Configure git + run: | + git config --global user.name "GitHub on behalf of Wellcome Collection" + git config --global user.email "wellcomedigitalplatform@wellcome.ac.uk" + git remote set-url origin https://x-access-token:${{ steps.generate-token.outputs.token }}@github.com/${{ github.repository }}.git + + - name: Commit and push changes + run: | + git checkout main + git pull + git add CHANGELOG.md + git rm RELEASE.md + + NEW_TAG="v${{ steps.create-release.outputs.new-version }}" + git commit -m "$(printf "Bump version to ${NEW_TAG}\n\n[skip ci]")" + git tag ${NEW_TAG} + + git push origin main + git push origin --tags \ No newline at end of file diff --git a/.github/workflows/tf_formatting.yml b/.github/workflows/tf_formatting.yml new file mode 100644 index 0000000..617a305 --- /dev/null +++ b/.github/workflows/tf_formatting.yml @@ -0,0 +1,39 @@ +# Runs auto-formatting script on push to any branch +name: "Run terraform formatting" + +on: + push: + branches-ignore: + - main + workflow_call: + +permissions: + id-token: write + contents: write + +jobs: + autoformat: + name: autoformat + runs-on: ubuntu-latest + steps: + - name: Check out project + uses: actions/checkout@v4 + + - name: terraform format (recursive) + uses: dflook/terraform-fmt@v1 + + - name: Check for formatting changes + id: check_formatting_changes + run: | + if [[ -n $(git status --porcelain) ]]; then + echo "changes=true" >> "$GITHUB_OUTPUT"; + fi + + - name: Commit and push formatting changes + if: steps.check_formatting_changes.outputs.changes == 'true' + run: | + git config user.name "Github on behalf of Wellcome Collection" + git config user.email "wellcomedigitalplatform@wellcome.ac.uk" + git commit -am "Apply auto-formatting rules" + git push + diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index c54292c..0000000 --- a/Dockerfile +++ /dev/null @@ -1,31 +0,0 @@ -FROM python:3 - -ARG COMMAND -VOLUME /workdir -WORKDIR /workdir - -ADD . /workdir - -# These instructions are taken from the Terraform docs for installing -# the CLI, retrieved 23 June 2023 -# -# See https://developer.hashicorp.com/terraform/tutorials/aws-get-started/install-cli -RUN apt-get update && apt-get install -y gnupg software-properties-common -RUN wget -O- https://apt.releases.hashicorp.com/gpg | \ - gpg --dearmor | \ - tee /usr/share/keyrings/hashicorp-archive-keyring.gpg -RUN echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] \ - https://apt.releases.hashicorp.com $(lsb_release -cs) main" | \ - tee /etc/apt/sources.list.d/hashicorp.list -RUN apt update && apt-get install -y terraform - -# This is to avoid warnings like: -# -# fatal: detected dubious ownership in repository at '/workdir' -# To add an exception for this directory, call: -# -# git config --global --add safe.directory /workdir -# -RUN git config --global --add safe.directory /workdir - -ENTRYPOINT ["/workdir/scripts/tooling.py"] \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 330ac69..0000000 --- a/docker-compose.yml +++ /dev/null @@ -1,14 +0,0 @@ -version: "3.3" -services: - tooling: - build: - context: . - volumes: - - .:/workdir - - ${HOME}/.ssh:/root/.ssh - - ${SSH_AUTH_SOCK}:/ssh-agent.sock - environment: - - SSH_AUTH_SOCK=/ssh-agent.sock - - BUILDKITE_BRANCH=buildkite - - BUILDKITE_REPO=git@github.com:wellcomecollection/terraform-aws-ecs-service.git - command: ["autoformat"] \ No newline at end of file diff --git a/gha_scripts/create_release.py b/gha_scripts/create_release.py new file mode 100644 index 0000000..b2111e5 --- /dev/null +++ b/gha_scripts/create_release.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python + +import datetime as dt +import os +import re +import sys + +RELEASE_TYPE = re.compile(r"^RELEASE_TYPE: +(major|minor|patch)") +VALID_RELEASE_TYPES = ('major', 'minor', 'patch') + +CHANGELOG_HEADER = re.compile(r"^## v\d+\.\d+\.\d+ - \d\d\d\d-\d\d-\d\d$") + + +def get_new_version_tag(latest_version: str, release_type: str): + version_info = [int(i) for i in latest_version.lstrip('v').split('.')] + + bump = VALID_RELEASE_TYPES.index(release_type) + version_info[bump] += 1 + for i in range(bump + 1, len(version_info)): + version_info[i] = 0 + return 'v' + '.'.join(map(str, version_info)) + + +def _get_changelog_version_heading(new_version_tag: str): + date = dt.datetime.utcnow().strftime('%Y-%m-%d') + return f'## {new_version_tag} - {date}' + + +def update_changelog(release_contents: str, new_version_tag: str): + with open(CHANGELOG_FILE) as f: + changelog_contents = f.read() + + assert '\r' not in changelog_contents + lines = changelog_contents.split('\n') + assert changelog_contents == '\n'.join(lines) + + for i, line in enumerate(lines): + if CHANGELOG_HEADER.match(line): + beginning = '\n'.join(lines[:i]) + rest = '\n'.join(lines[i:]) + assert '\n'.join((beginning, rest)) == changelog_contents + break + + new_version_heading = _get_changelog_version_heading(new_version_tag) + + new_changelog_parts = [ + beginning.strip(), + new_version_heading, + release_contents, + rest + ] + + with open(CHANGELOG_FILE, 'w') as f: + f.write('\n\n'.join(new_changelog_parts)) + +def parse_release_file(): + """ + Parses the release file, returning a tuple (release_type, release_contents) + """ + with open(RELEASE_FILE) as i: + release_contents = i.read() + + release_lines = release_contents.split('\n') + + m = RELEASE_TYPE.match(release_lines[0]) + if m is not None: + release_type = m.group(1) + if release_type not in VALID_RELEASE_TYPES: + print('Unrecognised release type %r' % (release_type,)) + sys.exit(1) + del release_lines[0] + release_contents = '\n'.join(release_lines).strip() + else: + print( + 'RELEASE.md does not start by specifying release type. The first ' + 'line of the file should be RELEASE_TYPE: followed by one of ' + 'major, minor, or patch, to specify the type of release that ' + 'this is (i.e. which version number to increment). Instead the ' + 'first line was %r' % (release_lines[0],) + ) + sys.exit(1) + + return release_type, release_contents + + +def create_release(): + latest_version_tag = sys.argv[1] + + release_type, release_contents = parse_release_file() + new_version_tag = get_new_version_tag(latest_version_tag, release_type) + + update_changelog(release_contents, new_version_tag) + +if __name__ == '__main__': + ROOT = sys.argv[2] + CHANGELOG_FILE = os.path.join(ROOT, 'CHANGELOG.md') + RELEASE_FILE = os.path.join(ROOT, 'RELEASE.md') + + create_release() \ No newline at end of file diff --git a/scripts/tooling.py b/scripts/tooling.py deleted file mode 100755 index 8959858..0000000 --- a/scripts/tooling.py +++ /dev/null @@ -1,338 +0,0 @@ -#!/usr/bin/env python3 - -# coding=utf-8 -# -# This file is derived from Hypothesis, which may be found at -# https://github.com/HypothesisWorks/hypothesis -# -# Specifically, it is a modified version of code in the tooling directory: -# https://github.com/HypothesisWorks/hypothesis/blob/54966e04d972800db558d76b076f9119e05e977d/tooling/src/hypothesistooling -# -# Most of this work is copyright (C) 2013-2017 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. -# -# This Source Code Form is subject to the terms of the Mozilla Public License, -# v. 2.0. If a copy of the MPL was not distributed with this file, You can -# obtain one at http://mozilla.org/MPL/2.0/. -# -# END HEADER - -from __future__ import division, print_function, absolute_import - -import os -import re -import sys -import subprocess -from datetime import datetime, timedelta - - -GIT_BRANCH = os.environ["BUILDKITE_BRANCH"] -REPO_URL = os.environ["BUILDKITE_REPO"] - - -def git(*args): - return subprocess.check_output(("git",) + args).decode("utf8").strip() - - -def setup_git(): - git('config', 'user.name', 'Buildkite on behalf of Wellcome Collection') - git('config', 'user.email', 'wellcomedigitalplatform@wellcome.ac.uk') - - try: - git('remote', 'add', 'ssh-origin', REPO_URL) - except subprocess.CalledProcessError: - print("Could not add ssh-origin (maybe already exists?)") - - -def tags(): - """ - Returns a list of all tags in the repo. - """ - git("fetch", "--tags") - all_tags = git("tag").splitlines() - - assert len(set(all_tags)) == len(all_tags) - - return set(all_tags) - - -def latest_version(): - """ - Returns the latest version, as specified by the Git tags. - """ - versions = [] - - for t in tags(): - assert t == t.strip() - parts = t.split(".") - assert len(parts) == 3, t - parts[0] = parts[0].lstrip("v") - v = tuple(map(int, parts)) - - versions.append((v, t)) - - _, latest = max(versions) - - assert latest in tags() - return latest - - -__version__ = latest_version() -__version_info__ = [int(i) for i in __version__.lstrip("v").split(".")] - - -ROOT = ( - subprocess.check_output(["git", "rev-parse", "--show-toplevel"]) - .decode("ascii") - .strip() -) - - -def hash_for_name(name): - return subprocess.check_output(["git", "rev-parse", name]).decode("ascii").strip() - - -def is_ancestor(a, b): - check = subprocess.call(["git", "merge-base", "--is-ancestor", a, b]) - assert 0 <= check <= 1 - return check == 0 - - -CHANGELOG_FILE = os.path.join(ROOT, "CHANGELOG.md") - - -def changelog(): - with open(CHANGELOG_FILE) as i: - return i.read() - - -def has_source_changes(version=None): - if version is None: - version = latest_version() - - tf_files = [f for f in modified_files() if f.strip().endswith(".tf")] - return len(tf_files) != 0 - - -def create_tag_and_push(): - assert __version__ not in tags() - - setup_git() - - git('tag', __version__) - - subprocess.check_call(['git', 'push', 'ssh-origin', 'HEAD:main']) - subprocess.check_call(['git', 'push', 'ssh-origin', '--tags']) - - -def modified_files(): - files = set() - for command in [ - ["git", "diff", "--name-only", "--diff-filter=d", latest_version(), "HEAD"], - ["git", "diff", "--name-only"], - ]: - diff_output = subprocess.check_output(command).decode("ascii") - for l in diff_output.split("\n"): - filepath = l.strip() - if filepath: - assert os.path.exists(filepath) - files.add(filepath) - return files - - -RELEASE_FILE = os.path.join(ROOT, "RELEASE.md") - - -def has_release(): - return os.path.exists(RELEASE_FILE) - - -CHANGELOG_HEADER = re.compile(r"^## v\d+\.\d+\.\d+ - \d\d\d\d-\d\d-\d\d$") -RELEASE_TYPE = re.compile(r"^RELEASE_TYPE: +(major|minor|patch)") - - -MAJOR = "major" -MINOR = "minor" -PATCH = "patch" - -VALID_RELEASE_TYPES = (MAJOR, MINOR, PATCH) - - -def parse_release_file(): - with open(RELEASE_FILE) as i: - release_contents = i.read() - - release_lines = release_contents.split("\n") - - m = RELEASE_TYPE.match(release_lines[0]) - if m is not None: - release_type = m.group(1) - if release_type not in VALID_RELEASE_TYPES: - print("Unrecognised release type %r" % (release_type,)) - sys.exit(1) - del release_lines[0] - release_contents = "\n".join(release_lines).strip() - else: - print( - "RELEASE.md does not start by specifying release type. The first " - "line of the file should be RELEASE_TYPE: followed by one of " - "major, minor, or patch, to specify the type of release that " - "this is (i.e. which version number to increment). Instead the " - "first line was %r" % (release_lines[0],) - ) - sys.exit(1) - - return release_type, release_contents - - -def update_changelog_and_version(): - global __version_info__ - global __version__ - - with open(CHANGELOG_FILE) as i: - contents = i.read() - assert "\r" not in contents - lines = contents.split("\n") - assert contents == "\n".join(lines) - for i, l in enumerate(lines): - if CHANGELOG_HEADER.match(l): - beginning = "\n".join(lines[:i]) - rest = "\n".join(lines[i:]) - assert "\n".join((beginning, rest)) == contents - break - - release_type, release_contents = parse_release_file() - - new_version = list(__version_info__) - bump = VALID_RELEASE_TYPES.index(release_type) - new_version[bump] += 1 - for i in range(bump + 1, len(new_version)): - new_version[i] = 0 - new_version = tuple(new_version) - new_version_string = "v" + ".".join(map(str, new_version)) - - now = datetime.utcnow() - - date = max([d.strftime("%Y-%m-%d") for d in (now, now + timedelta(hours=1))]) - - heading_for_new_version = "## " + " - ".join((new_version_string, date)) - - new_changelog_parts = [ - beginning.strip(), - "", - heading_for_new_version, - "", - release_contents, - "", - rest, - ] - - __version__ = new_version_string - __version_info__ = [int(i) for i in __version__.lstrip("v").split(".")] - - with open(CHANGELOG_FILE, "w") as o: - o.write("\n".join(new_changelog_parts)) - - -def update_for_pending_release(): - git("config", "user.name", "Buildkite on behalf of Wellcome") - git("config", "user.email", "wellcomedigitalplatform@wellcome.ac.uk") - - update_changelog_and_version() - - git("rm", RELEASE_FILE) - git("add", CHANGELOG_FILE) - - git("commit", "-m", "Bump version to %s and update changelog\n\n[skip ci]" % (__version__,)) - - -def changed_files(*args): - """ - Returns a set of changed files in a given commit range. - - :param commit_range: Arguments to pass to ``git diff``. - """ - files = set() - command = ["git", "diff", "--name-only"] + list(args) - diff_output = subprocess.check_output(command).decode("ascii") - for line in diff_output.splitlines(): - filepath = line.strip() - if filepath: - files.add(filepath) - return files - - -def autoformat(): - subprocess.check_call(["terraform", "fmt", "-recursive"]) - - if changed_files(): - print("*** There were changes from formatting, creating a commit", flush=True) - - setup_git() - - git("add", "--verbose", "--all") - git("commit", "-m", "Apply auto-formatting rules") - git("push", "ssh-origin", "HEAD:%s" % GIT_BRANCH) - - sys.exit(1) - else: - print("*** There were no changes from auto-formatting", flush=True) - - -def check_release_file(): - if has_source_changes(): - if not has_release(): - print( - "There are source changes but no RELEASE.md. Please create " - "one to describe your changes." - ) - sys.exit(1) - parse_release_file() - - -def release(): - last_release = latest_version() - - print( - "Latest version: %s" % last_release - ) - - HEAD = hash_for_name("HEAD") - MAIN = hash_for_name("origin/main") - print("Current head:", HEAD) - print("Current main:", MAIN) - - on_main = is_ancestor(HEAD, MAIN) - - if has_release(): - print("Updating changelog and version") - update_for_pending_release() - else: - print("Not deploying due to no release") - sys.exit(0) - - if not on_main: - print("Not deploying due to not being on main") - sys.exit(0) - - print("Release seems good. Pushing to GitHub now.") - - create_tag_and_push() - - -if __name__ == "__main__": - try: - command = sys.argv[1] - except IndexError: - sys.exit("Usage: %s " % __file__) - - if command == "autoformat": - check_release_file() - autoformat() - elif command == "release": - release() - else: - sys.exit("Unrecognised command: %s" % command)