From daf9e37c4c0853d1ee7a20e65289699498acbefe Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Tue, 19 Sep 2023 20:25:19 -0400 Subject: [PATCH] upgrade all CI workflows Signed-off-by: Alex Goodman --- .github/actions/bootstrap/action.yaml | 77 ++++++++++++++++++ .github/scripts/ci-check.sh | 11 +++ .github/scripts/trigger-release.sh | 50 ++++++++++++ .github/workflows/release.yaml | 74 +++++++----------- .github/workflows/validations.yaml | 108 ++++---------------------- .goreleaser.yaml | 6 +- Makefile | 61 +++++++++++---- 7 files changed, 231 insertions(+), 156 deletions(-) create mode 100644 .github/actions/bootstrap/action.yaml create mode 100755 .github/scripts/ci-check.sh create mode 100755 .github/scripts/trigger-release.sh diff --git a/.github/actions/bootstrap/action.yaml b/.github/actions/bootstrap/action.yaml new file mode 100644 index 0000000..00a9f17 --- /dev/null +++ b/.github/actions/bootstrap/action.yaml @@ -0,0 +1,77 @@ +name: "Bootstrap" + +description: "Bootstrap all tools and dependencies" +inputs: + go-version: + description: "Go version to install" + required: true + default: "1.21.x" + use-go-cache: + description: "Restore go cache" + required: true + default: "true" + cache-key-prefix: + description: "Prefix all cache keys with this value" + required: true + default: "831180ac25" + build-cache-key-prefix: + description: "Prefix build cache key with this value" + required: true + default: "f8b6d31dea" + bootstrap-apt-packages: + description: "Space delimited list of tools to install via apt" + default: "" + +runs: + using: "composite" + steps: + - uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe #v4.1.0 + with: + go-version: ${{ inputs.go-version }} + + - name: Restore tool cache + id: tool-cache + uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 #v3.3.2 + with: + path: ${{ github.workspace }}/.tmp + key: ${{ inputs.cache-key-prefix }}-${{ runner.os }}-tool-${{ hashFiles('Makefile') }} + + # note: we need to keep restoring the go mod cache before bootstrapping tools since `go install` is used in + # some installations of project tools. + - name: Restore go module cache + id: go-mod-cache + if: inputs.use-go-cache == 'true' + uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 #v3.3.2 + with: + path: | + ~/go/pkg/mod + key: ${{ inputs.cache-key-prefix }}-${{ runner.os }}-go-${{ inputs.go-version }}-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ inputs.cache-key-prefix }}-${{ runner.os }}-go-${{ inputs.go-version }}- + + - name: (cache-miss) Bootstrap project tools + shell: bash + if: steps.tool-cache.outputs.cache-hit != 'true' + run: make bootstrap-tools + + - name: Restore go build cache + id: go-cache + if: inputs.use-go-cache == 'true' + uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 #v3.3.2 + with: + path: | + ~/.cache/go-build + key: ${{ inputs.cache-key-prefix }}-${{ inputs.build-cache-key-prefix }}-${{ runner.os }}-go-${{ inputs.go-version }}-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ inputs.cache-key-prefix }}-${{ inputs.build-cache-key-prefix }}-${{ runner.os }}-go-${{ inputs.go-version }}- + + - name: (cache-miss) Bootstrap go dependencies + shell: bash + if: steps.go-mod-cache.outputs.cache-hit != 'true' && inputs.use-go-cache == 'true' + run: make bootstrap-go + + - name: Install apt packages + if: inputs.bootstrap-apt-packages != '' + shell: bash + run: | + DEBIAN_FRONTEND=noninteractive sudo apt update && sudo -E apt install -y ${{ inputs.bootstrap-apt-packages }} diff --git a/.github/scripts/ci-check.sh b/.github/scripts/ci-check.sh new file mode 100755 index 0000000..0ab83a3 --- /dev/null +++ b/.github/scripts/ci-check.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +red=$(tput setaf 1) +bold=$(tput bold) +normal=$(tput sgr0) + +# assert we are running in CI (or die!) +if [[ -z "$CI" ]]; then + echo "${bold}${red}This step should ONLY be run in CI. Exiting...${normal}" + exit 1 +fi diff --git a/.github/scripts/trigger-release.sh b/.github/scripts/trigger-release.sh new file mode 100755 index 0000000..c1a5432 --- /dev/null +++ b/.github/scripts/trigger-release.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +set -eu + +bold=$(tput bold) +normal=$(tput sgr0) + +if ! [ -x "$(command -v gh)" ]; then + echo "The GitHub CLI could not be found. To continue follow the instructions at https://github.com/cli/cli#installation" + exit 1 +fi + +gh auth status + +# we need all of the git state to determine the next version. Since tagging is done by +# the release pipeline it is possible to not have all of the tags from previous releases. +git fetch --tags + +# populates the CHANGELOG.md and VERSION files +echo "${bold}Generating changelog...${normal}" +make changelog 2> /dev/null + +NEXT_VERSION=$(cat VERSION) + +if [[ "$NEXT_VERSION" == "" || "${NEXT_VERSION}" == "(Unreleased)" ]]; then + echo "Could not determine the next version to release. Exiting..." + exit 1 +fi + +while true; do + read -p "${bold}Do you want to trigger a release for version '${NEXT_VERSION}'?${normal} [y/n] " yn + case $yn in + [Yy]* ) echo; break;; + [Nn]* ) echo; echo "Cancelling release..."; exit;; + * ) echo "Please answer yes or no.";; + esac +done + +echo "${bold}Kicking off release for ${NEXT_VERSION}${normal}..." +echo +gh workflow run release.yaml -f version=${NEXT_VERSION} + +echo +echo "${bold}Waiting for release to start...${normal}" +sleep 10 + +set +e + +echo "${bold}Head to the release workflow to monitor the release:${normal} $(gh run list --workflow=release.yaml --limit=1 --json url --jq '.[].url')" +id=$(gh run list --workflow=release.yaml --limit=1 --json databaseId --jq '.[].databaseId') +gh run watch $id --exit-status || (echo ; echo "${bold}Logs of failed step:${normal}" && GH_PAGER="" gh run view $id --log-failed) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index a3ded05..bad5240 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -1,29 +1,25 @@ -name: "Release" -on: - push: - # take no actions on push to any branch... - branches-ignore: - - "**" - # ... only act on release tags - tags: - - "v*" +permissions: + contents: read -env: - GO_VERSION: "1.21.x" +on: + workflow_dispatch: + inputs: + version: + description: tag the latest commit on main with the given version (prefixed with v) + required: true jobs: quality-gate: environment: release - runs-on: ubuntu-latest # This OS choice is arbitrary. None of the steps in this job are specific to either Linux or macOS. + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac #v4.0.0 - # we don't want to release commits that have been pushed and tagged, but not necessarily merged onto main - - name: Ensure tagged commit is on main + - name: Check if tag already exists + # note: this will fail if the tag already exists run: | - echo "Tag: ${GITHUB_REF##*/}" - git fetch origin main - git merge-base --is-ancestor ${GITHUB_REF##*/} origin/main && echo "${GITHUB_REF##*/} is a commit on main!" + [[ "${{ github.event.inputs.version }}" == v* ]] || (echo "version '${{ github.event.inputs.version }}' does not have a 'v' prefix" && exit 1) + git tag ${{ github.event.inputs.version }} - name: Check static analysis results uses: fountainhead/action-wait-for-check@297be350cf8393728ea4d4b39435c7d7ae167c93 #v1.1.0 @@ -52,40 +48,32 @@ jobs: release: needs: [quality-gate] - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 + permissions: + packages: write + contents: write steps: - - - uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe #v4.1.0 - with: - go-version: ${{ env.GO_VERSION }} - - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac #v4.0.0 with: fetch-depth: 0 - - name: Restore tool cache - id: tool-cache - uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 #v3.3.2 + - name: Bootstrap environment + uses: ./.github/actions/bootstrap with: - path: ${{ github.workspace }}/.tmp - key: ${{ runner.os }}-tool-${{ hashFiles('Makefile') }} + # use the same cache we used for building snapshots + build-cache-key-prefix: "snapshot" - - name: Restore go cache - id: go-cache - uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 #v3.3.2 - with: - path: ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go-${{ env.GO_VERSION }}- - - - name: (cache-miss) Bootstrap all project dependencies - if: steps.tool-cache.outputs.cache-hit != 'true' || steps.go-cache.outputs.cache-hit != 'true' - run: make bootstrap + - name: Tag release + run: | + git tag -a ${{ github.event.inputs.version }} -m "Release ${{ github.event.inputs.version }}" + git push origin --tags + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Build & publish release artifacts run: make release env: + # for creating the release (requires write access to packages and content) GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - uses: anchore/sbom-action@78fc58e266e87a38d4194b2137a3d4e9bcaf7ca1 #v0.14.3 @@ -96,12 +84,8 @@ jobs: with: status: ${{ job.status }} fields: repo,workflow,action,eventName - text: "A new Chronicle release is ready to be manually published: https://github.com/anchore/chronicle/releases" + text: "A new Chronicle release has been published: https://github.com/anchore/chronicle/releases" env: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_TOOLBOX_WEBHOOK_URL }} if: ${{ success() }} - - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 #v3.1.3 - with: - name: artifacts - path: dist/**/* diff --git a/.github/workflows/validations.yaml b/.github/workflows/validations.yaml index 3ac6e67..190f745 100644 --- a/.github/workflows/validations.yaml +++ b/.github/workflows/validations.yaml @@ -1,49 +1,22 @@ name: "Validations" on: + merge_group: workflow_dispatch: pull_request: push: branches: - main -env: - GO_VERSION: "1.21.x" - jobs: - Static-Analysis: # Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline name: "Static analysis" - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - - uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe #v4.1.0 - with: - go-version: ${{ env.GO_VERSION }} - - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac #v4.0.0 - - name: Restore tool cache - id: tool-cache - uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 #v3.3.2 - with: - path: ${{ github.workspace }}/.tmp - key: ${{ runner.os }}-tool-${{ hashFiles('Makefile') }} - - - name: Restore go cache - id: go-cache - uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 #v3.3.2 - with: - path: ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go-${{ env.GO_VERSION }}- - - - name: (cache-miss) Bootstrap all project dependencies - if: steps.tool-cache.outputs.cache-hit != 'true' || steps.go-cache.outputs.cache-hit != 'true' - run: make bootstrap - - - name: Bootstrap CI environment dependencies - run: make ci-bootstrap + - name: Bootstrap environment + uses: ./.github/actions/bootstrap - name: Run static analysis run: make static-analysis @@ -51,38 +24,15 @@ jobs: Unit-Test: # Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline name: "Unit tests" - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - - uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe #v4.1.0 - with: - go-version: ${{ env.GO_VERSION }} - - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac #v4.0.0 - - name: Restore tool cache - id: tool-cache - uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 #v3.3.2 - with: - path: ${{ github.workspace }}/.tmp - key: ${{ runner.os }}-tool-${{ hashFiles('Makefile') }} - - - name: Restore go cache - id: go-cache - uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 #v3.3.2 - with: - path: ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go-${{ env.GO_VERSION }}- - - - name: (cache-miss) Bootstrap all project dependencies - if: steps.tool-cache.outputs.cache-hit != 'true' || steps.go-cache.outputs.cache-hit != 'true' - run: make bootstrap + - name: Bootstrap environment + uses: ./.github/actions/bootstrap - - name: Bootstrap CI environment dependencies - run: make ci-bootstrap - - name: Build cache key for test-fixturew + - name: Build cache key for test-fixture run: make fixtures-fingerprint - name: Restore test-fixture cache @@ -95,45 +45,21 @@ jobs: - name: Run unit tests run: make unit - - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 #3.1.3 - with: - name: unit-test-results - path: test/results/**/* - Build-Snapshot-Artifacts: name: "Build snapshot artifacts" - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - - uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe #v4.1.0 - with: - go-version: ${{ env.GO_VERSION }} - - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac #v4.0.0 - - name: Restore tool cache - id: tool-cache - uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 #v3.3.2 - with: - path: ${{ github.workspace }}/.tmp - key: ${{ runner.os }}-tool-${{ hashFiles('Makefile') }} - - - name: Restore go cache - id: go-cache - uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 #v3.3.2 + - name: Bootstrap environment + uses: ./.github/actions/bootstrap with: - path: ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go-${{ env.GO_VERSION }}- - - - name: (cache-miss) Bootstrap all project dependencies - if: steps.tool-cache.outputs.cache-hit != 'true' || steps.go-cache.outputs.cache-hit != 'true' - run: make bootstrap + # why have another build cache key? We don't want unit/integration/etc test build caches to replace + # the snapshot build cache, which includes builds for all OSs and architectures. As long as this key is + # unique from the build-cache-key-prefix in other CI jobs, we should be fine. + # + # note: ideally this value should match what is used in release (just to help with build times). + build-cache-key-prefix: "snapshot" - name: Build snapshot artifacts run: make snapshot - - - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 #v3.1.3 - with: - name: artifacts - path: snapshot/**/* diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 0b09936..14bdfad 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -1,10 +1,8 @@ release: # If set to auto, will mark the release as not ready for production in case there is an indicator for this in the - # tag e.g. v1.0.0-rc1 .If set to true, will mark the release as not ready for production. + # tag (e.g. v1.0.0-rc1). If set to true, will mark the release as not ready for production. prerelease: auto - - # If set to true, will not auto-publish the release. This is done to allow us to review the changelog before publishing. - draft: true + draft: false builds: - id: chronicle diff --git a/Makefile b/Makefile index 23c9da4..b993a08 100644 --- a/Makefile +++ b/Makefile @@ -10,13 +10,14 @@ GOIMPORTS_CMD = $(TEMP_DIR)/gosimports -local github.com/anchore RELEASE_CMD = $(TEMP_DIR)/goreleaser release --rm-dist SNAPSHOT_CMD = $(RELEASE_CMD) --skip-publish --snapshot --skip-sign CHRONICLE_CMD = $(TEMP_DIR)/chronicle +GLOW_CMD = $(TEMP_DIR)/glow # Tool versions ################################# GOLANG_CI_VERSION = v1.54.2 GOBOUNCER_VERSION = v0.4.0 GORELEASER_VERSION = v1.17.0 GOSIMPORTS_VERSION = v0.3.8 -CHRONICLE_VERSION = latest +GLOW_VERSION := v1.5.1 # Formatting variables ################################# BOLD := $(shell tput -T linux bold) @@ -85,6 +86,9 @@ all: clean static-analysis test ## Run all linux-based checks .PHONY: test test: unit ## Run all tests + +## Bootstrapping targets ################################# + .PHONY: ci-bootstrap ci-bootstrap: DEBIAN_FRONTEND=noninteractive sudo apt update && sudo -E apt install -y bc jq libxml2-utils @@ -101,10 +105,12 @@ bootstrap-tools: $(TEMP_DIR) curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(TEMP_DIR)/ $(GOLANG_CI_VERSION) curl -sSfL https://raw.githubusercontent.com/wagoodman/go-bouncer/master/bouncer.sh | sh -s -- -b $(TEMP_DIR)/ $(GOBOUNCER_VERSION) # we purposefully use the latest version of chronicle released - curl -sSfL https://raw.githubusercontent.com/anchore/chronicle/main/install.sh | sh -s -- -b $(TEMP_DIR)/ $(CHRONICLE_VERSION) + #curl -sSfL https://raw.githubusercontent.com/anchore/chronicle/main/install.sh | sh -s -- -b $(TEMP_DIR)/ $(CHRONICLE_VERSION) + GOBIN="$(realpath $(TEMP_DIR))" go install ./cmd/chronicle GOBIN="$(realpath $(TEMP_DIR))" go install github.com/goreleaser/goreleaser@$(GORELEASER_VERSION) # the only difference between goimports and gosimports is that gosimports removes extra whitespace between import blocks (see https://github.com/golang/go/issues/20818) GOBIN="$(realpath $(TEMP_DIR))" go install github.com/rinchsan/gosimports/cmd/gosimports@$(GOSIMPORTS_VERSION) + GOBIN="$(realpath $(TEMP_DIR))" go install github.com/charmbracelet/glow@$(GLOW_VERSION) .PHONY: bootstrap-go bootstrap-go: @@ -114,6 +120,9 @@ bootstrap-go: bootstrap: $(RESULTS_DIR) bootstrap-go bootstrap-tools ## Download and install all go dependencies (+ prep tooling in the ./tmp dir) $(call title,Bootstrapping dependencies) + +## Static analysis targets ################################# + .PHONY: static-analysis static-analysis: lint check-go-mod-tidy check-licenses @@ -151,6 +160,9 @@ check-licenses: check-go-mod-tidy: @ .github/scripts/go-mod-tidy-check.sh && echo "go.mod and go.sum are tidy!" + +## Testing targets ################################# + .PHONY: unit unit: $(RESULTS_DIR) fixtures ## Run unit tests (with coverage) $(call title,Running unit tests) @@ -160,6 +172,8 @@ unit: $(RESULTS_DIR) fixtures ## Run unit tests (with coverage) @if [ $$(echo "$$(cat $(COVER_TOTAL)) >= $(COVERAGE_THRESHOLD)" | bc -l) -ne 1 ]; then echo "$(RED)$(BOLD)Failed coverage quality gate (> $(COVERAGE_THRESHOLD)%)$(RESET)" && false; fi +## Test-fixture-related targets ################################# + .PHONY: fixtures fixtures: $(call title,Generating test fixtures) @@ -169,6 +183,9 @@ fixtures: fixtures-fingerprint: find internal/git/test-fixtures/*.sh -type f -exec md5sum {} + | awk '{print $1}' | sort | md5sum | tee internal/git/test-fixtures/cache.fingerprint && echo "$(FIXTURE_CACHE_BUSTER)" >> internal/git/test-fixtures/cache.fingerprint + +## Build-related targets ################################# + .PHONY: build build: $(SNAPSHOT_DIR) ## Build release snapshot binaries and packages @@ -183,31 +200,40 @@ $(SNAPSHOT_DIR): ## Build snapshot release binaries and packages $(TEMP_DIR)/goreleaser build --snapshot --skip-validate --rm-dist --config $(TEMP_DIR)/goreleaser.yaml .PHONY: changelog -changelog: clean-changelog $(CHANGELOG) - @docker run -it --rm \ - -v $(shell pwd)/CHANGELOG.md:/CHANGELOG.md \ - rawkode/mdv \ - -t 748.5989 \ - /CHANGELOG.md +changelog: clean-changelog ## Generate and show the changelog for the current unreleased version + $(CHRONICLE_CMD) -vvv -n --version-file VERSION > $(CHANGELOG) + @$(GLOW_CMD) $(CHANGELOG) $(CHANGELOG): - $(TEMP_DIR)/chronicle > $(CHANGELOG) + $(CHRONICLE_CMD) -vvv > $(CHANGELOG) .PHONY: release -release: clean-dist $(CHANGELOG) ## Build and publish final binaries and packages. +release: + @.github/scripts/trigger-release.sh + +.PHONY: ci-release +ci-release: ci-check clean-dist $(CHANGELOG) $(call title,Publishing release artifacts) # create a config with the dist dir overridden echo "dist: $(DIST_DIR)" > $(TEMP_DIR)/goreleaser.yaml cat .goreleaser.yaml >> $(TEMP_DIR)/goreleaser.yaml - # release (note the version transformation from v0.7.0 --> 0.7.0) bash -c "\ - VERSION=$(VERSION:v%=%) \ - $(TEMP_DIR)/goreleaser \ - --rm-dist \ - --config $(TEMP_DIR)/goreleaser.yaml \ - --release-notes <(cat $(CHANGELOG))" + $(RELEASE_CMD) \ + --config $(TEMP_DIR)/goreleaser.yaml \ + --release-notes <(cat $(CHANGELOG)) \ + || (cat /tmp/quill-*.log && false)" + + # upload the version file that supports the application version update check (excluding pre-releases) + .github/scripts/update-version-file.sh "$(DIST_DIR)" "$(VERSION)" + + +## Cleanup targets ################################# + +.PHONY: ci-check +ci-check: + @.github/scripts/ci-check.sh .PHONY: clean clean: clean-dist clean-snapshot ## Remove previous builds, result reports, and test cache @@ -225,6 +251,9 @@ clean-dist: clean-changelog clean-changelog: rm -f $(CHANGELOG) + +## Halp! ################################# + .PHONY: help help: @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "$(BOLD)$(CYAN)%-25s$(RESET)%s\n", $$1, $$2}'