From c4a2e061b49846457eb9d04816c648c6720dfa04 Mon Sep 17 00:00:00 2001 From: Gregory Haddow Date: Wed, 8 Mar 2023 11:00:18 +0000 Subject: [PATCH] feat: Initial commit. --- .circleci/config.yml | 35 ++++++++++++++++ .circleci/test-deploy.yml | 31 ++++++++++++++ .gitignore | 3 ++ .yamllint | 7 ++++ LICENSE | 21 ++++++++++ README.md | 35 ++++++++++++++++ src/@orb.yml | 8 ++++ src/README.md | 26 ++++++++++++ src/commands/pass-fast.yml | 57 +++++++++++++++++++++++++ src/examples/basic-pass-fast.yml | 24 +++++++++++ src/examples/parallel-pass-fast.yml | 25 +++++++++++ src/scripts/pass-fast.sh | 64 +++++++++++++++++++++++++++++ 12 files changed, 336 insertions(+) create mode 100644 .circleci/config.yml create mode 100644 .circleci/test-deploy.yml create mode 100644 .gitignore create mode 100644 .yamllint create mode 100644 LICENSE create mode 100644 README.md create mode 100755 src/@orb.yml create mode 100644 src/README.md create mode 100755 src/commands/pass-fast.yml create mode 100755 src/examples/basic-pass-fast.yml create mode 100755 src/examples/parallel-pass-fast.yml create mode 100644 src/scripts/pass-fast.sh diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000..75fed76 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,35 @@ +version: 2.1 +setup: true +orbs: + orb-tools: circleci/orb-tools@11.5 + shellcheck: circleci/shellcheck@3.1 + +filters: &filters + tags: + only: /.*/ + +workflows: + lint-pack: + jobs: + - orb-tools/lint: + filters: *filters + - orb-tools/pack: + filters: *filters + - orb-tools/review: + filters: *filters + - shellcheck/check: + filters: *filters + - orb-tools/publish: + orb-name: ankorstore/toolbelt + vcs-type: << pipeline.project.type >> + requires: + [orb-tools/lint, orb-tools/review, orb-tools/pack, shellcheck/check] + # Use a context to hold your publishing token. + context: orb-publishing + filters: *filters + # Triggers the next workflow in the Orb Development Kit. + - orb-tools/continue: + pipeline-number: << pipeline.number >> + vcs-type: << pipeline.project.type >> + requires: [orb-tools/publish] + filters: *filters diff --git a/.circleci/test-deploy.yml b/.circleci/test-deploy.yml new file mode 100644 index 0000000..9ed3fca --- /dev/null +++ b/.circleci/test-deploy.yml @@ -0,0 +1,31 @@ +version: 2.1 +orbs: + toolbelt: ankorstore/toolbelt@dev:<> + orb-tools: circleci/orb-tools@11.5 + +filters: &filters + tags: + only: /.*/ + +workflows: + test-deploy: + jobs: + # Make sure to include "filters: *filters" in every test job you want to run as part of your deployment. + - command-tests: + filters: *filters + - orb-tools/pack: + filters: *filters + - orb-tools/publish: + orb-name: ankorstore/toolbelt + vcs-type: << pipeline.project.type >> + pub-type: production + requires: + - orb-tools/pack + context: orb-publishing + filters: + branches: + ignore: /.*/ + tags: + only: /^v[0-9]+\.[0-9]+\.[0-9]+$/ + +# VS Code Extension Version: 1.4.0 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3c88c1b --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +# orb.yml is "packed" from source, and not published directly from the repository. +orb.yml +.DS_Store \ No newline at end of file diff --git a/.yamllint b/.yamllint new file mode 100644 index 0000000..c9a8a2e --- /dev/null +++ b/.yamllint @@ -0,0 +1,7 @@ +extends: relaxed + +rules: + line-length: + max: 200 + allow-non-breakable-inline-mappings: true + diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..b32d28e --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2022 + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..5174a52 --- /dev/null +++ b/README.md @@ -0,0 +1,35 @@ +# Toolbelt + + +[![CircleCI Build Status](https://circleci.com/gh/ankorstore/orb-toolbelt.svg?style=shield "CircleCI Build Status")](https://circleci.com/gh/ankorstore/orb-toolbelt) [![CircleCI Orb Version](https://badges.circleci.com/orbs/ankorstore/toolbelt.svg)](https://circleci.com/orbs/registry/orb/ankorstore/toolbelt) [![GitHub License](https://img.shields.io/badge/license-MIT-lightgrey.svg)](https://raw.githubusercontent.com/ankorstore/orb-toolbelt/master/LICENSE) [![CircleCI Community](https://img.shields.io/badge/community-CircleCI%20Discuss-343434.svg)](https://discuss.circleci.com/c/ecosystem/orbs) + + + +A set of tools for crafting re-usable workflows + +--- + +## Resources + +[CircleCI Orb Registry Page](https://circleci.com/orbs/registry/orb/ankorstore/toolbelt) - The official registry page of this orb for all versions, executors, commands, and jobs described. + +[CircleCI Orb Docs](https://circleci.com/docs/2.0/orb-intro/#section=configuration) - Docs for using, creating, and publishing CircleCI Orbs. + +### How to Contribute + +We welcome [issues](https://github.com/ankorstore/orb-toolbelt/issues) to and [pull requests](https://github.com/ankorstore/orb-toolbelt/pulls) against this repository! + +### How to Publish An Update +1. Merge pull requests with desired changes to the main branch. + - For the best experience, squash-and-merge and use [Conventional Commit Messages](https://conventionalcommits.org/). +2. Find the current version of the orb. + - You can run `circleci orb info ankorstore/toolbelt | grep "Latest"` to see the current version. +3. Create a [new Release](https://github.com/ankorstore/orb-toolbelt/releases/new) on GitHub. + - Click "Choose a tag" and _create_ a new [semantically versioned](http://semver.org/) tag. (ex: v1.0.0) + - We will have an opportunity to change this before we publish if needed after the next step. +4. Click _"+ Auto-generate release notes"_. + - This will create a summary of all of the merged pull requests since the previous release. + - If you have used _[Conventional Commit Messages](https://conventionalcommits.org/)_ it will be easy to determine what types of changes were made, allowing you to ensure the correct version tag is being published. +5. Now ensure the version tag selected is semantically accurate based on the changes included. +6. Click _"Publish Release"_. + - This will push a new tag and trigger your publishing pipeline on CircleCI. \ No newline at end of file diff --git a/src/@orb.yml b/src/@orb.yml new file mode 100755 index 0000000..cb25168 --- /dev/null +++ b/src/@orb.yml @@ -0,0 +1,8 @@ +version: 2.1 + +description: > + A set of tools for crafting re-usable workflows + +# This information will be displayed in the orb registry and is not mandatory. +display: + source_url: "https://github.com/ankorstore/orb-toolbelt" diff --git a/src/README.md b/src/README.md new file mode 100644 index 0000000..b8e1015 --- /dev/null +++ b/src/README.md @@ -0,0 +1,26 @@ +# Orb Source + +Orbs are shipped as individual `orb.yml` files, however, to make development easier, it is possible to author an orb in _unpacked_ form, which can be _packed_ with the CircleCI CLI and published. + +The default `.circleci/config.yml` file contains the configuration code needed to automatically pack, test, and deploy any changes made to the contents of the orb source in this directory. + +## @orb.yml + +This is the entry point for our orb "tree", which becomes our `orb.yml` file later. + +Within the `@orb.yml` we generally specify 4 configuration keys + +**Keys** + +1. **version** + Specify version 2.1 for orb-compatible configuration `version: 2.1` +2. **description** + Give your orb a description. Shown within the CLI and orb registry +3. **display** + Specify the `home_url` referencing documentation or product URL, and `source_url` linking to the orb's source repository. +4. **orbs** + (optional) Some orbs may depend on other orbs. Import them here. + +## See: + - [Orb Author Intro](https://circleci.com/docs/2.0/orb-author-intro/#section=configuration) + - [Reusable Configuration](https://circleci.com/docs/2.0/reusing-config) diff --git a/src/commands/pass-fast.yml b/src/commands/pass-fast.yml new file mode 100755 index 0000000..ab5d4e1 --- /dev/null +++ b/src/commands/pass-fast.yml @@ -0,0 +1,57 @@ +description: > + This command will pass a job (or individual node of a parallel job) fast on retries when it has previously succeeded in this pipeline. + This preserves previous artifacts and optionally resubmits test results so that the resulting + job report and artifacts are consistent. + This command should be the first step in your job and all other steps should be provided as the `steps` parameter. +parameters: + pipeline-id: + type: string + description: The CircleCI Pipeline ID ( << pipeline.id >> ) + test-results: + type: string + description: Path for previous test results, must have been stored as an artifact + default: "" + steps: + type: steps + description: Job steps to execute if not passing fast +environment: + CIRCLE_PIPELINE_ID: << parameters.pipeline-id >> +steps: + - restore_cache: + name: Getting job history + keys: + - pass-{{ .Environment.CIRCLE_JOB }}-{{ .Environment.CIRCLE_NODE_INDEX }}-$CIRCLE_PIPELINE_ID + - run: + name: Restoring previous artifacts + command: << include(scripts/pass-fast.sh) >> + - store_artifacts: + name: Storing restored artifacts + path: . + destination: "" + - when: + condition: + not: + equal: [ "", << parameters.test-results >> ] + steps: + - store_test_results: + name: Upload previous test results for CircleCi + path: << parameters.test-results >> + - run: + name: Pass fast if we have a previous pass of this job in this pipeline + command: | + if [ "$PASS_FAST" = "true" ]; then + echo "This job has succeeded previously in this pipeline skipping..." + circleci-agent step halt + fi + - steps: << parameters.steps >> + - run: + name: Recording job success for this revision + command: | + mkdir .pass + PASS=$(echo "$CIRCLE_JOB-$CIRCLE_NODE_INDEX-$CIRCLE_PIPELINE_ID" | sed -e "s/[^[:alnum:]]/-/g" | tr -s "-" | tr A-Z a-z) + echo "Success!" > ".pass/$PASS" + - save_cache: + name: Saving job success for this revision + key: pass-{{ .Environment.CIRCLE_JOB }}-{{ .Environment.CIRCLE_NODE_INDEX }}-$CIRCLE_PIPELINE_ID + paths: + - .pass \ No newline at end of file diff --git a/src/examples/basic-pass-fast.yml b/src/examples/basic-pass-fast.yml new file mode 100755 index 0000000..6c49f17 --- /dev/null +++ b/src/examples/basic-pass-fast.yml @@ -0,0 +1,24 @@ +description: > + Pass a job fast on retries if its already succeeded in this pipeline. +usage: + version: 2.1 + orbs: + toolbelt: ankorstore/toolbelt@1.0.0 + workflows: + test: + jobs: + - test + jobs: + test: + docker: + - image: cimg/base + resource_class: small + steps: + - toolbelt/pass-fast: + steps: + - run: + name: Some work you dont want to repeat + command: echo "Doing some work you dont want to repeat"; + - run: + name: I wont be repeated once i have passed! + command: exit 0; \ No newline at end of file diff --git a/src/examples/parallel-pass-fast.yml b/src/examples/parallel-pass-fast.yml new file mode 100755 index 0000000..48b7f23 --- /dev/null +++ b/src/examples/parallel-pass-fast.yml @@ -0,0 +1,25 @@ +description: > + Pass each node of a parallel job fast on retries if its already succeeded in this pipeline. +usage: + version: 2.1 + orbs: + toolbelt: ankorstore/toolbelt@1.0.0 + workflows: + test: + jobs: + - test + jobs: + test: + docker: + - image: cimg/base + resource_class: small + parallelism: 2 + steps: + - toolbelt/pass-fast: + steps: + - run: + name: Some work you dont want to repeat + command: echo "Doing some work you dont want to repeat"; + - run: + name: The first node wont repeat once it has passed, the second node will always fail and so will be repeated. + command: exit $CIRCLE_NODE_INDEX; \ No newline at end of file diff --git a/src/scripts/pass-fast.sh b/src/scripts/pass-fast.sh new file mode 100644 index 0000000..c63d2a6 --- /dev/null +++ b/src/scripts/pass-fast.sh @@ -0,0 +1,64 @@ +#!/bin/bash +# Get workflows for the current pipeline and extract the previous execution of the current workflows name if there is one +get_workflows_in_pipeline() { + WORKFLOW_ENDPOINT="https://circleci.com/api/v2/workflow/$CIRCLE_WORKFLOW_ID" + curl -f -s WORKFLOW_ENDPOINT > /tmp/aks/current_wf.json + CIRCLE_WORKFLOW_NAME=$(jq -r '.name ' /tmp/aks/current_wf.json) + + WORKFLOWS_IN_PIPELINE_ENDPOINT="https://circleci.com/api/v2/pipeline/$CIRCLE_PIPELINE_ID/workflow?circle-token=${CIRCLE_TOKEN}" + curl -f -s $WORKFLOWS_IN_PIPELINE_ENDPOINT > /tmp/aks/pipeline_wf.json + PREVIOUS_WORKFLOW_ID=$(jq -r --arg current_workflow_id "$CIRCLE_WORKFLOW_ID" --arg current_workflow_name "$CIRCLE_WORKFLOW_NAME" '.items[] | select(.name == $current_workflow_name and .id != $current_workflow_id).id | values' /tmp/aks/pipeline_wf.json | head -n 1) +} + +get_job_from_previous_workflow() { + JOBS_IN_WORKFLOW_ENDPOINT="https://circleci.com/api/v2/workflow/${PREVIOUS_WORKFLOW_ID}/job?circle-token=${CIRCLE_TOKEN}" + curl -f -s $JOBS_IN_WORKFLOW_ENDPOINT > /tmp/aks/previous_wf_jobs.json + JOB_NUM=$(jq -r --arg curjobname "$CIRCLE_JOB" '.items[] | select(.name | test($curjobname)).job_number | values' /tmp/aks/previous_wf_jobs.json) +} + +get_artifacts_for_job() { + ARTIFACTS_URL="https://circleci.com/api/v2/project/gh/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME/$JOB_NUM/artifacts?circle-token=${CIRCLE_TOKEN}" + curl -f -s $ARTIFACTS_URL > /tmp/aks/artifacts.json + REQUIRED_ARTIFACTS=$(jq -r --argjson node_index "${CIRCLE_NODE_INDEX:0}" '.items[] | select(.node_index == $node_index) | "\(.url) \(.path)"' /tmp/aks/artifacts.json) + + if [ -z "$REQUIRED_ARTIFACTS" ]; then + echo "No Artifacts found." + exit 0; + fi + + while read -r ARTIFACT + do + set -- $ARTIFACT + echo "Downloading: $2" + curl -s -L --create-dirs -H "Circle-Token: $CIRCLE_TOKEN" -o "$2" "$1" + done <<< "$REQUIRED_ARTIFACTS" +} + +PASS_RECORD=$(echo "$CIRCLE_JOB-$CIRCLE_NODE_INDEX-$CIRCLE_PIPELINE_ID" | sed -e "s/[^[:alnum:]]/-/g" | tr -s "-" | tr A-Z a-z) +if [ -f ".pass/$PASS_RECORD" ]; then + echo "This job has succeeded previously in this workflow restoring artifacts..." + echo 'export PASS_FAST="true"' >> "$BASH_ENV" + mkdir -p /tmp/aks + + get_workflows_in_pipeline + + if [ -n "$PREVIOUS_WORKFLOW_ID" ]; then + echo "Getting job from previous workflow: $PREVIOUS_WORKFLOW_ID" + get_job_from_previous_workflow + if [ -n "$JOB_NUM" ]; then + echo "Getting artifacts from previous job: $JOB_NUM" + + mkdir -p $CIRCLE_WORKING_DIRECTORY + cd $CIRCLE_WORKING_DIRECTORY + + get_artifacts_for_job + else + echo "No previous execution of this job found in the workflow $PREVIOUS_WORKFLOW_ID" + fi + else + echo "No previous execution of this workflow found in this pipeline." + fi +else + echo "Job has not previously succeeded, no artifacts to restore" +fi +rm -rf .pass \ No newline at end of file