diff --git a/.circleci/test-deploy.yml b/.circleci/test-deploy.yml index 9ed3fca..e702904 100644 --- a/.circleci/test-deploy.yml +++ b/.circleci/test-deploy.yml @@ -11,8 +11,6 @@ workflows: test-deploy: jobs: # Make sure to include "filters: *filters" in every test job you want to run as part of your deployment. - - command-tests: - filters: *filters - orb-tools/pack: filters: *filters - orb-tools/publish: @@ -27,5 +25,3 @@ workflows: ignore: /.*/ tags: only: /^v[0-9]+\.[0-9]+\.[0-9]+$/ - -# VS Code Extension Version: 1.4.0 diff --git a/.gitignore b/.gitignore index 3c88c1b..fbb49fc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ # orb.yml is "packed" from source, and not published directly from the repository. orb.yml -.DS_Store \ No newline at end of file +.DS_Store +.idea diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..876eaa3 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,27 @@ +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.0.1 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-json + - id: check-xml + - id: check-merge-conflict + - id: check-added-large-files + - id: forbid-new-submodules + - id: detect-private-key + - id: check-shebang-scripts-are-executable + - id: check-case-conflict +- repo: https://github.com/koalaman/shellcheck-precommit + rev: v0.8.0 + hooks: + - id: shellcheck + args: ["--severity=warning"] +- repo: https://github.com/adrienverge/yamllint.git + rev: v1.17.0 + hooks: + - id: yamllint + args: [-c=.yamllint] diff --git a/src/@orb.yml b/src/@orb.yml index cb25168..14cde60 100755 --- a/src/@orb.yml +++ b/src/@orb.yml @@ -1,8 +1,9 @@ version: 2.1 description: > - A set of tools for crafting re-usable workflows + This orb provides a set of tools for crafting re-usable workflows. + + Such as a command `pass-fast` to assist only rerunning failed nodes of parallel jobs on job retries! -# This information will be displayed in the orb registry and is not mandatory. display: source_url: "https://github.com/ankorstore/orb-toolbelt" diff --git a/src/commands/pass-fast.yml b/src/commands/pass-fast.yml index ab5d4e1..60ddc66 100755 --- a/src/commands/pass-fast.yml +++ b/src/commands/pass-fast.yml @@ -14,8 +14,6 @@ parameters: steps: type: steps description: Job steps to execute if not passing fast -environment: - CIRCLE_PIPELINE_ID: << parameters.pipeline-id >> steps: - restore_cache: name: Getting job history @@ -23,7 +21,9 @@ steps: - pass-{{ .Environment.CIRCLE_JOB }}-{{ .Environment.CIRCLE_NODE_INDEX }}-$CIRCLE_PIPELINE_ID - run: name: Restoring previous artifacts - command: << include(scripts/pass-fast.sh) >> + environment: + CIRCLE_PIPELINE_ID: << parameters.pipeline-id >> + command: << include(scripts/pass-fast/restore.sh) >> - store_artifacts: name: Storing restored artifacts path: . @@ -38,20 +38,13 @@ steps: path: << parameters.test-results >> - run: name: Pass fast if we have a previous pass of this job in this pipeline - command: | - if [ "$PASS_FAST" = "true" ]; then - echo "This job has succeeded previously in this pipeline skipping..." - circleci-agent step halt - fi + command: << include(scripts/pass-fast/skip.sh) >> - steps: << parameters.steps >> - run: name: Recording job success for this revision - command: | - mkdir .pass - PASS=$(echo "$CIRCLE_JOB-$CIRCLE_NODE_INDEX-$CIRCLE_PIPELINE_ID" | sed -e "s/[^[:alnum:]]/-/g" | tr -s "-" | tr A-Z a-z) - echo "Success!" > ".pass/$PASS" + command: << include(scripts/pass-fast/record.sh) >> - save_cache: name: Saving job success for this revision key: pass-{{ .Environment.CIRCLE_JOB }}-{{ .Environment.CIRCLE_NODE_INDEX }}-$CIRCLE_PIPELINE_ID paths: - - .pass \ No newline at end of file + - .pass diff --git a/src/examples/basic-pass-fast.yml b/src/examples/basic-pass-fast.yml index 6c49f17..361d83f 100755 --- a/src/examples/basic-pass-fast.yml +++ b/src/examples/basic-pass-fast.yml @@ -21,4 +21,4 @@ usage: command: echo "Doing some work you dont want to repeat"; - run: name: I wont be repeated once i have passed! - command: exit 0; \ No newline at end of file + command: exit 0; diff --git a/src/examples/parallel-pass-fast.yml b/src/examples/parallel-pass-fast.yml index 48b7f23..d7c8cdc 100755 --- a/src/examples/parallel-pass-fast.yml +++ b/src/examples/parallel-pass-fast.yml @@ -22,4 +22,4 @@ usage: command: echo "Doing some work you dont want to repeat"; - run: name: The first node wont repeat once it has passed, the second node will always fail and so will be repeated. - command: exit $CIRCLE_NODE_INDEX; \ No newline at end of file + command: exit $CIRCLE_NODE_INDEX; diff --git a/src/scripts/pass-fast/record.sh b/src/scripts/pass-fast/record.sh new file mode 100755 index 0000000..f5b1d80 --- /dev/null +++ b/src/scripts/pass-fast/record.sh @@ -0,0 +1,4 @@ +#!/bin/bash +mkdir .pass +PASS_RECORD=$(echo "$CIRCLE_JOB-$CIRCLE_NODE_INDEX-$CIRCLE_PIPELINE_ID" | sed -e "s/[^[:alnum:]]/-/g" | tr -s "-" | tr "[:upper:]" "[:lower:]") +echo "Success!" > ".pass/$PASS_RECORD" \ No newline at end of file diff --git a/src/scripts/pass-fast.sh b/src/scripts/pass-fast/restore.sh similarity index 82% rename from src/scripts/pass-fast.sh rename to src/scripts/pass-fast/restore.sh index c63d2a6..b05b479 100644 --- a/src/scripts/pass-fast.sh +++ b/src/scripts/pass-fast/restore.sh @@ -2,23 +2,23 @@ # Get workflows for the current pipeline and extract the previous execution of the current workflows name if there is one get_workflows_in_pipeline() { WORKFLOW_ENDPOINT="https://circleci.com/api/v2/workflow/$CIRCLE_WORKFLOW_ID" - curl -f -s WORKFLOW_ENDPOINT > /tmp/aks/current_wf.json + curl -f -s "$WORKFLOW_ENDPOINT" > /tmp/aks/current_wf.json CIRCLE_WORKFLOW_NAME=$(jq -r '.name ' /tmp/aks/current_wf.json) WORKFLOWS_IN_PIPELINE_ENDPOINT="https://circleci.com/api/v2/pipeline/$CIRCLE_PIPELINE_ID/workflow?circle-token=${CIRCLE_TOKEN}" - curl -f -s $WORKFLOWS_IN_PIPELINE_ENDPOINT > /tmp/aks/pipeline_wf.json + curl -f -s "$WORKFLOWS_IN_PIPELINE_ENDPOINT" > /tmp/aks/pipeline_wf.json PREVIOUS_WORKFLOW_ID=$(jq -r --arg current_workflow_id "$CIRCLE_WORKFLOW_ID" --arg current_workflow_name "$CIRCLE_WORKFLOW_NAME" '.items[] | select(.name == $current_workflow_name and .id != $current_workflow_id).id | values' /tmp/aks/pipeline_wf.json | head -n 1) } get_job_from_previous_workflow() { JOBS_IN_WORKFLOW_ENDPOINT="https://circleci.com/api/v2/workflow/${PREVIOUS_WORKFLOW_ID}/job?circle-token=${CIRCLE_TOKEN}" - curl -f -s $JOBS_IN_WORKFLOW_ENDPOINT > /tmp/aks/previous_wf_jobs.json + curl -f -s "$JOBS_IN_WORKFLOW_ENDPOINT" > /tmp/aks/previous_wf_jobs.json JOB_NUM=$(jq -r --arg curjobname "$CIRCLE_JOB" '.items[] | select(.name | test($curjobname)).job_number | values' /tmp/aks/previous_wf_jobs.json) } get_artifacts_for_job() { ARTIFACTS_URL="https://circleci.com/api/v2/project/gh/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME/$JOB_NUM/artifacts?circle-token=${CIRCLE_TOKEN}" - curl -f -s $ARTIFACTS_URL > /tmp/aks/artifacts.json + curl -f -s "$ARTIFACTS_URL" > /tmp/aks/artifacts.json REQUIRED_ARTIFACTS=$(jq -r --argjson node_index "${CIRCLE_NODE_INDEX:0}" '.items[] | select(.node_index == $node_index) | "\(.url) \(.path)"' /tmp/aks/artifacts.json) if [ -z "$REQUIRED_ARTIFACTS" ]; then @@ -28,13 +28,17 @@ get_artifacts_for_job() { while read -r ARTIFACT do + # shellcheck disable=SC2086 set -- $ARTIFACT - echo "Downloading: $2" - curl -s -L --create-dirs -H "Circle-Token: $CIRCLE_TOKEN" -o "$2" "$1" + URL=$1 + shift + FILE_PATH="$*" + echo "Downloading: $FILE_PATH" + curl -s -L --create-dirs -H "Circle-Token: $CIRCLE_TOKEN" -o "$FILE_PATH" "$URL" done <<< "$REQUIRED_ARTIFACTS" } -PASS_RECORD=$(echo "$CIRCLE_JOB-$CIRCLE_NODE_INDEX-$CIRCLE_PIPELINE_ID" | sed -e "s/[^[:alnum:]]/-/g" | tr -s "-" | tr A-Z a-z) +PASS_RECORD=$(echo "$CIRCLE_JOB-$CIRCLE_NODE_INDEX-$CIRCLE_PIPELINE_ID" | sed -e "s/[^[:alnum:]]/-/g" | tr -s "-" | tr "[:upper:]" "[:lower:]") if [ -f ".pass/$PASS_RECORD" ]; then echo "This job has succeeded previously in this workflow restoring artifacts..." echo 'export PASS_FAST="true"' >> "$BASH_ENV" @@ -47,10 +51,7 @@ if [ -f ".pass/$PASS_RECORD" ]; then get_job_from_previous_workflow if [ -n "$JOB_NUM" ]; then echo "Getting artifacts from previous job: $JOB_NUM" - - mkdir -p $CIRCLE_WORKING_DIRECTORY - cd $CIRCLE_WORKING_DIRECTORY - + cd "$CIRCLE_WORKING_DIRECTORY" || exit get_artifacts_for_job else echo "No previous execution of this job found in the workflow $PREVIOUS_WORKFLOW_ID" diff --git a/src/scripts/pass-fast/skip.sh b/src/scripts/pass-fast/skip.sh new file mode 100755 index 0000000..681ee2e --- /dev/null +++ b/src/scripts/pass-fast/skip.sh @@ -0,0 +1,5 @@ +#!/bin/bash +if [ -n "$PASS_FAST" ]; then + echo "This job has succeeded previously in this pipeline skipping..." + circleci-agent step halt +fi \ No newline at end of file