From 27dae3590667a10f0ab11520e981a05dc932fb09 Mon Sep 17 00:00:00 2001 From: Gregory Haddow <93638800+ankorstore-haddowg@users.noreply.github.com> Date: Mon, 27 Mar 2023 08:29:28 +0100 Subject: [PATCH] fix: improve curl retry on failure (#26) --- src/scripts/fail-fast.sh | 2 +- src/scripts/get-job-artifacts.sh | 6 +++--- src/scripts/pass-fast/restore.sh | 10 +++++----- src/scripts/wait-for-job.sh | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/scripts/fail-fast.sh b/src/scripts/fail-fast.sh index 6b6eac4..fd58e18 100755 --- a/src/scripts/fail-fast.sh +++ b/src/scripts/fail-fast.sh @@ -11,4 +11,4 @@ if [ -z "$CIRCLE_TOKEN" ]; then fi echo "$WHY" -curl -f -s --retry 3 -X POST --header "Content-Type: application/json" "https://circleci.com/api/v2/workflow/$CIRCLE_WORKFLOW_ID/cancel?circle-token=$CIRCLE_TOKEN" +curl -f -s --retry 3 --retry-all-errors -X POST --header "Content-Type: application/json" "https://circleci.com/api/v2/workflow/$CIRCLE_WORKFLOW_ID/cancel?circle-token=$CIRCLE_TOKEN" diff --git a/src/scripts/get-job-artifacts.sh b/src/scripts/get-job-artifacts.sh index 915e7c5..db30179 100755 --- a/src/scripts/get-job-artifacts.sh +++ b/src/scripts/get-job-artifacts.sh @@ -19,7 +19,7 @@ WORKFLOW_JOBS_JSON=/tmp/aks/current_wf_jobs.json get_jobs_in_workflow() { local WORKFLOW_JOBS_URL="https://circleci.com/api/v2/workflow/$CIRCLE_WORKFLOW_ID/job?circle-token=$CIRCLE_TOKEN" - curl -f -s --retry 3 "$WORKFLOW_JOBS_URL" > "$WORKFLOW_JOBS_JSON" + curl -f -s --retry 3 --retry-all-errors "$WORKFLOW_JOBS_URL" > "$WORKFLOW_JOBS_JSON" } get_artifacts_for_jobs() { @@ -36,7 +36,7 @@ get_artifacts_for_jobs() { get_artifacts_for_job() { local ARTIFACTS_URL="https://circleci.com/api/v2/project/gh/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME/$JOB_NUM/artifacts?circle-token=$CIRCLE_TOKEN" - curl -f -s --retry 3 "$ARTIFACTS_URL" > /tmp/aks/artifacts.json + curl -f -s --retry 3 --retry-all-errors "$ARTIFACTS_URL" > /tmp/aks/artifacts.json local REQUIRED_ARTIFACTS REQUIRED_ARTIFACTS=$(jq -r --arg target_artifact_pattern "$TARGET_ARTIFACT_PATTERN" '.items[] | select(.path| test($target_artifact_pattern)) | "\(.url) \(.path)"' /tmp/aks/artifacts.json) @@ -61,7 +61,7 @@ get_artifacts_for_job() { fi echo "Downloading: $FILE_PATH" echo " => $OUTPUT_PATH" - curl -s -L --retry 3 --create-dirs -H "Circle-Token: $CIRCLE_TOKEN" -o "$OUTPUT_PATH" "$URL" + curl -s -L --retry 3 --retry-all-errors --create-dirs -H "Circle-Token: $CIRCLE_TOKEN" -o "$OUTPUT_PATH" "$URL" done <<< "$REQUIRED_ARTIFACTS" } diff --git a/src/scripts/pass-fast/restore.sh b/src/scripts/pass-fast/restore.sh index 7381817..8d5b495 100755 --- a/src/scripts/pass-fast/restore.sh +++ b/src/scripts/pass-fast/restore.sh @@ -2,26 +2,26 @@ # Get workflows for the current pipeline and extract the previous execution of the current workflows name if there is one get_workflows_in_pipeline() { local WORKFLOW_ENDPOINT="https://circleci.com/api/v2/workflow/$CIRCLE_WORKFLOW_ID?circle-token=$CIRCLE_TOKEN" - curl -f -s --retry 3 "$WORKFLOW_ENDPOINT" > /tmp/aks/current_wf.json + curl -f -s --retry 3 --retry-all-errors "$WORKFLOW_ENDPOINT" > /tmp/aks/current_wf.json local CIRCLE_WORKFLOW_NAME CIRCLE_WORKFLOW_NAME=$(jq -r '.name ' /tmp/aks/current_wf.json) local WORKFLOWS_IN_PIPELINE_ENDPOINT="https://circleci.com/api/v2/pipeline/$CIRCLE_PIPELINE_ID/workflow?circle-token=$CIRCLE_TOKEN" - curl -f -s --retry 3 "$WORKFLOWS_IN_PIPELINE_ENDPOINT" > /tmp/aks/pipeline_wf.json + curl -f -s --retry 3 --retry-all-errors "$WORKFLOWS_IN_PIPELINE_ENDPOINT" > /tmp/aks/pipeline_wf.json PREVIOUS_WORKFLOW_ID=$(jq -r --arg current_workflow_id "$CIRCLE_WORKFLOW_ID" --arg current_workflow_name "$CIRCLE_WORKFLOW_NAME" '.items[] | select(.name == $current_workflow_name and .id != $current_workflow_id).id | values' /tmp/aks/pipeline_wf.json | head -n 1) } # Get jobs from the previous workflow and extract the job number for this current jobs previous build. get_job_from_previous_workflow() { local JOBS_IN_WORKFLOW_ENDPOINT="https://circleci.com/api/v2/workflow/$PREVIOUS_WORKFLOW_ID/job?circle-token=$CIRCLE_TOKEN" - curl -f -s --retry 3 "$JOBS_IN_WORKFLOW_ENDPOINT" > /tmp/aks/previous_wf_jobs.json + curl -f -s --retry 3 --retry-all-errors "$JOBS_IN_WORKFLOW_ENDPOINT" > /tmp/aks/previous_wf_jobs.json JOB_NUM=$(jq -r --arg current_job_name "$CIRCLE_JOB" '.items[] | select(.name | test($current_job_name)).job_number | values' /tmp/aks/previous_wf_jobs.json) } # Download all artifacts from a job get_artifacts_for_job() { local ARTIFACTS_URL="https://circleci.com/api/v2/project/gh/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME/$JOB_NUM/artifacts?circle-token=$CIRCLE_TOKEN" - curl -f -s --retry 3 "$ARTIFACTS_URL" > /tmp/aks/artifacts.json + curl -f -s --retry 3 --retry-all-errors "$ARTIFACTS_URL" > /tmp/aks/artifacts.json local REQUIRED_ARTIFACTS REQUIRED_ARTIFACTS=$(jq -r --argjson node_index "${CIRCLE_NODE_INDEX:-0}" '.items[] | select(.node_index == $node_index) | "\(.url) \(.path)"' /tmp/aks/artifacts.json) @@ -38,7 +38,7 @@ get_artifacts_for_job() { shift local FILE_PATH="$*" echo "Downloading: $FILE_PATH" - curl -s -L --retry 3 --create-dirs -H "Circle-Token: $CIRCLE_TOKEN" -o "$FILE_PATH" "$URL" + curl -s -L --retry 3 --retry-all-errors --create-dirs -H "Circle-Token: $CIRCLE_TOKEN" -o "$FILE_PATH" "$URL" done <<< "$REQUIRED_ARTIFACTS" } diff --git a/src/scripts/wait-for-job.sh b/src/scripts/wait-for-job.sh index 788f226..7d18690 100755 --- a/src/scripts/wait-for-job.sh +++ b/src/scripts/wait-for-job.sh @@ -19,7 +19,7 @@ get_job_status() { local NUMBER="" local WORKFLOW_JOBS_URL="https://circleci.com/api/v2/workflow/$CIRCLE_WORKFLOW_ID/job?circle-token=$CIRCLE_TOKEN" local WORKFLOW_JOBS_JSON="/tmp/aks/wf_$CIRCLE_WORKFLOW_ID.json" - curl -f -s --retry 3 "$WORKFLOW_JOBS_URL" > "$WORKFLOW_JOBS_JSON" + curl -f -s --retry 3 --retry-all-errors "$WORKFLOW_JOBS_URL" > "$WORKFLOW_JOBS_JSON" STATUS=$(jq -r ".items[] | select(.name==\"$NAME_OF_JOB\") | .status | values" "$WORKFLOW_JOBS_JSON") NUMBER=$(jq -r ".items[] | select(.name==\"$NAME_OF_JOB\") | .job_number | values" "$WORKFLOW_JOBS_JSON") echo "$STATUS $NUMBER"