diff --git a/src/commands/get-job-artifacts.yml b/src/commands/get-job-artifacts.yml index 7865493..b5e555c 100644 --- a/src/commands/get-job-artifacts.yml +++ b/src/commands/get-job-artifacts.yml @@ -21,6 +21,10 @@ parameters: type: integer default: 10 description: Number of parallel downloads + unzip: + type: boolean + default: true + description: Unzip any tarballs in the downloaded artifacts steps: - run: name: Downloading artifacts @@ -30,4 +34,5 @@ steps: PRESERVE_PATHS: << parameters.preserve-paths >> TARGET_ARTIFACT_PATTERN: << parameters.path-pattern >> PROCESSES: << parameters.processes >> + UNZIP: << parameters.unzip >> command: << include(scripts/get-job-artifacts.sh) >> diff --git a/src/commands/store_compressed_artifacts.yml b/src/commands/store_compressed_artifacts.yml new file mode 100644 index 0000000..1ff9501 --- /dev/null +++ b/src/commands/store_compressed_artifacts.yml @@ -0,0 +1,18 @@ +description: > + Store artifacts as a gzip compressed tarball + + This can be a performance improvement when storing a large number of files or very large but highly compressible files such as logs. +parameters: + tarball-name: + type: string + description: Name for the compressed tarball, must be unique within this job + path: + type: string + description: Path to file or directory to store +steps: + - run: + name: Compressing << parameters.tarball-name >> + command: tar -czf << parameters.tarball-name >>.tar.gz << parameters.path >> + - store_artifacts: + name: Store << parameters.tarball-name >> + path: << parameters.tarball-name >>.tar.gz diff --git a/src/scripts/get-job-artifacts.sh b/src/scripts/get-job-artifacts.sh index 60c8850..4b88622 100755 --- a/src/scripts/get-job-artifacts.sh +++ b/src/scripts/get-job-artifacts.sh @@ -72,6 +72,8 @@ get_artifacts_for_job() { fi local tmp_config tmp_config=$(mktemp) + local tmp_zipped + tmp_zipped=$(mktemp) while read -r ARTIFACT do # shellcheck disable=SC2086 @@ -92,9 +94,17 @@ get_artifacts_for_job() { echo "url $URL" echo "output \"$OUTPUT_PATH\"" } >> "$tmp_config" + if [[ "$FILE_PATH" == *.tar.gz ]]; then + echo "$OUTPUT_PATH" >> "$tmp_zipped" + fi done <<< "$REQUIRED_ARTIFACTS" curl -K "$tmp_config" -s -L --retry 3 --retry-all-errors --create-dirs -H "Circle-Token: $CIRCLE_TOKEN" --parallel --parallel-immediate --parallel-max "$PROCESSES" rm "$tmp_config" + if [ "$UNZIP" = 1 ] && [ -s "$tmp_zipped" ]; then + echo "Unzipping files:" + cat "$tmp_zipped" + xargs -n 1 tar -xzf < "$tmp_zipped" + fi } if [ -n "$TARGET_PATH" ]; then