diff --git a/CHANGELOG.md b/CHANGELOG.md index f886f36..48ed304 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # viash-actions v6.0.1 +## New functionality + +* `sync-and-cache`: Added an action for syncing and caching test resources (PR #31). + ## Minor changes * `project/detect-changed-components`: Make action less verbose by not printing out every changed file for every component (PR #32). @@ -8,6 +12,9 @@ * `project/detect-changed-components`: Output the `full_name` and `main_script_type` of each component (PR #33). +## Bug fixes + +* `check-concurrent-pr`: Added input parameter `github_token` for checking concurrent PRs (PR #31). # viash-actions v6.0.0 diff --git a/Makefile b/Makefile index 1421f8d..5ac2d7e 100644 --- a/Makefile +++ b/Makefile @@ -1,15 +1,17 @@ # Set up some variables for consistency QMD_FILES := $(shell find . -name "README.qmd") # Find all README.qmd files MD_FILES := $(patsubst %.qmd,%.md,$(QMD_FILES)) # Corresponding MD filenames +ACTION_FILES := $(shell find . -name "action.yml") # Find all action.yml files # Default target: build all MD files all: $(MD_FILES) -README.md: README.qmd - quarto render $< %/README.md: %/README.qmd %/action.yml quarto render $< +README.md: README.qmd $(ACTION_FILES) + quarto render $< + # Clean up generated MD files clean: rm -f $(MD_FILES) diff --git a/README.md b/README.md index 1b009db..4e8a972 100644 --- a/README.md +++ b/README.md @@ -12,12 +12,15 @@ There are also some actions that are commonly used in Viash projects: 1. [`project/build-target`](project/build-target) - Build target directory -2. [`project/detect-changed-components`](project/detect-changed-components) - +2. [`project/check-concurrent-pr`](project/check-concurrent-pr) - Check + for concurrent PRs +3. [`project/detect-changed-components`](project/detect-changed-components) - Detect components with changed files -3. [`project/is-pr`](project/is-pr) - Is PR 4. [`project/sync-and-cache-s3`](project/sync-and-cache-s3) - Sync and cache an S3 bucket -5. [`project/update-docker-engine`](project/update-docker-engine) - +5. [`project/sync-and-cache`](project/sync-and-cache) - Sync and cache + test resources specified by the project config +6. [`project/update-docker-engine`](project/update-docker-engine) - Update Docker Engine Finally, there are some [Viash diff --git a/project/check-concurrent-pr/README.md b/project/check-concurrent-pr/README.md index 609639e..a318c0f 100644 --- a/project/check-concurrent-pr/README.md +++ b/project/check-concurrent-pr/README.md @@ -17,6 +17,9 @@ request, this action will: ### Inputs +- `github_token`: - *required*. The GitHub token to use for the GitHub + CLI. + ### Outputs - `run`: Returns “true” if the branch has a PR and this run was not diff --git a/project/check-concurrent-pr/action.yml b/project/check-concurrent-pr/action.yml index d3c3917..338514d 100644 --- a/project/check-concurrent-pr/action.yml +++ b/project/check-concurrent-pr/action.yml @@ -7,6 +7,10 @@ description: > - Return 'true' if the event is not a push event - Return 'true' if the commit message contains 'ci force' - Return 'false' if a PR exists for the branch, else 'true' +inputs: + github_token: + description: 'The GitHub token to use for the GitHub CLI.' + required: true outputs: run: description: 'Returns "true" if the branch has a PR and this run was not triggered by a push event, else "false".' @@ -44,4 +48,4 @@ runs: echo "run=false" >> $GITHUB_OUTPUT fi env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_TOKEN: ${{ inputs.github_token }} diff --git a/project/sync-and-cache/README.md b/project/sync-and-cache/README.md new file mode 100644 index 0000000..7bbf8d4 --- /dev/null +++ b/project/sync-and-cache/README.md @@ -0,0 +1,61 @@ + + +# sync-and-cache + + + +Sync and cache test resources. This action will sync test resources from +different sources (e.g. S3) and cache them for later use. The cache key +is based on the hash of the test resources. For this action to work, the +Viash project config should contain a list of test resources to sync. + +Supported storage types: + +- `s3`: Syncs resources from an S3 bucket. +- Create a GitHub issue if you need support for other storage types. + +Example: + +``` yaml +info: + test_resources: + - type: s3 + path: s3://my-bucket/my-folder + dest: my-folder +``` + +### Inputs + +- `project_config`: - *optional*. Path to the project configuration + file. +- `cache_key_prefix`: - *optional*. A prefix for the cache hash key. + Prefix is also used for restoring stale cache if no cache hit occurred + for key. + +### Outputs + +- `cache_key`: Caching key. +- `dest_paths`: Paths to the synced resources. + +## Examples + +``` yaml +name: Demo of sync-and-cache + +on: + push: + pull_request: + +jobs: + demo: + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v3 + + - name: Sync and cache test resources specified by the project config + uses: viash-io/viash-actions/sync-and-cache +``` diff --git a/project/sync-and-cache/README.qmd b/project/sync-and-cache/README.qmd new file mode 100644 index 0000000..f655b27 --- /dev/null +++ b/project/sync-and-cache/README.qmd @@ -0,0 +1,63 @@ +--- +format: gfm +--- + +```{r} +#| include: false +library(tidyverse) +action <- yaml::read_yaml("action.yml") +action_name <- basename(getwd()) +``` + +# `r action_name` + + + +`r action$description` + +### Inputs + +```{r} +#| echo: false +lines <- map_chr(names(action$inputs), function(name) { + input <- action$inputs[[name]] + required <- ifelse (input$required %||% FALSE, "required", "optional") + glue::glue("* `{name}`: - _{required}_. {input$description}") +}) +knitr::asis_output(paste0(lines, collapse = "\n")) +``` + +### Outputs + +```{r} +#| echo: false +lines <- map_chr(names(action$outputs), function(name) { + output <- action$outputs[[name]] + glue::glue("* `{name}`: {output$description}") +}) +knitr::asis_output(paste0(lines, collapse = "\n")) +``` + +## Examples + +```yaml +name: Demo of `r action_name` + +on: + push: + pull_request: + +jobs: + demo: + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v3 + + - name: `r action$name` + uses: viash-io/viash-actions/sync-and-cache +``` + diff --git a/project/sync-and-cache/action.yml b/project/sync-and-cache/action.yml new file mode 100644 index 0000000..5e7331d --- /dev/null +++ b/project/sync-and-cache/action.yml @@ -0,0 +1,170 @@ +name: Sync and cache test resources specified by the project config +author: Data Intuitive +description: | + Sync and cache test resources. This action will sync test resources from + different sources (e.g. S3) and cache them for later use. The cache key is + based on the hash of the test resources. For this action to work, the Viash + project config should contain a list of test resources to sync. + + Supported storage types: + + - `s3`: Syncs resources from an S3 bucket. + - Create a GitHub issue if you need support for other storage types. + + Example: + + ```yaml + info: + test_resources: + - type: s3 + path: s3://my-bucket/my-folder + dest: my-folder + ``` +inputs: + project_config: + required: false + description: Path to the project configuration file. + default: _viash.yaml + cache_key_prefix: + required: false + description: A prefix for the cache hash key. Prefix is also used for restoring stale cache if no cache hit occurred for key. + default: cachekey__ +outputs: + cache_key: + description: Caching key. + value: ${{ steps.cache_key.outputs.cache_key }} + dest_paths: + description: Paths to the synced resources. + value: ${{ steps.test_resources.outputs.dest_paths }} +runs: + using: 'composite' + steps: + - name: Test resources + id: test_resources + shell: bash + run: | + # reformat the test resources into a pseudo-json that can be read line-by-line by bash + test_resources=$( + yq e \ + '.info.test_resources[] | "{type: " + (.type // "s3") + ", path: " + .path + ", dest: " + .dest + "}"' \ + "${{ inputs.project_config }}" + ) + + # fetch the test resource destination paths + dest_paths=$( + yq e '.info.test_resources[] | .dest' "${{ inputs.project_config }}" + ) + + # output multiline string + cat >> $GITHUB_OUTPUT <> $GITHUB_OUTPUT + + - name: Print resources + shell: bash + run: | + echo "### Cache key: ${{ steps.cache_key.outputs.cache_key }}" + echo + + echo "### Contents of 'test_resources':" + echo + echo "${{ steps.test_resources.outputs.test_resources }}" + echo + + echo "### Contents of 'dest_paths':" + echo + echo "${{ steps.test_resources.outputs.dest_paths }}" + echo + + # initialize cache + - name: Cache resources + uses: actions/cache@v4 + with: + path: ${{ steps.test_resources.outputs.dest_paths }} + key: ${{ steps.cache_key.outputs.cache_key }} + restore-keys: ${{ inputs.cache_key_prefix }} + + # sync if need be + - name: Sync resources + shell: bash + run: | + function sync_s3() { + local s3_path="$1" + local dest_path="$2" + AWS_EC2_METADATA_DISABLED=true \ + aws s3 sync \ + "$s3_path" \ + "$dest_path" \ + --no-sign-request + } + + echo "${{ steps.test_resources.outputs.test_resources }}" | \ + while read -r line; do + type=$(echo "$line" | yq e '.type') + path=$(echo "$line" | yq e '.path') + dest=$(echo "$line" | yq e '.dest') + + echo "Syncing '$path' to '$dest'..." + + if [ "$type" == "s3" ]; then + sync_s3 "$path" "$dest" + fi + done + + - name: List resources + shell: bash + run: | + echo "${{ steps.test_resources.outputs.test_resources }}" | \ + while read -r line; do + type=$(echo "$line" | yq e '.type') + path=$(echo "$line" | yq e '.path') + dest=$(echo "$line" | yq e '.dest') + + echo "====================================" + echo "type: $type" + echo "path: $path" + echo "dest: $dest" + tree $dest -L 3 + echo "" + echo "====================================" + done diff --git a/project/update-docker-engine/README.md b/project/update-docker-engine/README.md index 5999562..ebdc28a 100644 --- a/project/update-docker-engine/README.md +++ b/project/update-docker-engine/README.md @@ -32,5 +32,5 @@ jobs: uses: actions/checkout@v3 - name: Update Docker Engine - uses: viash-io/viash-actions/update-docker-engine@v6 + uses: viash-io/viash-actions/project/update-docker-engine@v6 ```