diff --git a/CHANGELOG.md b/CHANGELOG.md index 16896d6..edeb5fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,32 @@ +# viash-actions v6.0.0 + +## Breaking changes + +* `update-docker-engine` was moved to `project/update-docker-engine` (PR #27). + +* `pro/build-nextflow-params`: Arguments `components` and `workflows` were removed in favour for the `target_dir` argument (PR #27). + +* `pro/build-nextflow-schemas`: Arguments `components` and `workflows` were removed in favour for the `target_dir` argument (PR #27). + +* Removed `viash-hub/deploy`: Viash Hub pulls source code from GitHub and builds Viash packages accordingly, as opposed to pushing deployments using GitHub Actions (PR #27). + +## New functionality + +* `project/build-target`: Created an action for building a target directory using `viash ns build` and generating a matrix of the components for which Docker images need to be built and pushed (PR #27). + +* `project/check-concurrent-pr`: Created an action for checking if a push event is concurrent with a pull request (PR #27). + +## Major changes + +* All actions were updated to work with Viash 0.9 and later (PR #27). + +## Documentation + +* Updated all READMEs to be more consistent (PR #27). + +* Added a Makefile for generating READMEs more efficiently (PR #27). + + # viash-actions v5.3.1 ## Bug fixes @@ -20,13 +49,13 @@ ## Minor changes -* `project/detect-changed-components`: bump `tj-actions/changed-files` to `42.0.2` (PR #24) +* `project/detect-changed-components`: bump `tj-actions/changed-files` to `42.0.2` (PR #24). # viash-actions v5.1.0 ## New functionality -* `pro/build-nextflow-schemas`: Add optional flag to allow dataset input for nf-tower (PR #23) +* `pro/build-nextflow-schemas`: Add optional flag to allow dataset input for nf-tower (PR #23). # viash-actions v5.0.0 diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..1421f8d --- /dev/null +++ b/Makefile @@ -0,0 +1,15 @@ +# Set up some variables for consistency +QMD_FILES := $(shell find . -name "README.qmd") # Find all README.qmd files +MD_FILES := $(patsubst %.qmd,%.md,$(QMD_FILES)) # Corresponding MD filenames + +# Default target: build all MD files +all: $(MD_FILES) + +README.md: README.qmd + quarto render $< +%/README.md: %/README.qmd %/action.yml + quarto render $< + +# Clean up generated MD files +clean: + rm -f $(MD_FILES) diff --git a/README.md b/README.md index a421b22..1b009db 100644 --- a/README.md +++ b/README.md @@ -1,23 +1,50 @@ # Github Actions for Viash -This repository stores [Github Actions](https://github.com/features/actions) useful for building and publishing [Viash](https://viash.io) components: -1. [viash-io/viash-actions/setup](setup) - Install Viash -2. [viash-io/viash-actions/ns-list](ns-list) - List Viash components in a repository -3. [viash-io/viash-actions/ns-build](ns-build) - Build a namespace from many viash config files +This repository stores Github Actions useful for building and publishing +[Viash](https://viash.io) components: -There are also a few actions that are commonly used in Viash projects: +1. [`ns-build`](ns-build) - viash ns build +2. [`ns-list`](ns-list) - viash ns list +3. [`setup`](setup) - Setup Viash -1. [viash-io/viash-actions/project/cache-and-sync-s3](project/cache-and-sync-s3) - Sync and cache an S3 bucket -2. [viash-io/viash_actions/project/detect-changed-components](project/detect-changed-components) - Detect changed Viash components +There are also some actions that are commonly used in Viash projects: -We recommend using `v3` for your actions. +1. [`project/build-target`](project/build-target) - Build target + directory +2. [`project/detect-changed-components`](project/detect-changed-components) - + Detect components with changed files +3. [`project/is-pr`](project/is-pr) - Is PR +4. [`project/sync-and-cache-s3`](project/sync-and-cache-s3) - Sync and + cache an S3 bucket +5. [`project/update-docker-engine`](project/update-docker-engine) - + Update Docker Engine -## Release Management +Finally, there are some [Viash +Pro](https://www.data-intuitive.com/services/viashpro.html) actions: + +1. [`pro/build-nextflow-params`](pro/build-nextflow-params) - Build + parameter yaml +2. [`pro/build-nextflow-schemas`](pro/build-nextflow-schemas) - Build + nf-tower schemas +3. [`pro/generate-documentation-qmd`](pro/generate-documentation-qmd) - + Build and publish qmd -This repository uses [GitHub's recommended release management for actions](https://docs.github.com/en/actions/creating-actions/about-custom-actions#using-release-management-for-actions): +We recommend using `v6` for your actions. + +## Release Management -* GitHub releases with tags are used for updates on the actions. -* Semantic versioning is used, with major, minor and possibly patch release. -* Major versions (such as `v1`) will always point to the last minor or patch release for this major version. (when `v1.0.2` is out, `v1` will point to this update to). This means using `viash-io/viash-actions/setup@v1` in your workflow file will automatically get the updated versions. Using `viash-io/viash-actions/setup@v1.0.2` will pin a specific release. -* Major version changes (`v1` to `v2`) will often come with breaking changes, and workflows might require manual updates. \ No newline at end of file +This repository uses [GitHub’s recommended release management for +actions](https://docs.github.com/en/actions/creating-actions/about-custom-actions#using-release-management-for-actions): + +- GitHub releases with tags are used for updates on the actions. +- Semantic versioning is used, with major, minor and possibly patch + release. +- Major versions (such as `v1`) will always point to the last minor or + patch release for this major version. (when `v1.0.2` is out, `v1` will + point to this update to). This means using + `viash-io/viash-actions/setup@v1` in your workflow file will + automatically get the updated versions. Using + `viash-io/viash-actions/setup@v1.0.2` will pin a specific release. +- Major version changes (`v1` to `v2`) will often come with breaking + changes, and workflows might require manual updates. diff --git a/README.qmd b/README.qmd new file mode 100644 index 0000000..aadffec --- /dev/null +++ b/README.qmd @@ -0,0 +1,54 @@ +--- +title: Github Actions for Viash +format: gfm +--- + +```{r} +#| include: false +library(tidyverse) + +repo_name <- "viash-io/viash-actions" +latest_stable_version <- "v6" + +actions <- tibble( + file = list.files(".", full.names = TRUE, recursive = TRUE, pattern = "action.yml"), + path = gsub("^\\./(.*)/action.yml$", "\\1", file), + action = map(file, yaml::read_yaml), + action_name = map_chr(action, "name"), + namespace = ifelse(grepl("/", path), gsub("/.*", "", path), ""), + str = paste0("1. [`", path, "`](", path, ") - ", action_name, "\n") +) +``` + + +This repository stores Github Actions useful for building and publishing [Viash](https://viash.io) components: + +```{r output="asis"} +#| echo: false +cat(actions %>% filter(namespace == "") %>% pull(str) %>% paste(collapse = "")) +``` + +There are also some actions that are commonly used in Viash projects: + +```{r output="asis"} +#| echo: false +cat(actions %>% filter(namespace == "project") %>% pull(str) %>% paste(collapse = "")) +``` + +Finally, there are some [Viash Pro](https://www.data-intuitive.com/services/viashpro.html) actions: + +```{r output="asis"} +#| echo: false +cat(actions %>% filter(namespace == "pro") %>% pull(str) %>% paste(collapse = "")) +``` + +We recommend using ``r latest_stable_version`` for your actions. + +## Release Management + +This repository uses [GitHub's recommended release management for actions](https://docs.github.com/en/actions/creating-actions/about-custom-actions#using-release-management-for-actions): + +* GitHub releases with tags are used for updates on the actions. +* Semantic versioning is used, with major, minor and possibly patch release. +* Major versions (such as `v1`) will always point to the last minor or patch release for this major version. (when `v1.0.2` is out, `v1` will point to this update to). This means using `viash-io/viash-actions/setup@v1` in your workflow file will automatically get the updated versions. Using `viash-io/viash-actions/setup@v1.0.2` will pin a specific release. +* Major version changes (`v1` to `v2`) will often come with breaking changes, and workflows might require manual updates. \ No newline at end of file diff --git a/ns-build/README.md b/ns-build/README.md index 323d8b1..b2fcde9 100644 --- a/ns-build/README.md +++ b/ns-build/README.md @@ -1,5 +1,11 @@ -ns-build -================ + + +# ns-build + + Build a namespace from many viash config files. @@ -11,7 +17,7 @@ each of the arguments. We recommend using a Linux or MacOS runner if possible. -### Inputs available +### Inputs - `query`: - *optional*. Filter which components get selected by component and namespace name. Can be a regex. Example: @@ -22,39 +28,55 @@ We recommend using a Linux or MacOS runner if possible. component name. Can be a regex. Example: “^component1”. - `src`: - *optional*. A source directory containing viash config files, possibly structured in a hierarchical folder structure. Default: src/. -- `platform`: - *optional*. Acts as a regular expression to filter the - platform ids specified in the found config files. If this is not - provided, all platforms will be used. If no platforms are defined in a - config, the native platform will be used. In addition, the path to a - platform yaml file can also be specified. +- `platform`: - *optional*. (viash \< 0.9.0) Acts as a regular + expression to filter the platform ids specified in the found config + files. If this is not provided, all platforms will be used. If no + platforms are defined in a config, the native platform will be used. + In addition, the path to a platform yaml file can also be specified. +- `runner`: - *optional*. (viash \>= 0.9.0) Acts as a regular expression + to filter the runner ids specified in the found config files. If this + is not provided, all runners will be used. If no runners are defined + in a config, the executable runner will be used. +- `engine`: - *optional*. (viash \>= 0.9.0) Acts as a regular expression + to filter the engine ids specified in the found config files. If this + is not provided, all engines will be used. If no engines are defined + in a config, the native engine will be used. - `parallel`: - *optional*. Whether or not to run the process in parallel. - `config_mod`: - *optional*. Modify a viash config at runtime using - dynamic config modding. + config_mod. - `target`: - *optional*. A target directory to build the executables into. Default: target/. -- `setup`: - *optional*. Which setup strategy for creating the container - to use \[Docker Platform only\]. +- `colorize`: - *optional*. Specify whether the console output should be + colorized. If not specified, we attempt to detect this automatically. + Possible values are: “true”, “false”, “auto”. +- `loglevel`: - *optional*. Specify the log level in us. Possible values + are: “error”, “warn”, “info”, “debug”, “trace”. +- `setup`: - *optional*. Which docker_setup_strategy for creating the + container to use \[Docker Engine only\]. + +### Outputs + +- `output`: The output of ‘viash ns build’. ## Examples ``` yaml -name: demo of viash ns build +name: Demo of ns-build on: - branch: main + push: jobs: viash-linux: runs-on: ubuntu-latest steps: - - uses: viash-io/viash-actions/setup@v3 + - name: Check out repository + uses: actions/checkout@v3 - - uses: viash-io/viash-actions/ns-build@v3 - viash-macos: - runs-on: macos-latest - steps: - - uses: viash-io/viash-actions/setup@v3 + - name: Install Viash + uses: viash-io/viash-actions/setup@v6 - - uses: viash-io/viash-actions/ns-build@v3 + - name: viash ns build + uses: viash-io/viash-actions/ns-build@v6 ``` diff --git a/ns-build/README.qmd b/ns-build/README.qmd index bbcb14d..9626936 100644 --- a/ns-build/README.qmd +++ b/ns-build/README.qmd @@ -1,15 +1,21 @@ --- -title: ns-build format: gfm --- ```{r} #| include: false library(tidyverse) -# action <- yaml::read_yaml("ns-build/action.yml") action <- yaml::read_yaml("action.yml") +action_name <- basename(getwd()) ``` +# `r action_name` + + + `r action$description` ## Usage @@ -18,7 +24,7 @@ This action will run `viash ns build`. See the reference documentation on [ns bu We recommend using a Linux or MacOS runner if possible. -### Inputs available +### Inputs ```{r} #| echo: false @@ -30,26 +36,36 @@ lines <- map_chr(names(action$inputs), function(name) { knitr::asis_output(paste0(lines, collapse = "\n")) ``` +### Outputs + +```{r} +#| echo: false +lines <- map_chr(names(action$outputs), function(name) { + output <- action$outputs[[name]] + glue::glue("* `{name}`: {output$description}") +}) +knitr::asis_output(paste0(lines, collapse = "\n")) +``` + ## Examples ```yaml -name: demo of viash ns build +name: Demo of `r action_name` on: - branch: main + push: jobs: viash-linux: runs-on: ubuntu-latest steps: - - uses: viash-io/viash-actions/setup@v3 + - name: Check out repository + uses: actions/checkout@v3 - - uses: viash-io/viash-actions/ns-build@v3 - viash-macos: - runs-on: macos-latest - steps: - - uses: viash-io/viash-actions/setup@v3 + - name: Install Viash + uses: viash-io/viash-actions/setup@v6 - - uses: viash-io/viash-actions/ns-build@v3 + - name: `r action$name` + uses: viash-io/viash-actions/ns-build@v6 ``` diff --git a/ns-build/action.yml b/ns-build/action.yml index 8efd0ef..7189f55 100644 --- a/ns-build/action.yml +++ b/ns-build/action.yml @@ -1,3 +1,7 @@ +######################################################## +# DO NOT EDIT THIS FILE MANUALLY # +# Please run 'Rscript ns-build/generate_yml.R' instead # +######################################################## name: viash ns build author: Data Intuitive description: Build a namespace from many viash config files. @@ -15,28 +19,52 @@ inputs: regex. Example: "^component1".' required: false src: - description: ' A source directory containing viash config files, possibly structured + description: 'A source directory containing viash config files, possibly structured in a hierarchical folder structure. Default: src/.' required: false platform: - description: Acts as a regular expression to filter the platform ids specified - in the found config files. If this is not provided, all platforms will be used. - If no platforms are defined in a config, the native platform will be used. In - addition, the path to a platform yaml file can also be specified. + description: (viash < 0.9.0) Acts as a regular + expression to filter the platform ids specified in the found config files. If + this is not provided, all platforms will be used. If no platforms are defined + in a config, the native platform will be used. In addition, the path to a platform + yaml file can also be specified. + required: false + runner: + description: (viash >= 0.9.0) Acts as a regular expression to filter the runner ids specified in + the found config files. If this is not provided, all runners will be used. If + no runners are defined in a config, the executable runner will be used. + required: false + engine: + description: (viash >= 0.9.0) Acts as a regular expression to filter the engine ids specified in + the found config files. If this is not provided, all engines will be used. If + no engines are defined in a config, the native engine will be used. required: false parallel: description: Whether or not to run the process in parallel. required: false config_mod: - description: Modify a viash config at runtime using dynamic config modding. + description: Modify a viash config at runtime using config_mod. required: false target: description: 'A target directory to build the executables into. Default: target/.' required: false + colorize: + description: 'Specify whether the console output should be colorized. If not specified, + we attempt to detect this automatically. Possible values are: "true", "false", + "auto".' + required: false + loglevel: + description: 'Specify the log level in us. Possible values are: "error", "warn", + "info", "debug", "trace".' + required: false setup: - description: Which setup strategy for creating the container to use [Docker Platform - only]. + description: Which docker_setup_strategy for creating the container to use [Docker + Engine only]. required: false +outputs: + output: + description: The output of 'viash ns build'. runs: using: node20 main: index.js + diff --git a/ns-build/generate_yml.R b/ns-build/generate_yml.R index 4393bab..41842ae 100644 --- a/ns-build/generate_yml.R +++ b/ns-build/generate_yml.R @@ -9,15 +9,15 @@ subcommands <- c("ns", "build") #################### common code # fetch schema -out <- system("viash export cli_schema", intern = TRUE) +out <- system("viash export cli_schema --format json", intern = TRUE) schema <- jsonlite::fromJSON(out, simplifyVector = FALSE) # go down schema to find the right command for (i in seq_along(subcommands)) { - subcommand <- subcommands[[i]] - schema <- schema[[which(sapply(schema, function(li) li$name == subcommand))]] + subcommand_names <- sapply(schema, function(li) li$name) + schema <- schema[[which(subcommand_names == subcommands[[i]])]] - if (i != length(subcommands)) { + if ("subcommands" %in% names(schema) && i < length(subcommands)) { schema <- schema$subcommands } } @@ -41,15 +41,24 @@ input_values <- sapply(selected_opts, function(opt) { ) setNames(list(out), opt$name) }) +# workaround for viash 0.9.0-RC6, should be fixed in 0.9.0 +if (!"platform" %in% names(input_values)) { + input_values$platform <- list( + description = "Acts as a regular expression to filter the platform ids specified in the found config files. If this is not provided, all platforms will be used. If no platforms are defined in a config, the native platform will be used. In addition, the path to a platform yaml file can also be specified. Deprecated in Viash 0.9.0, will be removed in Viash 1.0.0.", + required = FALSE + ) +} + out <- list( name = schema$bannerCommand, + author = "Data Intuitive", description = schema$bannerDescription, inputs = input_values, - # outputs = list( - # output = list( - # description = "A list of all of the components found. By default this will be a yaml, unless the format argument was set to 'json'." - # ) - # ), + outputs = list( + output = list( + description = "The output of 'viash ns build'." + ) + ), runs = list( using = "node20", main = "index.js" @@ -64,8 +73,15 @@ handlers <- list( return(result) } ) -yaml::write_yaml( +yaml_str <- yaml::as.yaml( out, - dest_file, handlers = handlers ) +out_str <- c( + "########################################################", + "# DO NOT EDIT THIS FILE MANUALLY #", + "# Please run 'Rscript ns-build/generate_yml.R' instead #", + "########################################################", + yaml_str +) +writeLines(out_str, dest_file) diff --git a/ns-build/index.js b/ns-build/index.js index bb67009..6bc34e5 100644 --- a/ns-build/index.js +++ b/ns-build/index.js @@ -22,7 +22,7 @@ async function run() { // todo: can I extract these inputNames from the action.yml? const inputNames = [ "query", "query_namespace", "query_name", "src", "platform", - "parallel", "config_mod", "target", "setup" + "runner", "engine", "parallel", "config_mod", "target", "setup" ] const booleanInputs = ["parallel"] const multilineInputs = ["config_mod"] diff --git a/ns-list/README.md b/ns-list/README.md index d7fa2d1..5f17867 100644 --- a/ns-list/README.md +++ b/ns-list/README.md @@ -1,5 +1,11 @@ -ns-list -================ + + +# ns-list + + List a namespace containing many viash config files. @@ -11,7 +17,7 @@ each of the arguments. We recommend using a Linux or MacOS runner if possible. -### Inputs available +### Inputs - `query`: - *optional*. Filter which components get selected by component and namespace name. Can be a regex. Example: @@ -21,66 +27,101 @@ We recommend using a Linux or MacOS runner if possible. - `query_name`: - *optional*. Filter which components get selected by component name. Can be a regex. Example: “^component1”. - `src`: - *optional*. A source directory containing viash config files, - possibly structured in a hierarchical folder structure. Default: - “src/”. -- `platform`: - *optional*. Acts as a regular expression to filter the - platform ids specified in the found config files. If this is not - provided, all platforms will be used. If no platforms are defined in a - config, the native platform will be used. In addition, the path to a - platform yaml file can also be specified. + possibly structured in a hierarchical folder structure. Default: src/. +- `platform`: - *optional*. (viash \< 0.9.0) Use –runner and –engine + instead. Acts as a regular expression to filter the platform ids + specified in the found config files. If this is not provided, all + platforms will be used. If no platforms are defined in a config, the + native platform will be used. In addition, the path to a platform yaml + file can also be specified. +- `runner`: - *optional*. (viash \>= 0.9.0) Acts as a regular expression + to filter the runner ids specified in the found config files. If this + is not provided, all runners will be used. If no runners are defined + in a config, the executable runner will be used. +- `engine`: - *optional*. (viash \>= 0.9.0) Acts as a regular expression + to filter the engine ids specified in the found config files. If this + is not provided, all engines will be used. If no engines are defined + in a config, the native engine will be used. - `config_mod`: - *optional*. Modify a viash config at runtime using - dynamic config modding. + config_mod. +- `colorize`: - *optional*. Specify whether the console output should be + colorized. If not specified, we attempt to detect this automatically. + Possible values are: “true”, “false”, “auto”. +- `loglevel`: - *optional*. Specify the log level in us. Possible values + are: “error”, “warn”, “info”, “debug”, “trace”. - `format`: - *optional*. Which output format to use. Possible values - are: “yaml”, “json”. Default is “yaml”. -- `parse_argument_groups`: - *optional*. Whether or not to postprocess - each component’s argument groups. + are: “yaml”, “json”. +- `parse_argument_groups`: - *optional*. DEPRECATED. This is now always + enabled. Whether or not to postprocess each component’s + argument_groups. - `output_file`: - *optional*. Path of a file to which the output will be written. If not set, this action will create a file with a random name in `RUNNER_TEMP`. ### Outputs -- `output`: A list of all of the components found. By default this will - be a yaml, unless the format argument was set to ‘json’. +- `output`: The output of the ‘viash ns list’ command, which is a list + of all of the components found. By default this will be a yaml, unless + the format argument was set to ‘json’. - `output_file`: Path of a file to which the output was written (same as - inputs.output_file). We recommend using this property for capturing - the action’s output because there is a limit in the object size that - github actions can manage. Additionally, if you use this property - instead of a static file path, changing the location of the output - file will not require you to adjust settings for downstream actions as - well. + \`inputs.output_file\`\`). We recommend using this property for + capturing the action’s output because there is a limit in the object + size that github actions can manage. Additionally, if you use this + property instead of a static file path, changing the location of the + output file will not require you to adjust settings for downstream + actions as well. +- `output_matrix`: A simplified version of the output, which is a list + of components with fields ‘name’, ‘namespace’, ‘full_name’, ‘config’, + and ‘dir’. -## Examples +## Usage ``` yaml -name: demo of viash ns list +name: Demo of ns-list -on: - branch: main +on: + push: jobs: - viash-linux: + nslist: runs-on: ubuntu-latest + + outputs: + output_file: ${{ steps.ns_list.outputs.output_file }} + output_matrix: ${{ steps.ns_list.outputs.output_matrix }} + steps: - - uses: viash-io/viash-actions/setup@v3 + - name: Install Viash + uses: viash-io/viash-actions/setup@v6 - - id: ns_list - uses: viash-io/viash-actions/ns-list@v3 + - name: Check out repository + uses: actions/checkout@v3 - - run: | - # do something with the output yaml - echo "${{steps.ns_list.outputs.output}}" - viash-macos: - runs-on: macos-latest - steps: - - uses: viash-io/viash-actions/setup@v3 + - name: viash ns list + uses: viash-io/viash-actions/ns-list@v6 - - id: ns_list - uses: viash-io/viash-actions/ns-list@v3 - with: - format: json + - name: View output + run: | + cat "${{steps.ns_list.outputs.output_file}}" + + use_matrix: + runs-on: ubuntu-latest + needs: nslist + + strategy: + matrix: + component: ${{fromJson(needs.nslist.outputs.output_matrix)}} - - run: | - # do something with the output json - echo "${{steps.ns_list.outputs.output}}" + steps: + - name: Do something for every component + run: | + echo "Name: ${{ matrix.component.name }}" + echo "Namespace: ${{ matrix.component.namespace }}" + echo "Runner: ${{ matrix.component.runner }}" + echo "Engine: ${{ matrix.component.engine }}" + echo "Config: ${{ matrix.component.config }}" + echo "Executable: ${{ matrix.component.executable }}" + echo "Src dir: ${{ matrix.component.src_dir }}" + echo "Target dir: ${{ matrix.component.target_dir }}" + echo "Full name: ${{ matrix.component.full_name }}" ``` diff --git a/ns-list/README.qmd b/ns-list/README.qmd index e4b6d2a..9f60ac6 100644 --- a/ns-list/README.qmd +++ b/ns-list/README.qmd @@ -1,15 +1,21 @@ --- -title: ns-list format: gfm --- ```{r} #| include: false library(tidyverse) -# action <- yaml::read_yaml("ns-list/action.yml") action <- yaml::read_yaml("action.yml") +action_name <- basename(getwd()) ``` +# `r action_name` + + + `r action$description` ## Usage @@ -18,7 +24,7 @@ This action will run `viash ns list`. See the reference documentation on [ns lis We recommend using a Linux or MacOS runner if possible. -### Inputs available +### Inputs ```{r} #| echo: false @@ -41,38 +47,54 @@ lines <- map_chr(names(action$outputs), function(name) { knitr::asis_output(paste0(lines, collapse = "\n")) ``` -## Examples +## Usage ```yaml -name: demo of viash ns list +name: Demo of `r action_name` -on: - branch: main +on: + push: jobs: - viash-linux: + nslist: runs-on: ubuntu-latest + + outputs: + output_file: ${{ steps.ns_list.outputs.output_file }} + output_matrix: ${{ steps.ns_list.outputs.output_matrix }} + steps: - - uses: viash-io/viash-actions/setup@v3 + - name: Install Viash + uses: viash-io/viash-actions/setup@v6 - - id: ns_list - uses: viash-io/viash-actions/ns-list@v3 + - name: Check out repository + uses: actions/checkout@v3 - - run: | - # do something with the output yaml - echo "${{steps.ns_list.outputs.output}}" - viash-macos: - runs-on: macos-latest - steps: - - uses: viash-io/viash-actions/setup@v3 + - name: `r action$name` + uses: viash-io/viash-actions/ns-list@v6 - - id: ns_list - uses: viash-io/viash-actions/ns-list@v3 - with: - format: json + - name: View output + run: | + cat "${{steps.ns_list.outputs.output_file}}" + + use_matrix: + runs-on: ubuntu-latest + needs: nslist + + strategy: + matrix: + component: ${{fromJson(needs.nslist.outputs.output_matrix)}} - - run: | - # do something with the output json - echo "${{steps.ns_list.outputs.output}}" + steps: + - name: Do something for every component + run: | + echo "Name: ${{ matrix.component.name }}" + echo "Namespace: ${{ matrix.component.namespace }}" + echo "Runner: ${{ matrix.component.runner }}" + echo "Engine: ${{ matrix.component.engine }}" + echo "Config: ${{ matrix.component.config }}" + echo "Executable: ${{ matrix.component.executable }}" + echo "Src dir: ${{ matrix.component.src_dir }}" + echo "Target dir: ${{ matrix.component.target_dir }}" + echo "Full name: ${{ matrix.component.full_name }}" ``` - diff --git a/ns-list/action.yml b/ns-list/action.yml index b1ad7e5..244306c 100644 --- a/ns-list/action.yml +++ b/ns-list/action.yml @@ -1,3 +1,7 @@ +######################################################## +# DO NOT EDIT THIS FILE MANUALLY # +# Please run 'Rscript ns-list/generate_yml.R' instead # +######################################################## name: viash ns list author: Data Intuitive description: List a namespace containing many viash config files. @@ -16,39 +20,64 @@ inputs: required: false src: description: 'A source directory containing viash config files, possibly structured - in a hierarchical folder structure. Default: "src/".' + in a hierarchical folder structure. Default: src/.' required: false platform: - description: Acts as a regular expression to filter the platform ids specified - in the found config files. If this is not provided, all platforms will be used. - If no platforms are defined in a config, the native platform will be used. In - addition, the path to a platform yaml file can also be specified. + description: (viash < 0.9.0) Use --runner and --engine instead. Acts as a regular + expression to filter the platform ids specified in the found config files. If + this is not provided, all platforms will be used. If no platforms are defined + in a config, the native platform will be used. In addition, the path to a platform + yaml file can also be specified. + required: false + runner: + description: (viash >= 0.9.0) Acts as a regular expression to filter the runner ids specified in + the found config files. If this is not provided, all runners will be used. If + no runners are defined in a config, the executable runner will be used. + required: false + engine: + description: (viash >= 0.9.0) Acts as a regular expression to filter the engine ids specified in + the found config files. If this is not provided, all engines will be used. If + no engines are defined in a config, the native engine will be used. required: false config_mod: - description: Modify a viash config at runtime using dynamic config modding. + description: Modify a viash config at runtime using config_mod. + required: false + colorize: + description: 'Specify whether the console output should be colorized. If not specified, + we attempt to detect this automatically. Possible values are: "true", "false", + "auto".' + required: false + loglevel: + description: 'Specify the log level in us. Possible values are: "error", "warn", + "info", "debug", "trace".' required: false format: - description: 'Which output format to use. Possible values are: "yaml", "json". Default is "yaml".' + description: 'Which output format to use. Possible values are: "yaml", "json".' required: false parse_argument_groups: - description: Whether or not to postprocess each component's argument groups. + description: DEPRECATED. This is now always enabled. Whether or not to postprocess + each component's argument_groups. required: false output_file: + description: Path of a file to which the output will be written. If not set, this + action will create a file with a random name in `RUNNER_TEMP`. required: false - description: > - Path of a file to which the output will be written. If not set, - this action will create a file with a random name in `RUNNER_TEMP`. outputs: output: - description: A list of all of the components found. By default this will be a - yaml, unless the format argument was set to 'json'. + description: The output of the 'viash ns list' command, which is a list of all + of the components found. By default this will be a yaml, unless the format argument + was set to 'json'. output_file: - description: > - Path of a file to which the output was written (same as inputs.output_file). - We recommend using this property for capturing the action's output because - there is a limit in the object size that github actions can manage. Additionally, - if you use this property instead of a static file path, changing the location of - the output file will not require you to adjust settings for downstream actions as well. + description: Path of a file to which the output was written (same as `inputs.output_file``). + We recommend using this property for capturing the action's output because there + is a limit in the object size that github actions can manage. Additionally, + if you use this property instead of a static file path, changing the location + of the output file will not require you to adjust settings for downstream actions + as well. + output_matrix: + description: A simplified version of the output, which is a list of components + with fields 'name', 'namespace', 'full_name', 'config', and 'dir'. runs: using: node20 main: index.js + diff --git a/ns-list/generate_yml.R b/ns-list/generate_yml.R index 10f6da8..bcb3c95 100644 --- a/ns-list/generate_yml.R +++ b/ns-list/generate_yml.R @@ -9,15 +9,15 @@ subcommands <- c("ns", "list") #################### common code # fetch schema -out <- system("viash export cli_schema", intern = TRUE) +out <- system("viash export cli_schema --format json", intern = TRUE) schema <- jsonlite::fromJSON(out, simplifyVector = FALSE) # go down schema to find the right command for (i in seq_along(subcommands)) { - subcommand <- subcommands[[i]] - schema <- schema[[which(sapply(schema, function(li) li$name == subcommand))]] + subcommand_names <- sapply(schema, function(li) li$name) + schema <- schema[[which(subcommand_names == subcommands[[i]])]] - if (i != length(subcommands)) { + if ("subcommands" %in% names(schema) && i < length(subcommands)) { schema <- schema$subcommands } } @@ -41,13 +41,34 @@ input_values <- sapply(selected_opts, function(opt) { ) setNames(list(out), opt$name) }) + +# workaround for viash 0.9.0-RC6, should be fixed in 0.9.0 +if (!"platform" %in% names(input_values)) { + input_values$platform <- list( + description = "Acts as a regular expression to filter the platform ids specified in the found config files. If this is not provided, all platforms will be used. If no platforms are defined in a config, the native platform will be used. In addition, the path to a platform yaml file can also be specified. Deprecated in Viash 0.9.0, will be removed in Viash 1.0.0.", + required = FALSE + ) +} + +input_values$output_file <- list( + description = "Path of a file to which the output will be written. If not set, this action will create a file with a random name in `RUNNER_TEMP`.", + required = FALSE +) + out <- list( name = schema$bannerCommand, + author = "Data Intuitive", description = schema$bannerDescription, inputs = input_values, outputs = list( output = list( - description = "A list of all of the components found. By default this will be a yaml, unless the format argument was set to 'json'." + description = "The output of the 'viash ns list' command, which is a list of all of the components found. By default this will be a yaml, unless the format argument was set to 'json'." + ), + output_file = list( + description = "Path of a file to which the output was written (same as `inputs.output_file``). We recommend using this property for capturing the action's output because there is a limit in the object size that github actions can manage. Additionally, if you use this property instead of a static file path, changing the location of the output file will not require you to adjust settings for downstream actions as well." + ), + output_matrix = list( + description = "A simplified version of the output, which is a list of components with fields 'name', 'namespace', 'full_name', 'config', and 'dir'." ) ), runs = list( @@ -64,8 +85,15 @@ handlers <- list( return(result) } ) -yaml::write_yaml( +yaml_str <- yaml::as.yaml( out, - dest_file, handlers = handlers ) +out_str <- c( + "########################################################", + "# DO NOT EDIT THIS FILE MANUALLY #", + "# Please run 'Rscript ns-list/generate_yml.R' instead #", + "########################################################", + yaml_str +) +writeLines(out_str, dest_file) diff --git a/ns-list/index.js b/ns-list/index.js index 6960b2d..88f4517 100644 --- a/ns-list/index.js +++ b/ns-list/index.js @@ -24,8 +24,8 @@ async function run() { // skip "parallel" // todo: can I extract these inputNames from the action.yml? const inputNames = [ - "query", "query_namespace", "query_name", "src", "platform", - "config_mod", "format", "parse_argument_groups" + "query", "query_namespace", "query_name", "src", "runner", "engine", + "platform", "config_mod", "format", "parse_argument_groups" ] const booleanInputs = ["parse_argument_groups"] const multilineInputs = ["config_mod"] @@ -64,6 +64,41 @@ async function run() { // pass output core.setOutput("output", myOutput); + + // parse json in myOutput + let components = [] + if (core.getInput("format") == "json") { + components = JSON.parse(myOutput); + } else if (core.getInput("format") == "yaml") { + const yaml = require('js-yaml'); + components = yaml.load(myOutput); + } + + // turn into matrix + const matrix = components.map(function(component) { + const comp_name = component?.name ?? component?.functionality?.name + const comp_namespace = component?.namespace ?? component?.functionality?.namespace + const comp_fullname = comp_namespace ? comp_namespace + "/" + comp_name : comp_name + const comp_config = component?.build_info?.config ?? component?.info?.config + const comp_dir = comp_config ? path.dirname(comp_config) : undefined + // todo: if component has a test_resource with a nextflow_script, get the entrypoint? + return { + name: comp_name, + namespace: comp_namespace, + full_name: comp_fullname, + config: comp_config, + dir: comp_dir + } + }) + // return matrix as json + let matrix_str; + if (core.getInput("format") == "json") { + matrix_str = JSON.stringify(matrix); + } else if (core.getInput("format") == "yaml") { + const yaml = require('js-yaml'); + matrix_str = yaml.dump(matrix); + } + core.setOutput("output_matrix", matrix_str); } catch (error) { core.setFailed(error.message); } diff --git a/pro/build-nextflow-params/README.md b/pro/build-nextflow-params/README.md index 4ffdc26..fd56de2 100644 --- a/pro/build-nextflow-params/README.md +++ b/pro/build-nextflow-params/README.md @@ -1,47 +1,51 @@ -# build-nextflow-params - -This action will build nextflow parameter file templates for Viash workflows and components in a folder. - -## Usage -This action will look for viash configs for workflows and components in a directory, -and will generate (nextflow parameter file templates)[https://www.nextflow.io/docs/latest/cli.html#run]. -For workflows, the parameter files will saved next to the Viash config, for components will output to the target directory (see viash ns build). -This component required Viash to be installed and available in `$PATH`. -We recommend to use [viash-actions/setup](https://github.com/viash-io/viash-actions/tree/main/setup) before you run `build-nextflow-params`. -### Inputs available +# build-nextflow-params -It is _required_ to set at least one argument for `components` or for `worklows`: + -* `components` - _optional_. Location of the viash configurations for components. Will most often be a folder from a repository that has been cloned in a previous step of your workflow using [checkout](https://github.com/marketplace/actions/checkout). Ex: `example_repository/src`. +This action will build nextflow parameter file templates (see nextflow’s +-params-file option) for Viash workflows and components in a repository. +For workflows, will save the build parameter files next to the Viash +config. For components, will output to the target directory (see viash +ns build). -* `workflows` - _optional_. Location of the viash configurations for workflows. Will most often be a folder from a repository that has been cloned in a previous step of your workflow using [checkout](https://github.com/marketplace/actions/checkout). Ex: `example_repository/workfows`. +### Inputs available -* `viash_pro_token` - Viash Pro token. +- `target_dir`: *(Optional)* The target directory. If not specified, the + `target` path specified in the Viash project config (`_viash.yaml`) + will be used. If that is not specified, the default `target/nextflow` + will be used. +- `viash_pro_token`: Viash Pro token. +- `tools_version`: *(Optional)* Release of Viash tools to use. Default: + `latest`. -* `tools_version` - _optional_. Release of Viash tools to use. Will use the latest release by default. Ex: `main_build` +## Usage -## Examples +``` yaml +name: Demo of build-nextflow-params -```yaml -name: demo of build-nextflow-params on: - pull_request: + push: + jobs: demo: runs-on: ubuntu-latest steps: - - uses: viash-io/viash-actions/setup@v3 - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - name: Install Viash + uses: viash-io/viash-actions/setup@v6 + + - name: Check out repository + uses: actions/checkout@v3 - - name: Build nextflow parameter files - uses: viash-io/viash-actions/build-nextflow-params@v3 + - name: Build parameter yaml + uses: viash-io/viash-actions/pro/build-nextflow-params@v6 with: - workflows: workflows - token: ${{ secrets.GTHB_PAT }} - tools_version: 'main_build' -``` \ No newline at end of file + target: target/ + token: ${{ secrets.VIASH_PRO_TOKEN }} + tools_version: main_build +``` diff --git a/pro/build-nextflow-params/README.qmd b/pro/build-nextflow-params/README.qmd new file mode 100644 index 0000000..e2f1f0a --- /dev/null +++ b/pro/build-nextflow-params/README.qmd @@ -0,0 +1,72 @@ +--- +format: gfm +--- + +```{r} +#| include: false +library(tidyverse) +action <- yaml::read_yaml("action.yml") +action_name <- basename(getwd()) +``` + +# `r action_name` + + + +`r action$description` + +### Inputs available + +```{r} +#| echo: false +lines <- map_chr(names(action$inputs), function(name) { + input <- action$inputs[[name]] + required <- ifelse(input$required %||% FALSE, "", "_(Optional)_ ") + description <- gsub(".[ \n\r]*$", ".", input$description) + default <- ifelse(!is.null(input$default), paste0(" Default: `", input$default, "`."), "") + glue::glue("* `{name}`: {required}{description}{default}") +}) +knitr::asis_output(paste0(lines, collapse = "\n")) +``` + +```{r} +#| echo: false +lines <- map_chr(names(action$outputs), function(name) { + output <- action$outputs[[name]] + glue::glue("* `{name}`: {output$description}") +}) +if (length(lines) > 0) { + knitr::asis_output(paste0("### Outputs", "", lines, collapse = "\n")) +} +``` + + +## Usage + +```yaml +name: Demo of `r action_name` + +on: + push: + +jobs: + demo: + runs-on: ubuntu-latest + + steps: + - name: Install Viash + uses: viash-io/viash-actions/setup@v6 + + - name: Check out repository + uses: actions/checkout@v3 + + - name: `r action$name` + uses: viash-io/viash-actions/pro/build-nextflow-params@v6 + with: + target: target/ + token: ${{ secrets.VIASH_PRO_TOKEN }} + tools_version: main_build +``` diff --git a/pro/build-nextflow-params/action.yml b/pro/build-nextflow-params/action.yml index ca43517..1ed2cf7 100644 --- a/pro/build-nextflow-params/action.yml +++ b/pro/build-nextflow-params/action.yml @@ -6,12 +6,9 @@ description: > For workflows, will save the build parameter files next to the Viash config. For components, will output to the target directory (see viash ns build). inputs: - components: + target_dir: required: false - description: 'Location of the viash configuration for components.' - workflows: - required: false - description: 'Location of the viash configurations for workflows.' + description: 'The target directory. If not specified, the `target` path specified in the Viash project config (`_viash.yaml`) will be used. If that is not specified, the default `target/nextflow` will be used.' viash_pro_token: description: Viash Pro token. required: true @@ -41,61 +38,36 @@ runs: path: viash_tools ref: ${{ steps.latestrelease.outputs.releasetag }} - - uses: viash-io/viash-actions/ns-list@v5 - id: nslistcomponents - if: inputs.components - with: - src: ${{ inputs.components }} - format: "json" - platform: "nextflow" + - name: Determine target directory + shell: bash + id: defaults + run: | + TARGET_DIR=${{ inputs.target_dir }} - - uses: viash-io/viash-actions/ns-list@v5 - id: nslistworkflows - if: inputs.workflows - with: - src: ${{ inputs.workflows }} - format: "json" + if [ -z "$TARGET_DIR" ]; then + TARGET_DIR=$(yq -r '.target // "target/nextflow"' _viash.yaml) + fi + + echo "target_dir=$TARGET_DIR" >> $GITHUB_OUTPUT - name: 'Generate param files' shell: bash id: generate-params run: | command -v viash >/dev/null 2>&1 || { echo >&2 "Viash could not be found."; exit 127; } - # Check if at least some input is defined. - if [ ! -f "${{ steps.nslistcomponents.outputs.output_file }}" ] && [ ! -f "${{ steps.nslistworkflows.outputs.output_file }}" ]; then - echo "One of the workflows or components properties must be set." - exit 1; - fi - # Process components - if [ -f "${{ steps.nslistcomponents.outputs.output_file }}" ]; then - # Get config paths - component_inputs=$(jq -r '[.[] | .info.config] | join(";")' "${{ steps.nslistcomponents.outputs.output_file }}") - # Get output paths - jq_query='[.[] | "target/nextflow/" + .functionality.namespace + "/" + .functionality.name + "/nextflow_params.yaml"] | join(";")' - component_outputs_params=$(jq -r "$jq_query" "${{ steps.nslistcomponents.outputs.output_file }}") - fi - # Process workflows - if [ -f "${{ steps.nslistworkflows.outputs.output_file }}" ]; then - # Get config paths - workflow_inputs=$(jq -r '[.[] | select((.functionality.resources[].type // []) | contains("nextflow_script")) | .info.config] | join(";")' "${{ steps.nslistworkflows.outputs.output_file }}") - # Get output paths - jq_query='[.[] | select((.functionality.resources[].type // []) | contains("nextflow_script")) | .info.config | capture("^(?.*\/)").dir + "/nextflow_params.yaml"] | join(";")' - workflow_outputs_params=$(jq -r "$jq_query" "${{ steps.nslistworkflows.outputs.output_file }}") - fi - inputs=() - outputs=() - echo "Workflow inputs: $workflow_inputs" - echo "Component inputs: $component_inputs" - if [ -n "$workflow_inputs" ]; then - echo "Adding workflow inputs" - inputs+=(--input "$workflow_inputs") - outputs+=(--output "$workflow_outputs_params") - fi - if [ -n "$component_inputs" ]; then - echo "Adding component inputs" - inputs+=(--input "$component_inputs") - outputs+=(--output "$component_outputs_params") - fi - echo "Running viash_tools/target/docker/nextflow/generate_params/generate_params ${inputs[@]} ${outputs[@]}" - viash_tools/target/docker/nextflow/generate_params/generate_params "${inputs[@]}" "${outputs[@]}" - echo "Created $outputs" \ No newline at end of file + + TARGET_DIR="${{ steps.defaults.outputs.target_dir }}" + + BUILT_CONFIGS=$(find "$TARGET_DIR" -name .config.vsh.yaml) + echo "Detected configs:" + echo "$BUILT_CONFIGS" + echo + + NEXTFLOW_PARAMS=$(echo "$BUILT_CONFIGS" | sed 's/.config.vsh.yaml/nextflow_params.yaml/g') + echo "Building Nextflow param files:" + echo "$NEXTFLOW_PARAMS" + echo + + viash_tools/target/docker/nextflow/generate_params/generate_params \ + --input $(echo "$BUILT_CONFIGS" | paste -sd ";") \ + --output $(echo "$NEXTFLOW_PARAMS" | paste -sd ";") diff --git a/pro/build-nextflow-schemas/README.md b/pro/build-nextflow-schemas/README.md index c4835cc..3be36f3 100644 --- a/pro/build-nextflow-schemas/README.md +++ b/pro/build-nextflow-schemas/README.md @@ -1,47 +1,52 @@ -# build-nextflow-schemas -This action will build nextflow schemas for Viash workflows and components in a folder. -## Usage +# build-nextflow-schemas -This action will look for viash configs for workflows and components in a directory, -and will generate [nextflow pipeline JSON schemas](https://help.tower.nf/latest/pipeline-schema/overview/). -For workflows, the schemas will saved next to the Viash config, for components will output to the target directory (see viash ns build). This component required Viash to be installed and available in `$PATH`. We recommend to use [viash-actions/setup](https://github.com/viash-io/viash-actions/tree/main/setup) before you run `build-nextflow-schemas`. + -We recommend using a Linux or MacOS runner if possible. +This action will build nextflow schemas for Viash workflows and +components in a repository. For workflows, will save the build schemas +next to the Viash config. For components, will output to the target +directory (see viash ns build). ### Inputs available -It is _required_ to set at least one argument for `components` or for `worklows`: - -* `components` - _optional_. Location of the viash configurations for components. Will most often be a folder from a repository that has been cloned in a previous step of your workflow using [checkout](https://github.com/marketplace/actions/checkout). Ex: `example_repository/src`. - -* `workflows` - _optional_. Location of the viash configurations for workflows. Will most often be a folder from a repository that has been cloned in a previous step of your workflow using [checkout](https://github.com/marketplace/actions/checkout). Ex: `example_repository/workfows`. +- `target_dir`: *(Optional)* The target directory. If not specified, the + `target` path specified in the Viash project config (`_viash.yaml`) + will be used. If that is not specified, the default `target/nextflow` + will be used. +- `viash_pro_token`: Viash Pro token. +- `tools_version`: *(Optional)* Release of Viash tools to use. Default: + `latest`. +- `enable_dataset_input`: *(Optional)* enable nf-tower dataset inpu. + Default: `false`. -* `viash_pro_token` - Viash Pro token. - -* `tools_version` - _optional_. Release of Viash tools to use. Will use the latest release by default. Ex: `main_build` +## Usage -## Examples +``` yaml +name: Demo of build-nextflow-schemas -```yaml -name: demo of generate_documentation_qmd on: - pull_request: + push: + jobs: demo: runs-on: ubuntu-latest steps: - - uses: viash-io/viash-actions/setup@v3 - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - name: Install Viash + uses: viash-io/viash-actions/setup@v6 + + - name: Check out repository + uses: actions/checkout@v3 - - name: Build nextflow schemas - uses: viash-io/viash-actions/build-nextflow-schemas@v3 + - name: Build nf-tower schemas + uses: viash-io/viash-actions/pro/build-nextflow-schemas@v6 with: - workflows: workflows - token: ${{ secrets.GTHB_PAT }} - tools_version: 'main_build' -``` \ No newline at end of file + target: target/ + token: ${{ secrets.VIASH_PRO_TOKEN }} + tools_version: main_build +``` diff --git a/pro/build-nextflow-schemas/README.qmd b/pro/build-nextflow-schemas/README.qmd new file mode 100644 index 0000000..e89b88e --- /dev/null +++ b/pro/build-nextflow-schemas/README.qmd @@ -0,0 +1,72 @@ +--- +format: gfm +--- + +```{r} +#| include: false +library(tidyverse) +action <- yaml::read_yaml("action.yml") +action_name <- basename(getwd()) +``` + +# `r action_name` + + + +`r action$description` + +### Inputs available + +```{r} +#| echo: false +lines <- map_chr(names(action$inputs), function(name) { + input <- action$inputs[[name]] + required <- ifelse(input$required %||% FALSE, "", "_(Optional)_ ") + description <- gsub(".[ \n\r]*$", ".", input$description) + default <- ifelse(!is.null(input$default), paste0(" Default: `", input$default, "`."), "") + glue::glue("* `{name}`: {required}{description}{default}") +}) +knitr::asis_output(paste0(lines, collapse = "\n")) +``` + +```{r} +#| echo: false +lines <- map_chr(names(action$outputs), function(name) { + output <- action$outputs[[name]] + glue::glue("* `{name}`: {output$description}") +}) +if (length(lines) > 0) { + knitr::asis_output(paste0("### Outputs", "", lines, collapse = "\n")) +} +``` + + +## Usage + +```yaml +name: Demo of `r action_name` + +on: + push: + +jobs: + demo: + runs-on: ubuntu-latest + + steps: + - name: Install Viash + uses: viash-io/viash-actions/setup@v6 + + - name: Check out repository + uses: actions/checkout@v3 + + - name: `r action$name` + uses: viash-io/viash-actions/pro/build-nextflow-schemas@v6 + with: + target: target/ + token: ${{ secrets.VIASH_PRO_TOKEN }} + tools_version: main_build +``` diff --git a/pro/build-nextflow-schemas/action.yml b/pro/build-nextflow-schemas/action.yml index c43d3d0..c0b6f55 100644 --- a/pro/build-nextflow-schemas/action.yml +++ b/pro/build-nextflow-schemas/action.yml @@ -5,28 +5,21 @@ description: > For workflows, will save the build schemas next to the Viash config. For components, will output to the target directory (see viash ns build). inputs: - components: + target_dir: required: false - description: 'Location of the viash configuration for components.' - type: string - workflows: - required: false - description: 'Location of the viash configurations for workflows.' + description: 'The target directory. If not specified, the `target` path specified in the Viash project config (`_viash.yaml`) will be used. If that is not specified, the default `target/nextflow` will be used.' viash_pro_token: description: Viash Pro token. required: true - type: string tools_version: description: > Release of Viash tools to use. required: false default: 'latest' - type: string enable_dataset_input: description: 'enable nf-tower dataset input' required: false - default: false - type: boolean + default: "false" runs: using: 'composite' steps: @@ -49,20 +42,17 @@ runs: path: viash_tools ref: ${{ steps.latestrelease.outputs.releasetag }} - - uses: viash-io/viash-actions/ns-list@v5 - id: nslistcomponents - if: inputs.components - with: - src: ${{ inputs.components }} - format: "json" - platform: "nextflow" + - name: Determine target directory + shell: bash + id: defaults + run: | + TARGET_DIR=${{ inputs.target_dir }} + + if [ -z "$TARGET_DIR" ]; then + TARGET_DIR=$(yq -r '.target // "target/nextflow"' _viash.yaml) + fi - - uses: viash-io/viash-actions/ns-list@v5 - id: nslistworkflows - if: inputs.workflows - with: - src: ${{ inputs.workflows }} - format: "json" + echo "target_dir=$TARGET_DIR" >> $GITHUB_OUTPUT - name: 'Generate schemas' shell: bash @@ -70,56 +60,19 @@ runs: run: | command -v viash >/dev/null 2>&1 || { echo >&2 "Viash could not be found."; exit 127; } - # Check if at least some input is defined. - if [ ! -f "${{ steps.nslistcomponents.outputs.output_file }}" ] && [ ! -f "${{ steps.nslistworkflows.outputs.output_file }}" ]; then - echo "One of the workflows or components properties must be set." - exit 1; - fi - - # Process components - if [ -f "${{ steps.nslistcomponents.outputs.output_file }}" ]; then - # Get config paths - component_inputs=$(jq -r '[.[] | .info.config] | join(";")' "${{ steps.nslistcomponents.outputs.output_file }}") - - # Get output paths - jq_query='[.[] | "target/nextflow/" + .functionality.namespace + "/" + .functionality.name + "/nextflow_schema.json"] | join(";")' - component_outputs_schema=$(jq -r "$jq_query" "${{ steps.nslistcomponents.outputs.output_file }}") - fi - - # Process workflows - if [ -f "${{ steps.nslistworkflows.outputs.output_file }}" ]; then - # Get config paths - workflow_inputs=$(jq -r '[.[] | select((.functionality.resources[].type // []) | contains("nextflow_script")) | .info.config] | join(";")' "${{ steps.nslistworkflows.outputs.output_file }}") - - # Get output paths - jq_query='[.[] | select((.functionality.resources[].type // []) | contains("nextflow_script")) | .info.config | capture("^(?.*\/)").dir + "/nextflow_schema.json"] | join(";")' - workflow_outputs_schema=$(jq -r "$jq_query" "${{ steps.nslistworkflows.outputs.output_file }}") - fi + TARGET_DIR="${{ steps.defaults.outputs.target_dir }}" - inputs=() - outputs=() + BUILT_CONFIGS=$(find "$TARGET_DIR" -name .config.vsh.yaml) + echo "Detected configs:" + echo "$BUILT_CONFIGS" + echo - echo "Workflow inputs: $workflow_inputs" - echo "Component inputs: $component_inputs" - - if [ -n "$workflow_inputs" ]; then - echo "Adding workflow inputs" - inputs+=(--input "$workflow_inputs") - outputs+=(--output "$workflow_outputs_schema") - fi - - if [ -n "$component_inputs" ]; then - echo "Adding component inputs" - inputs+=(--input "$component_inputs") - outputs+=(--output "$component_outputs_schema") - fi - - extra_params=() - if [ "${{ inputs.enable_dataset_input }}" == "true" ]; then - extra_params+=("--enable_dataset_input") - fi + NEXTFLOW_SCHEMAS=$(echo "$BUILT_CONFIGS" | sed 's/.config.vsh.yaml/nextflow_schema.json/g') + echo "Building Nextflow schema files:" + echo "$NEXTFLOW_SCHEMAS" + echo - echo "Running viash_tools/target/docker/nextflow/generate_schema/generate_schema ${inputs[@]} ${outputs[@]} ${extra_params[@]}" - viash_tools/target/docker/nextflow/generate_schema/generate_schema "${inputs[@]}" "${outputs[@]}" "${extra_params[@]}" - - echo "Created $outputs_schema" \ No newline at end of file + viash_tools/target/docker/nextflow/generate_schema/generate_schema \ + --input "$(echo "$BUILT_CONFIGS" | paste -sd ";")" \ + --output "$(echo "$NEXTFLOW_SCHEMAS" | paste -sd ";")" \ + $([[ "${{ inputs.enable_dataset_input }}" == "true" ]] && echo "--enable_dataset_input") diff --git a/pro/generate-documentation-qmd/README.md b/pro/generate-documentation-qmd/README.md index f032df0..0a16432 100644 --- a/pro/generate-documentation-qmd/README.md +++ b/pro/generate-documentation-qmd/README.md @@ -1,5 +1,12 @@ + + # generate-documentation-qmd + + Create quarto markdown files for viash components or workflows, based on a jinja template. @@ -43,26 +50,33 @@ before you run `generate_documentation_qmd`. - `src`: *(Optional)* An override for the `--src` parameter in `viash ns list`. Example: `src/`. -## Examples +## Usage ``` yaml -name: demo of generate-documentation-qmd +name: Demo of generate-documentation-qmd on: - pull_request: + push: jobs: demo: runs-on: ubuntu-latest steps: - - uses: viash-io/viash-actions/setup@v3 - - uses: actions/checkout@v3 - - uses: viash-io/viash-actions/generate-documentation-qmd@v3 + - name: Install Viash + uses: viash-io/viash-actions/setup@v6 + + - name: Check out repository + uses: actions/checkout@v3 + with: + path: project + + - name: Build and publish qmd + uses: viash-io/viash-actions/generate-documentation-qmd@v6 with: - project_dir: ./ + project_dir: project src: src - output_dir: /website/components + output_dir: website dest_path: "{type}s/{namespace}/{name}.qmd" git_repo: openpipelines-bio/openpipelines token: ${{ secrets.GTHB_PAT }} diff --git a/pro/generate-documentation-qmd/README.qmd b/pro/generate-documentation-qmd/README.qmd index bd7813f..aa09f92 100644 --- a/pro/generate-documentation-qmd/README.qmd +++ b/pro/generate-documentation-qmd/README.qmd @@ -1,5 +1,4 @@ --- -title: generate-documentation-qmd format: gfm --- @@ -7,8 +6,16 @@ format: gfm #| include: false library(tidyverse) action <- yaml::read_yaml("action.yml") +action_name <- basename(getwd()) ``` +# `r action_name` + + + `r action$description` ### Inputs available @@ -40,23 +47,26 @@ if (length(lines) > 0) { ## Usage ```yaml -name: demo of generate-documentation-qmd +name: Demo of `r action_name` on: - pull_request: + push: jobs: demo: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - path: project + - name: Install Viash + uses: viash-io/viash-actions/setup@v6 - - uses: viash-io/viash-actions/setup@v3 + - name: Check out repository + uses: actions/checkout@v3 + with: + path: project - - uses: viash-io/viash-actions/generate-documentation-qmd@v3 + - name: `r action$name` + uses: viash-io/viash-actions/generate-documentation-qmd@v6 with: project_dir: project src: src diff --git a/project/build-target/README.md b/project/build-target/README.md new file mode 100644 index 0000000..8762f82 --- /dev/null +++ b/project/build-target/README.md @@ -0,0 +1,59 @@ + + +# build-target + + + +Build a target directory using the viash ns build command and create a +matrix of the executables which have Docker containers that need to be +built. + +### Inputs + +- `version`: - *optional*. Version name to use for the build. If not + specified, `${BRANCH_NAME}_build` will be used. +- `target_branch`: - *optional*. Branch to deploy to. If not specified, + `${BRANCH_NAME}_build` will be used. +- `image_tag`: - *optional*. Force the built components to use a + specific Docker image tag. +- `viash_pro_token`: - *optional*. Viash pro token + +### Outputs + +- `version`: The version that was used for the build. +- `target_branch`: The branch that the build was deployed to. +- `docker_matrix`: Matrix of executables for which Docker containers + need to be built. + +## Examples + +``` yaml +name: Demo of build-target + +on: + push: + pull_request: + +jobs: + demo: + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: viash-io/viash-actions/setup@v6 + + - name: Build target directory + id: build_target + uses: viash-io/viash-actions/build-target@v6 + + - name: Check output + run: | + # view target directory + ls -l target/ + + # do something with the output + echo "${{steps.build_target.outputs.docker_component_matrix}}" + echo "${{steps.build_target.outputs.component_matrix}}" +``` diff --git a/project/build-target/README.qmd b/project/build-target/README.qmd new file mode 100644 index 0000000..0e118b0 --- /dev/null +++ b/project/build-target/README.qmd @@ -0,0 +1,72 @@ +--- +format: gfm +--- + +```{r} +#| include: false +library(tidyverse) +action <- yaml::read_yaml("action.yml") +action_name <- basename(getwd()) +``` + +# `r action_name` + + + +`r action$description` + +### Inputs + +```{r} +#| echo: false +lines <- map_chr(names(action$inputs), function(name) { + input <- action$inputs[[name]] + required <- ifelse (input$required %||% FALSE, "required", "optional") + glue::glue("* `{name}`: - _{required}_. {input$description}") +}) +knitr::asis_output(paste0(lines, collapse = "\n")) +``` + +### Outputs + +```{r} +#| echo: false +lines <- map_chr(names(action$outputs), function(name) { + output <- action$outputs[[name]] + glue::glue("* `{name}`: {output$description}") +}) +knitr::asis_output(paste0(lines, collapse = "\n")) +``` + +## Examples + +```yaml +name: Demo of `r action_name` + +on: + push: + pull_request: + +jobs: + demo: + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: viash-io/viash-actions/setup@v6 + + - name: `r action$name` + id: build_target + uses: viash-io/viash-actions/build-target@v6 + + - name: Check output + run: | + # view target directory + ls -l target/ + + # do something with the output + echo "${{steps.build_target.outputs.docker_component_matrix}}" + echo "${{steps.build_target.outputs.component_matrix}}" +``` diff --git a/project/build-target/action.yml b/project/build-target/action.yml new file mode 100644 index 0000000..8078bf9 --- /dev/null +++ b/project/build-target/action.yml @@ -0,0 +1,142 @@ +name: Build target directory +author: Data Intuitive +description: > + Build a target directory using the viash ns build command + and create a matrix of the executables which have Docker + containers that need to be built. + +inputs: + version: + description: 'Version name to use for the build. If not specified, `${BRANCH_NAME}_build` will be used.' + required: false + target_branch: + description: 'Branch to deploy to. If not specified, `${BRANCH_NAME}_build` will be used.' + required: false + image_tag: + description: 'Force the built components to use a specific Docker image tag.' + required: false + viash_pro_token: + description: 'Viash pro token' + required: false + +outputs: + version: + description: 'The version that was used for the build.' + value: ${{ steps.defaults.outputs.version }} + target_branch: + description: 'The branch that the build was deployed to.' + value: ${{ steps.defaults.outputs.target_branch }} + docker_matrix: + description: Matrix of executables for which Docker containers need to be built. + value: ${{ steps.find_docker_executables.outputs.output_matrix }} + +runs: + using: 'composite' + steps: + - name: Determine version tag from branch name + shell: bash + id: defaults + run: | + BRANCH_NAME=$(echo $GITHUB_REF | sed 's/refs\/heads\///' | sed 's/[^a-zA-Z0-9_]/_/g') + VERSION=${{ inputs.version }} + if [ -z "$VERSION" ]; then + VERSION="${BRANCH_NAME}_build" + fi + echo "version=$VERSION" >> $GITHUB_OUTPUT + + TARGET_BRANCH=${{ inputs.target_branch }} + if [ -z "$TARGET_BRANCH" ]; then + TARGET_BRANCH="${BRANCH_NAME}_build" + fi + echo "target_branch=$TARGET_BRANCH" >> $GITHUB_OUTPUT + + TARGET_DIR=$(yq -r '.target // "target"' _viash.yaml) + echo "target_dir=$TARGET_DIR" >> $GITHUB_OUTPUT + + VIASH_VERSION=`viash -v | grep -oP 'viash \K[0-9.]+'` + if [[ -z "${{inputs.image_tag }}" ]]; then + CONFIG_MOD="" + elif [ "$(printf '%s\n' "0.9.0" "$VIASH_VERSION" | sort -V | head -n1)" == "0.9.0" ]; then + CONFIG_MOD='.platforms[.type == "docker"].target_tag := "${{ inputs.image_tag }}"' + else + CONFIG_MOD='.engines[.type == "docker"].target_tag := "${{ inputs.image_tag }}"' + fi + echo "config_mod=$CONFIG_MOD" >> $GITHUB_OUTPUT + + - name: Remove target folder from .gitignore + shell: bash + run: | + TARGET_DIR="${{ steps.defaults.outputs.target_dir }}" + sed -i "/^\\/*${TARGET_DIR}.*/d" .gitignore + + - name: Set version in _viash.yaml + shell: bash + run: | + yq eval '.version = "${{ steps.defaults.outputs.version }}"' -i _viash.yaml + + - uses: viash-io/viash-actions/ns-build@v6 + with: + parallel: true + config_mod: ${{ steps.defaults.outputs.config_mod }} + + - name: Build nextflow schemas + if: ${{ inputs.viash_pro_token != '' }} + uses: viash-io/viash-actions/pro/build-nextflow-params@v6 + with: + target_dir: ${{ steps.defaults.outputs.target_dir }} + viash_pro_token: ${{ inputs.viash_pro_token }} + tools_version: 'main_build' + enable_dataset_input: true + + - name: Build parameter files + if: ${{ inputs.viash_pro_token != '' }} + uses: viash-io/viash-actions/pro/build-nextflow-params@v6 + with: + target_dir: ${{ steps.defaults.outputs.target_dir }} + viash_pro_token: ${{ inputs.viash_pro_token }} + tools_version: 'main_build' + + - name: Find Docker Executables that need to be built + id: find_docker_executables + shell: bash + run: | + TARGET_DIR="${{ steps.defaults.outputs.target_dir }}" + + BUILT_CONFIGS=`find "$TARGET_DIR" -name .config.vsh.yaml` + + OUTPUT="[" + + for yaml in $BUILT_CONFIGS; do + RUNNER=`yq -r ".build_info.runner" $yaml` + ENGINE=`yq -r ".build_info.engine // .info.platform" $yaml` + + # skip if not an executable or does not use a docker engine + if [ "$RUNNER" != "executable" ] || [[ "$ENGINE" != *"docker"* ]]; then + continue + fi + NAME=`yq -r ".name // .functionality.name" $yaml` + NAMESPACE=`yq -r '.namespace // .functionality.namespace // ""' $yaml` + if [ -z "$NAMESPACE" ]; then + FULL_NAME="$NAME" + else + FULL_NAME="$NAMESPACE/$NAME" + fi + CONFIG=`yq -r ".build_info.config // .info.config" $yaml` + SRC_DIR=`dirname $CONFIG` + TARGET_DIR=`dirname $yaml` + EXECUTABLE=`yq -r ".build_info.executable // .info.executable" $yaml` + + echo "Found Docker executable: $FULL_NAME" + + JSON_FIELDS="" + for key in NAME NAMESPACE RUNNER ENGINE CONFIG EXECUTABLE SRC_DIR TARGET_DIR FULL_NAME; do + key_lower=`echo $key | tr '[:upper:]' '[:lower:]'` + JSON_FIELDS="$JSON_FIELDS\"${key_lower}\":\"${!key}\"," + done + + OUTPUT="$OUTPUT{${JSON_FIELDS%,}}," + done + + OUTPUT="${OUTPUT%,}]" + + echo "output_matrix=$OUTPUT" >> $GITHUB_OUTPUT diff --git a/project/check-concurrent-pr/README.md b/project/check-concurrent-pr/README.md new file mode 100644 index 0000000..609639e --- /dev/null +++ b/project/check-concurrent-pr/README.md @@ -0,0 +1,47 @@ + + +# check-concurrent-pr + + + +Assuming that an action is triggered by both a push event and a pull +request, this action will: + +- Return ‘true’ if the branch is the default branch +- Return ‘true’ if the event is not a push event +- Return ‘true’ if the commit message contains ‘ci force’ +- Return ‘false’ if a PR exists for the branch, else ‘true’ + +### Inputs + +### Outputs + +- `run`: Returns “true” if the branch has a PR and this run was not + triggered by a push event, else “false”. + +## Examples + +``` yaml +name: Demo of check-concurrent-pr + +on: + push: + pull_request: + +jobs: + demo: + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v3 + + - name: Check for concurrent PRs + uses: viash-io/viash-actions/is-pr@v6 + id: is_pr + + - run: | + echo "Is PR: ${{ steps.is_pr.outputs.check }}" +``` diff --git a/project/check-concurrent-pr/README.qmd b/project/check-concurrent-pr/README.qmd new file mode 100644 index 0000000..f86e30e --- /dev/null +++ b/project/check-concurrent-pr/README.qmd @@ -0,0 +1,67 @@ +--- +format: gfm +--- + +```{r} +#| include: false +library(tidyverse) +action <- yaml::read_yaml("action.yml") +action_name <- basename(getwd()) +``` + +# `r action_name` + + + +`r action$description` + +### Inputs + +```{r} +#| echo: false +lines <- map_chr(names(action$inputs), function(name) { + input <- action$inputs[[name]] + required <- ifelse (input$required %||% FALSE, "required", "optional") + glue::glue("* `{name}`: - _{required}_. {input$description}") +}) +knitr::asis_output(paste0(lines, collapse = "\n")) +``` + +### Outputs + +```{r} +#| echo: false +lines <- map_chr(names(action$outputs), function(name) { + output <- action$outputs[[name]] + glue::glue("* `{name}`: {output$description}") +}) +knitr::asis_output(paste0(lines, collapse = "\n")) +``` + +## Examples + +```yaml +name: Demo of `r action_name` + +on: + push: + pull_request: + +jobs: + demo: + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v3 + + - name: `r action$name` + uses: viash-io/viash-actions/is-pr@v6 + id: is_pr + + - run: | + echo "Is PR: ${{ steps.is_pr.outputs.check }}" +``` + diff --git a/project/check-concurrent-pr/action.yml b/project/check-concurrent-pr/action.yml new file mode 100644 index 0000000..d3c3917 --- /dev/null +++ b/project/check-concurrent-pr/action.yml @@ -0,0 +1,47 @@ +name: Check for concurrent PRs +author: Data Intuitive +description: > + Assuming that an action is triggered by both a push event and a pull request, this action will: + + - Return 'true' if the branch is the default branch + - Return 'true' if the event is not a push event + - Return 'true' if the commit message contains 'ci force' + - Return 'false' if a PR exists for the branch, else 'true' +outputs: + run: + description: 'Returns "true" if the branch has a PR and this run was not triggered by a push event, else "false".' + value: ${{ steps.check.outputs.run }} +runs: + using: 'composite' + steps: + - name: 'Check whether the branch has a PR and if this run was triggered by a push event' + shell: bash + id: check + run: | + # if the branch is the default branch, always run + if [[ "$GITHUB_REF_NAME" == "$GITHUB_DEFAULT_BRANCH" ]]; then + echo "run=true" >> $GITHUB_OUTPUT + exit 0 + fi + + # if the event is not a push event, always run + if [[ "${{ github.event_name == 'push' }}" == "false" ]]; then + echo "run=true" >> $GITHUB_OUTPUT + exit 0 + fi + + # if the commit message contains 'ci force', always run + if [[ "${{ contains(github.event.head_commit.message, 'ci force') }}" == "true" ]]; then + echo "run=true" >> $GITHUB_OUTPUT + exit 0 + fi + + # if a PR exists for the branch, do not run + pull_request=$(gh pr list -R ${{ github.repository }} -H ${{ github.ref_name }} --json url --state open --limit 1 | jq '.[0].url') + if [[ "$pull_request" == "null" ]]; then + echo "run=true" >> $GITHUB_OUTPUT + else + echo "run=false" >> $GITHUB_OUTPUT + fi + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/project/detect-changed-components/README.md b/project/detect-changed-components/README.md index 0ff2b45..ee95fd8 100644 --- a/project/detect-changed-components/README.md +++ b/project/detect-changed-components/README.md @@ -1,23 +1,31 @@ -# Detect components with changed files -Filter a json file produced by viash ns list on whether -the component has been changed with respect to the -default branch. -## Inputs +# detect-changed-components - * `input_file`: (Required) Path to a viash ns list json file. + -## Outputs +Filter a json file produced by viash ns list on whether the component +has been changed with respect to the default branch. - * `output_file`: Path to the filtered viash ns list json file, - filtered on whether one of the resources of the component has - been changed or not. +### Inputs + +- `input_file`: - *required*. Path to a viash ns list json file + +### Outputs + +- `output_file`: Path to the filtered viash ns list json file, filtered + on whether one of the resources of the component has been changed or + not. + +- `output_matrix`: Matrix of components that have changed resources. ## Examples -```yaml -name: demo of detect-changed +``` yaml +name: Demo of detect-changed-components on: push: @@ -27,15 +35,16 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - - id: ns_list - uses: viash-io/viash-actions/ns-list@v3 - with: - format: json - - - id: ns_list_changed - uses: viash-io/viash-actions/detect-changed-components@v3 - with: - input_file: ${{ steps.ns_list.outputs.output_file }} + - uses: actions/checkout@v3 + + - id: ns_list + uses: viash-io/viash-actions/ns-list@v6 + with: + format: json + + - name: Detect components with changed files + id: ns_list_changed + uses: viash-io/viash-actions/detect-changed-components@v6 + with: + input_file: ${{ steps.ns_list.outputs.output_file }} ``` diff --git a/project/detect-changed-components/README.qmd b/project/detect-changed-components/README.qmd new file mode 100644 index 0000000..cf75b2a --- /dev/null +++ b/project/detect-changed-components/README.qmd @@ -0,0 +1,69 @@ +--- +format: gfm +--- + +```{r} +#| include: false +library(tidyverse) +action <- yaml::read_yaml("action.yml") +action_name <- basename(getwd()) +``` + +# `r action_name` + + + +`r action$description` + +### Inputs + +```{r} +#| echo: false +lines <- map_chr(names(action$inputs), function(name) { + input <- action$inputs[[name]] + required <- ifelse (input$required %||% FALSE, "required", "optional") + glue::glue("* `{name}`: - _{required}_. {input$description}") +}) +knitr::asis_output(paste0(lines, collapse = "\n")) +``` + +### Outputs + +```{r} +#| echo: false +lines <- map_chr(names(action$outputs), function(name) { + output <- action$outputs[[name]] + glue::glue("* `{name}`: {output$description}") +}) +knitr::asis_output(paste0(lines, collapse = "\n")) +``` + +## Examples + +```yaml +name: Demo of `r action_name` + +on: + push: + +jobs: + demo: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + + - id: ns_list + uses: viash-io/viash-actions/ns-list@v6 + with: + format: json + + - name: `r action$name` + id: ns_list_changed + uses: viash-io/viash-actions/detect-changed-components@v6 + with: + input_file: ${{ steps.ns_list.outputs.output_file }} +``` diff --git a/project/detect-changed-components/action.yml b/project/detect-changed-components/action.yml index a873898..1cfda0e 100644 --- a/project/detect-changed-components/action.yml +++ b/project/detect-changed-components/action.yml @@ -14,9 +14,20 @@ outputs: Path to the filtered viash ns list json file, filtered on whether one of the resources of the component has been changed or not. value: ${{ steps.filter.outputs.output_file }} + output_matrix: + description: | + Matrix of components that have changed resources. + value: ${{ steps.set_matrix.outputs.matrix }} runs: using: 'composite' steps: + - name: Get head git commit message + shell: bash + id: get_head_commit_message + run: | + head_commit_message=$(git show -s --format=%s ${{ github.event.pull_request.head.sha || github.sha }}) + echo "HEAD_COMMIT_MESSAGE=$head_commit_message" >> "$GITHUB_OUTPUT" + - name: Get changed files id: changed-files uses: tj-actions/changed-files@v42.0.5 @@ -87,22 +98,33 @@ runs: # iterate over the components for component in "${components[@]}"; do - config_path=$(jq -cr '.info.config' <<< "$component" ) + config_path=$(jq -cr '.build_info.config // .info.config' <<< "$component" ) + config_dir=`dirname "$config_path"` echo "Checking '$config_path'" # get the components resources readarray -t resources < <( jq -cr ' ( - [.info.config] + + # viash config + [.build_info.config // (.info.config)] + + # viash >= 0.9 resources [ - .functionality.resources[], .functionality.test_resources[] + ((.resources // []) + (.test_resources // []))[] + | .path + | sub("^(?[^/])"; "'$config_dir'/" + .match) + | sub("^/"; "") + ] + + # viash < 0.9 resources + [ + (.functionality.resources // [])[], (.functionality.test_resources // [])[] | select(.path and .parent) | (.parent | sub("^file:(//)?"; "") | sub("/[^/]*$"; "")) + "/" + .path ] )[] ' <<< "$component" ) + # check if resource is in the list of changed resources should_add_component=0 for resource in "${resources[@]}"; do @@ -111,6 +133,7 @@ runs: echo "Resource $resource does not exist." exit 1 fi + # resolve paths wrt current dir (only needed for viash < 0.9) resource_real_path=$(realpath "$resource") resource_arr=("$resource_real_path") # Resolve resources that are a directory into files @@ -140,4 +163,20 @@ runs: # reformat as json and write to file jq -cs '.' <<< "${output_array[*]}" > "$output_file" - echo "output_file=$output_file" >> $GITHUB_OUTPUT \ No newline at end of file + echo "output_file=$output_file" >> $GITHUB_OUTPUT + + - id: set_matrix + shell: bash + run: | + comp_file="${{ contains(steps.get_head_commit_message.outputs.HEAD_COMMIT_MESSAGE, 'ci force') && inputs.input_file || steps.filter.outputs.output_file }}" + matrix=$( + jq -c '[ .[] | + { + "name": (.name // .functionality.name), + "namespace": (.namespace // .functionality.namespace // ""), + "config": (.build_info.config // .info.config), + "dir": (.build_info.config // .info.config) | capture("^(?.*\/)").dir + } + ]' $comp_file + ) + echo "matrix=$matrix" >> $GITHUB_OUTPUT diff --git a/project/sync-and-cache-s3/README.md b/project/sync-and-cache-s3/README.md index 30c830a..29ab670 100644 --- a/project/sync-and-cache-s3/README.md +++ b/project/sync-and-cache-s3/README.md @@ -1,35 +1,45 @@ -# Sync and Cache S3 Bucket -A GitHub Action to sync an S3 bucket to a local folder and create a cache. -For now, this will only be able to sync public S3 buckets. +# sync-and-cache-s3 -## Inputs + - * `s3_bucket`: (Required) Path to an S3 bucket. - * `dest_path`: (Required) Where to store the data. - * `cache_key_prefix`: (Optional) A prefix for the cache hash key. Prefix is also used for restoring stale cache if no cache hit occurred for key. Default value is `cachekey__`. +Sync an S3 bucket to a folder and create a cache. -## Outputs +### Inputs - * `cache_key`: The caching key, which is the hash of the contents of the S3 bucket. +- `s3_bucket`: - *required*. Path to an S3 bucket. +- `dest_path`: - *required*. Where to store the data. +- `cache_key_prefix`: - *optional*. A prefix for the cache hash key. + Prefix is also used for restoring stale cache if no cache hit occurred + for key. + +### Outputs + +- `cache_key`: Caching key. ## Examples -```yaml -name: demo of sync-and-cache-s3 +``` yaml +name: Demo of sync-and-cache-s3 on: push: + pull_request: jobs: demo: runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: viash-io/viash-actions/sync-and-cache-s3 - with: - s3_bucket: s3:/mybucket/resources - dest_path: resources + - name: Check out repository + uses: actions/checkout@v3 + + - name: Sync and cache an S3 bucket + uses: viash-io/viash-actions/sync-and-cache-s3 + with: + s3_bucket: s3:/mybucket/resources + dest_path: resources ``` diff --git a/project/sync-and-cache-s3/README.qmd b/project/sync-and-cache-s3/README.qmd new file mode 100644 index 0000000..c1bfc68 --- /dev/null +++ b/project/sync-and-cache-s3/README.qmd @@ -0,0 +1,66 @@ +--- +format: gfm +--- + +```{r} +#| include: false +library(tidyverse) +action <- yaml::read_yaml("action.yml") +action_name <- basename(getwd()) +``` + +# `r action_name` + + + +`r action$description` + +### Inputs + +```{r} +#| echo: false +lines <- map_chr(names(action$inputs), function(name) { + input <- action$inputs[[name]] + required <- ifelse (input$required %||% FALSE, "required", "optional") + glue::glue("* `{name}`: - _{required}_. {input$description}") +}) +knitr::asis_output(paste0(lines, collapse = "\n")) +``` + +### Outputs + +```{r} +#| echo: false +lines <- map_chr(names(action$outputs), function(name) { + output <- action$outputs[[name]] + glue::glue("* `{name}`: {output$description}") +}) +knitr::asis_output(paste0(lines, collapse = "\n")) +``` + +## Examples + +```yaml +name: Demo of `r action_name` + +on: + push: + pull_request: + +jobs: + demo: + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v3 + + - name: `r action$name` + uses: viash-io/viash-actions/sync-and-cache-s3 + with: + s3_bucket: s3:/mybucket/resources + dest_path: resources +``` + diff --git a/project/sync-and-cache-s3/action.yml b/project/sync-and-cache-s3/action.yml index fcd725f..c09496e 100644 --- a/project/sync-and-cache-s3/action.yml +++ b/project/sync-and-cache-s3/action.yml @@ -56,4 +56,4 @@ runs: - name: List resources shell: bash run: | - tree ${{ inputs.dest_path }} -L 3 \ No newline at end of file + tree ${{ inputs.dest_path }} -L 3 diff --git a/project/update-docker-engine/README.md b/project/update-docker-engine/README.md new file mode 100644 index 0000000..5999562 --- /dev/null +++ b/project/update-docker-engine/README.md @@ -0,0 +1,36 @@ + + +# update-docker-engine + + + +Install the latest Docker Engine or update an existing one to the latest +available version. Note that this action only works on Ubuntu based +images. + +### Inputs + +### Outputs + +## Examples + +``` yaml +name: Demo of update-docker-engine + +on: + push: + pull_request: + +jobs: + demo: + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v3 + + - name: Update Docker Engine + uses: viash-io/viash-actions/update-docker-engine@v6 +``` diff --git a/project/update-docker-engine/README.qmd b/project/update-docker-engine/README.qmd new file mode 100644 index 0000000..6dc9829 --- /dev/null +++ b/project/update-docker-engine/README.qmd @@ -0,0 +1,63 @@ +--- +format: gfm +--- + +```{r} +#| include: false +library(tidyverse) +action <- yaml::read_yaml("action.yml") +action_name <- basename(getwd()) +``` + +# `r action_name` + + + +`r action$description` + +### Inputs + +```{r} +#| echo: false +lines <- map_chr(names(action$inputs), function(name) { + input <- action$inputs[[name]] + required <- ifelse (input$required %||% FALSE, "required", "optional") + glue::glue("* `{name}`: - _{required}_. {input$description}") +}) +knitr::asis_output(paste0(lines, collapse = "\n")) +``` + +### Outputs + +```{r} +#| echo: false +lines <- map_chr(names(action$outputs), function(name) { + output <- action$outputs[[name]] + glue::glue("* `{name}`: {output$description}") +}) +knitr::asis_output(paste0(lines, collapse = "\n")) +``` + +## Examples + +```yaml +name: Demo of `r action_name` + +on: + push: + pull_request: + +jobs: + demo: + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v3 + + - name: `r action$name` + uses: viash-io/viash-actions/project/update-docker-engine@v6 +``` + diff --git a/update-docker-engine/action.yml b/project/update-docker-engine/action.yml similarity index 96% rename from update-docker-engine/action.yml rename to project/update-docker-engine/action.yml index 33cd328..9eed2f2 100644 --- a/update-docker-engine/action.yml +++ b/project/update-docker-engine/action.yml @@ -19,4 +19,4 @@ runs: "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | \ sudo tee /etc/apt/sources.list.d/docker.list > /dev/null sudo apt-get update - sudo apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin \ No newline at end of file + sudo apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin diff --git a/setup/README.md b/setup/README.md index 767ed40..4536ebf 100644 --- a/setup/README.md +++ b/setup/README.md @@ -1,54 +1,51 @@ -# setup -Setup a Viash release (https://github.com/viash-io/viash/releases) using GitHub Actions. -## Usage +# setup -This action will use `wget get.viash.io` to install Viash in the `$HOME/.local/bin` directory and add that folder to the PATH. + -We recommend using a Linux or MacOS runner if possible. +This action will setup Viash from the git repository +https://github.com/viash-io/viash/ -### Inputs available +### Inputs -* `version` - _optional_. If provided, the specific Viash version will be installed. Ex: `version: 0.6.0`. +- `version`: - *optional*. The version of Viash to use. Either a release + tag (eg. 0.6.0) or “@develop” for latest development version. If + missing, uses latest stable version. - ```yaml - steps: - - uses: viash-io/viash-actions/setup@v3 - with: - version: 0.6.0 - ``` - - If not provided, `setup` will use the latest _released_ version of Viash. - - If the latest development build is desired, use `version: develop`. +### Outputs + +- `version`: The installed version of viash. ## Examples -```yaml -name: demo of setup +``` yaml +name: Demo of setup -on: - branch: main +on: + push: + pull_request: jobs: viash-linux: runs-on: ubuntu-latest steps: - - uses: viash-io/viash-actions/setup@v3 + - uses: viash-io/viash-actions/setup@v6 - run: | viash --version viash-windows: runs-on: windows-latest steps: - - uses: viash-io/viash-actions/setup@v3 + - uses: viash-io/viash-actions/setup@v6 - run: | viash --version viash-macos: runs-on: macos-latest steps: - - uses: viash-io/viash-actions/setup@v3 + - uses: viash-io/viash-actions/setup@v6 - run: | viash --version ``` - diff --git a/setup/README.qmd b/setup/README.qmd new file mode 100644 index 0000000..a29c416 --- /dev/null +++ b/setup/README.qmd @@ -0,0 +1,73 @@ +--- +format: gfm +--- + +```{r} +#| include: false +library(tidyverse) +action <- yaml::read_yaml("action.yml") +action_name <- basename(getwd()) +``` + +# `r action_name` + + + +`r action$description` + +### Inputs + +```{r} +#| echo: false +lines <- map_chr(names(action$inputs), function(name) { + input <- action$inputs[[name]] + required <- ifelse (input$required %||% FALSE, "required", "optional") + glue::glue("* `{name}`: - _{required}_. {input$description}") +}) +knitr::asis_output(paste0(lines, collapse = "\n")) +``` + +### Outputs + +```{r} +#| echo: false +lines <- map_chr(names(action$outputs), function(name) { + output <- action$outputs[[name]] + glue::glue("* `{name}`: {output$description}") +}) +knitr::asis_output(paste0(lines, collapse = "\n")) +``` + +## Examples + +```yaml +name: Demo of `r action_name` + +on: + push: + pull_request: + +jobs: + viash-linux: + runs-on: ubuntu-latest + steps: + - uses: viash-io/viash-actions/setup@v6 + - run: | + viash --version + viash-windows: + runs-on: windows-latest + steps: + - uses: viash-io/viash-actions/setup@v6 + - run: | + viash --version + viash-macos: + runs-on: macos-latest + steps: + - uses: viash-io/viash-actions/setup@v6 + - run: | + viash --version +``` + diff --git a/setup/action.yml b/setup/action.yml index fa1bd32..1164622 100644 --- a/setup/action.yml +++ b/setup/action.yml @@ -3,7 +3,7 @@ author: Data Intuitive description: 'This action will setup Viash from the git repository https://github.com/viash-io/viash/' inputs: version: - description: 'The version of Viash to use. Either a release tag (eg. 0.6.0) or "develop" for latest built dev version. If missing, uses latest stable version.' + description: 'The version of Viash to use. Either a release tag (eg. 0.6.0) or "@develop" for latest development version. If missing, uses latest stable version.' required: false outputs: version: diff --git a/update-docker-engine/README.md b/update-docker-engine/README.md deleted file mode 100644 index 0be0b35..0000000 --- a/update-docker-engine/README.md +++ /dev/null @@ -1,23 +0,0 @@ -# Install or Update Latest Available Docker - -Install the latest Docker Engine or update an existing one to the latest available version. -Note that this action only works on Ubuntu based images. - - -## Examples - -```yaml -name: demo of update-docker-engine - -on: - push: - -jobs: - demo: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - - - uses: viash-io/viash-actions/update-docker-engine@v3 -``` diff --git a/viash-hub/deploy/README.md b/viash-hub/deploy/README.md deleted file mode 100644 index ebc4afc..0000000 --- a/viash-hub/deploy/README.md +++ /dev/null @@ -1,53 +0,0 @@ -# Deploy - -This action will deploy/push build artifacts to Viash-Hub. -The functionality is greatly inspired by `peaceiris/actions-gh-pages@v3` but tweaked and minimized for our use case. - -## Usage - -This action will deploy/pull a branch from a repository and replace all content with the available build artifacts from the pipeline. - -If the `github_repo` is specified, the action will first pull the branch from the GitHub repository and then push it to the Viash-Hub repository. -If the `github_repo` is not specified, the action will first pull the branch from the Viash-Hub repository and then push it back to the Viash-Hub repository. - -### Inputs available - - `github_token`: GitHub token. - `viash_hub_token`: Viash-Hub token. - `viash_hub_repo`: user/repo_name on Viash-Hub. - `github_repo`: user/repo_name on GitHub from where to get the base branch information. - `src_branch`: Source Git branch name. - `dest_branch`: Destination Git branch name. - `commit_message`: Override the commit message for the deployment. - -## Examples - -```yaml -name: demo of viash-hub/push -on: - pull_request: -jobs: - demo: - runs-on: ubuntu-latest - - steps: - - uses: viash-io/viash-actions/setup@v4 - - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - uses: viash-io/viash-actions/ns-build@v4 - with: - parallel: true - - - name: Deploy to Viash-Hub - uses: viash-io/viash-actions/viash-hub/push@v4 - with: - github_token: ${{ github.token }} - viash_hub_token: ${{ secrets.VIASHHUB_PAT }} - viash_hub_repo: openpipelines-bio/openpipeline - github_repo: openpipelines-bio/openpipeline - src_branch: main_build - dest_branch: main_build -``` \ No newline at end of file diff --git a/viash-hub/deploy/action.yml b/viash-hub/deploy/action.yml deleted file mode 100644 index afbb16e..0000000 --- a/viash-hub/deploy/action.yml +++ /dev/null @@ -1,67 +0,0 @@ -name: 'Deploy to viash-hub' -author: Data Intuitive -description: > - This action will push build artifacts to Viash-Hub. -inputs: - github_token: - required: false - description: GitHub token. - viash_hub_token: - description: Viash-Hub token. - required: true - viash_hub_repo: - description: user/repo_name on Viash-Hub. - required: true - github_repo: - description: user/repo_name on GitHub from where to get the base branch information. - required: false - src_branch: - description: Source Git branch name. - required: true - dest_branch: - description: Destination Git branch name. - required: true - commit_message: - description: User defined custom commit message. - required: false -runs: - using: 'composite' - steps: - - name: 'Determine temporary working directory' - shell: bash - id: workdir - run: | - repo_dir=/home/runner/viash-hub-push-`date +%s` - echo "working directory: $repo_dir" - echo "REPO_DIR=$repo_dir" >> $GITHUB_ENV - - name: 'Pull existing branch from GitHub' - if: ${{ inputs.github_repo != '' }} - shell: bash - id: pullfromgithub - run: | - # removed --depth=1 given that we rewrite the remote history. - git clone --single-branch --branch ${{ inputs.src_branch }} --no-checkout https://x-access-token:${{ inputs.github_token }}@github.com/${{ inputs.github_repo }}.git ${{ env.REPO_DIR }} - - name: 'Pull existing branch from ViashHub' - if: ${{ inputs.github_repo == '' }} - shell: bash - id: pullfromviashhub - run: | - git clone --single-branch --branch ${{ inputs.src_branch }} --depth=1 --no-checkout https://x-access-token:${{ inputs.viash_hub_token }}@viash-hub.com/${{ inputs.viash_hub_repo }}.git ${{ env.REPO_DIR }} - - name: 'Push to Viash-Hub' - shell: bash - id: pushviashhub - run: | - cd ${{ env.REPO_DIR }} - rsync -a --exclude='.git' ${{ github.workspace }}/ . - git add --all - git remote add viash-hub https://x-access-token:${{ inputs.viash_hub_token }}@viash-hub.com/${{ inputs.viash_hub_repo }}.git - git config user.name ${{ github.actor }} - git config user.email ${{ github.actor }}@users.noreply.github.com - if [ -z "${{ inputs.commit_message }}" ] - then - git commit -m "deploy: ${{ github.sha }}" - else - git commit -m "${{ inputs.commit_message }}" - fi - git push -f viash-hub ${{ inputs.dest_branch }} - \ No newline at end of file