Skip to content

Commit

Permalink
Merge pull request #2680 from mashehu/add-rocreate
Browse files Browse the repository at this point in the history
Add new command `nf-core rocrate` to create a Research Object (RO) crate for a pipeline
  • Loading branch information
mashehu authored Oct 23, 2024
2 parents 1ef9134 + ad7b895 commit 796c219
Show file tree
Hide file tree
Showing 12 changed files with 679 additions and 27 deletions.
6 changes: 6 additions & 0 deletions .github/actions/create-lint-wf/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,12 @@ runs:
run: find nf-core-testpipeline -type f -exec sed -i 's/zenodo.XXXXXX/zenodo.123456/g' {} \;
working-directory: create-lint-wf

# Add empty ro-crate file
- name: add empty ro-crate file
shell: bash
run: touch nf-core-testpipeline/ro-crate-metadata.json
working-directory: create-lint-wf

# Run nf-core pipelines linting
- name: nf-core pipelines lint
shell: bash
Expand Down
8 changes: 8 additions & 0 deletions .github/workflows/create-test-lint-wf-template.yml
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,9 @@ jobs:
run: |
cd create-test-lint-wf
nf-core --log-file log.txt pipelines create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface" --template-yaml template_skip_${{ matrix.TEMPLATE }}.yml
# fake ro-crate
touch my-prefix-testpipeline/ro-crate-metadata.json
git commit -am "add ro-crate"
- name: run the pipeline
run: |
Expand Down Expand Up @@ -147,6 +150,11 @@ jobs:
run: find my-prefix-testpipeline -type f -exec sed -i 's/zenodo.XXXXXX/zenodo.123456/g' {} \;
working-directory: create-test-lint-wf

# Add empty ro-crate file
- name: add empty ro-crate file
run: touch my-prefix-testpipeline/ro-crate-metadata.json
working-directory: create-test-lint-wf

# Run nf-core linting
- name: nf-core pipelines lint
run: nf-core --log-file log.txt --hide-progress pipelines lint --dir my-prefix-testpipeline --fail-warned
Expand Down
44 changes: 42 additions & 2 deletions nf_core/__main__.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import logging
import os
import sys
from pathlib import Path

import rich
import rich.console
Expand Down Expand Up @@ -35,6 +36,7 @@
pipelines_launch,
pipelines_lint,
pipelines_list,
pipelines_rocrate,
pipelines_schema_build,
pipelines_schema_docs,
pipelines_schema_lint,
Expand Down Expand Up @@ -85,7 +87,7 @@
},
{
"name": "For developers",
"commands": ["create", "lint", "bump-version", "sync", "schema", "create-logo"],
"commands": ["create", "lint", "bump-version", "sync", "schema", "rocrate", "create-logo"],
},
],
"nf-core modules": [
Expand Down Expand Up @@ -569,6 +571,44 @@ def command_pipelines_list(ctx, keywords, sort, json, show_archived):
pipelines_list(ctx, keywords, sort, json, show_archived)


# nf-core pipelines rocrate
@pipelines.command("rocrate")
@click.argument(
"pipeline_dir",
type=click.Path(exists=True),
default=Path.cwd(),
required=True,
metavar="<pipeline directory>",
)
@click.option(
"-j",
"--json_path",
default=Path.cwd(),
type=str,
help="Path to save RO Crate metadata json file to",
)
@click.option("-z", "--zip_path", type=str, help="Path to save RO Crate zip file to")
@click.option(
"-pv",
"--pipeline_version",
type=str,
help="Version of pipeline to use for RO Crate",
default="",
)
@click.pass_context
def rocrate(
ctx,
pipeline_dir: str,
json_path: str,
zip_path: str,
pipeline_version: str,
):
"""
Make an Research Object Crate
"""
pipelines_rocrate(ctx, pipeline_dir, json_path, zip_path, pipeline_version)


# nf-core pipelines sync
@pipelines.command("sync")
@click.pass_context
Expand Down Expand Up @@ -1692,7 +1732,7 @@ def command_schema_validate(pipeline, params):
@click.option(
"--url",
type=str,
default="https://nf-co.re/pipeline_schema_builder",
default="https://oldsite.nf-co.re/pipeline_schema_builder",
help="Customise the builder URL (for development work)",
)
def command_schema_build(directory, no_prompts, web_only, url):
Expand Down
28 changes: 28 additions & 0 deletions nf_core/commands_pipelines.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
import os
import sys
from pathlib import Path
from typing import Optional, Union

import rich

Expand Down Expand Up @@ -277,6 +278,33 @@ def pipelines_list(ctx, keywords, sort, json, show_archived):
stdout.print(list_workflows(keywords, sort, json, show_archived))


# nf-core pipelines rocrate
def pipelines_rocrate(
ctx,
pipeline_dir: Union[str, Path],
json_path: Optional[Union[str, Path]],
zip_path: Optional[Union[str, Path]],
pipeline_version: str,
) -> None:
from nf_core.pipelines.rocrate import ROCrate

if json_path is None and zip_path is None:
log.error("Either `--json_path` or `--zip_path` must be specified.")
sys.exit(1)
else:
pipeline_dir = Path(pipeline_dir)
if json_path is not None:
json_path = Path(json_path)
if zip_path is not None:
zip_path = Path(zip_path)
try:
rocrate_obj = ROCrate(pipeline_dir, pipeline_version)
rocrate_obj.create_rocrate(pipeline_dir, json_path=json_path, zip_path=zip_path)
except (UserWarning, LookupError, FileNotFoundError) as e:
log.error(e)
sys.exit(1)


# nf-core pipelines sync
def pipelines_sync(ctx, directory, from_branch, pull_request, github_repository, username, template_yaml, force_pr):
"""
Expand Down
4 changes: 2 additions & 2 deletions nf_core/components/info.py
Original file line number Diff line number Diff line change
Expand Up @@ -211,9 +211,9 @@ def get_local_yaml(self) -> Optional[Dict]:
return yaml.safe_load(fh)
log.debug(f"{self.component_type[:-1].title()} '{self.component}' meta.yml not found locally")

return None
return {}

def get_remote_yaml(self) -> Optional[dict]:
def get_remote_yaml(self) -> Optional[Dict]:
"""Attempt to get the meta.yml file from a remote repo.
Returns:
Expand Down
4 changes: 3 additions & 1 deletion nf_core/modules/modules_json.py
Original file line number Diff line number Diff line change
Expand Up @@ -1119,8 +1119,10 @@ def dump(self, run_prettier: bool = False) -> None:
"""
Sort the modules.json, and write it to file
"""
# Sort the modules.json
if self.modules_json is None:
self.load()
if self.modules_json is not None:
# Sort the modules.json
self.modules_json["repos"] = nf_core.utils.sort_dictionary(self.modules_json["repos"])
if run_prettier:
dump_json_with_prettier(self.modules_json_path, self.modules_json)
Expand Down
14 changes: 13 additions & 1 deletion nf_core/pipelines/lint/files_exist.py
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ def files_exist(self) -> Dict[str, List[str]]:
conf/igenomes.config
.github/workflows/awstest.yml
.github/workflows/awsfulltest.yml
ro-crate-metadata.json
Files that *must not* be present, due to being renamed or removed in the template:
Expand Down Expand Up @@ -171,6 +172,7 @@ def files_exist(self) -> Dict[str, List[str]]:
[Path(".github", "workflows", "awstest.yml")],
[Path(".github", "workflows", "awsfulltest.yml")],
[Path("modules.json")],
[Path("ro-crate-metadata.json")],
]

# List of strings. Fails / warns if any of the strings exist.
Expand Down Expand Up @@ -198,6 +200,12 @@ def files_exist(self) -> Dict[str, List[str]]:
]
files_warn_ifexists = [Path(".travis.yml")]

files_hint = [
[
["ro-crate-metadata.json"],
". Run `nf-core rocrate` to generate this file. Read more about RO-Crates in the [nf-core/tools docs](https://nf-co.re/tools#create-a-ro-crate-metadata-file).",
],
]
# Remove files that should be ignored according to the linting config
ignore_files = self.lint_config.get("files_exist", []) if self.lint_config is not None else []

Expand Down Expand Up @@ -225,7 +233,11 @@ def pf(file_path: Union[str, Path]) -> Path:
if any([pf(f).is_file() for f in files]):
passed.append(f"File found: {self._wrap_quotes(files)}")
else:
warned.append(f"File not found: {self._wrap_quotes(files)}")
hint = ""
for file_hint in files_hint:
if file_hint[0] == files:
hint = str(file_hint[1])
warned.append(f"File not found: {self._wrap_quotes(files)}{hint}")

# Files that cause an error if they exist
for file in files_fail_ifexists:
Expand Down
Loading

0 comments on commit 796c219

Please sign in to comment.