diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 4ecfbfe..b290e09 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -10,15 +10,7 @@ "vscode": { // Set *default* container specific settings.json values on container create. "settings": { - "python.defaultInterpreterPath": "/opt/conda/bin/python", - "python.linting.enabled": true, - "python.linting.pylintEnabled": true, - "python.formatting.autopep8Path": "/opt/conda/bin/autopep8", - "python.formatting.yapfPath": "/opt/conda/bin/yapf", - "python.linting.flake8Path": "/opt/conda/bin/flake8", - "python.linting.pycodestylePath": "/opt/conda/bin/pycodestyle", - "python.linting.pydocstylePath": "/opt/conda/bin/pydocstyle", - "python.linting.pylintPath": "/opt/conda/bin/pylint" + "python.defaultInterpreterPath": "/opt/conda/bin/python" }, // Add the IDs of extensions you want installed when the container is created. diff --git a/.editorconfig b/.editorconfig index b6b3190..72dda28 100644 --- a/.editorconfig +++ b/.editorconfig @@ -18,7 +18,16 @@ end_of_line = unset insert_final_newline = unset trim_trailing_whitespace = unset indent_style = unset -indent_size = unset +[/subworkflows/nf-core/**] +charset = unset +end_of_line = unset +insert_final_newline = unset +trim_trailing_whitespace = unset +indent_style = unset [/assets/email*] indent_size = unset + +# ignore python and markdown +[*.{py,md}] +indent_style = unset diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 1189151..8225096 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -9,9 +9,8 @@ Please use the pre-filled template to save time. However, don't be put off by this template - other more general issues and suggestions are welcome! Contributions to the code are even more welcome ;) -:::info -If you need help using or modifying nf-core/rangeland then the best place to ask is on the nf-core Slack [#rangeland](https://nfcore.slack.com/channels/rangeland) channel ([join our Slack here](https://nf-co.re/join/slack)). -::: +> [!NOTE] +> If you need help using or modifying nf-core/rangeland then the best place to ask is on the nf-core Slack [#rangeland](https://nfcore.slack.com/channels/rangeland) channel ([join our Slack here](https://nf-co.re/join/slack)). ## Contribution workflow @@ -20,13 +19,19 @@ If you'd like to write some code for nf-core/rangeland, the standard workflow is 1. Check that there isn't already an issue about your idea in the [nf-core/rangeland issues](https://github.com/nf-core/rangeland/issues) to avoid duplicating work. If there isn't one already, please create one so that others know you're working on this 2. [Fork](https://help.github.com/en/github/getting-started-with-github/fork-a-repo) the [nf-core/rangeland repository](https://github.com/nf-core/rangeland) to your GitHub account 3. Make the necessary changes / additions within your forked repository following [Pipeline conventions](#pipeline-contribution-conventions) -4. Use `nf-core schema build` and add any new parameters to the pipeline JSON schema (requires [nf-core tools](https://github.com/nf-core/tools) >= 1.10). +4. Use `nf-core pipelines schema build` and add any new parameters to the pipeline JSON schema (requires [nf-core tools](https://github.com/nf-core/tools) >= 1.10). 5. Submit a Pull Request against the `dev` branch and wait for the code to be reviewed and merged If you're not used to this workflow with git, you can start with some [docs from GitHub](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests) or even their [excellent `git` resources](https://try.github.io/). ## Tests +You have the option to test your changes locally by running the pipeline. For receiving warnings about process selectors and other `debug` information, it is recommended to use the debug profile. Execute all the tests with the following command: + +```bash +nf-test test --profile debug,test,docker --verbose +``` + When you create a pull request with changes, [GitHub Actions](https://github.com/features/actions) will run automatic tests. Typically, pull-requests are only fully reviewed when these tests are passing, though of course we can help out before then. @@ -35,7 +40,7 @@ There are typically two types of tests that run: ### Lint tests `nf-core` has a [set of guidelines](https://nf-co.re/developers/guidelines) which all pipelines must adhere to. -To enforce these and ensure that all pipelines stay in sync, we have developed a helper tool which runs checks on the pipeline code. This is in the [nf-core/tools repository](https://github.com/nf-core/tools) and once installed can be run locally with the `nf-core lint ` command. +To enforce these and ensure that all pipelines stay in sync, we have developed a helper tool which runs checks on the pipeline code. This is in the [nf-core/tools repository](https://github.com/nf-core/tools) and once installed can be run locally with the `nf-core pipelines lint ` command. If any failures or warnings are encountered, please follow the listed URL for more documentation. @@ -70,7 +75,7 @@ If you wish to contribute a new step, please use the following coding standards: 2. Write the process block (see below). 3. Define the output channel if needed (see below). 4. Add any new parameters to `nextflow.config` with a default (see below). -5. Add any new parameters to `nextflow_schema.json` with help text (via the `nf-core schema build` tool). +5. Add any new parameters to `nextflow_schema.json` with help text (via the `nf-core pipelines schema build` tool). 6. Add sanity checks and validation for all relevant parameters. 7. Perform local tests to validate that the new code works as expected. 8. If applicable, add a new test command in `.github/workflow/ci.yml`. @@ -81,13 +86,13 @@ If you wish to contribute a new step, please use the following coding standards: Parameters should be initialised / defined with default values in `nextflow.config` under the `params` scope. -Once there, use `nf-core schema build` to add to `nextflow_schema.json`. +Once there, use `nf-core pipelines schema build` to add to `nextflow_schema.json`. ### Default processes resource requirements -Sensible defaults for process resource requirements (CPUs / memory / time) for a process should be defined in `conf/base.config`. These should generally be specified generic with `withLabel:` selectors so they can be shared across multiple processes/steps of the pipeline. A nf-core standard set of labels that should be followed where possible can be seen in the [nf-core pipeline template](https://github.com/nf-core/tools/blob/master/nf_core/pipeline-template/conf/base.config), which has the default process as a single core-process, and then different levels of multi-core configurations for increasingly large memory requirements defined with standardised labels. +Sensible defaults for process resource requirements (CPUs / memory / time) for a process should be defined in `conf/base.config`. These should generally be specified generic with `withLabel:` selectors so they can be shared across multiple processes/steps of the pipeline. A nf-core standard set of labels that should be followed where possible can be seen in the [nf-core pipeline template](https://github.com/nf-core/tools/blob/main/nf_core/pipeline-template/conf/base.config), which has the default process as a single core-process, and then different levels of multi-core configurations for increasingly large memory requirements defined with standardised labels. -The process resources can be passed on to the tool dynamically within the process with the `${task.cpu}` and `${task.memory}` variables in the `script:` block. +The process resources can be passed on to the tool dynamically within the process with the `${task.cpus}` and `${task.memory}` variables in the `script:` block. ### Naming schemes @@ -98,7 +103,7 @@ Please use the following naming schemes, to make it easy to understand what is g ### Nextflow version bumping -If you are using a new feature from core Nextflow, you may bump the minimum required version of nextflow in the pipeline with: `nf-core bump-version --nextflow . [min-nf-version]` +If you are using a new feature from core Nextflow, you may bump the minimum required version of nextflow in the pipeline with: `nf-core pipelines bump-version --nextflow . [min-nf-version]` ### Images and figures diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index b716bb3..58ec59b 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -17,8 +17,9 @@ Learn more about contributing: [CONTRIBUTING.md](https://github.com/nf-core/rang - [ ] If you've fixed a bug or added code that should be tested, add tests! - [ ] If you've added a new tool - have you followed the pipeline conventions in the [contribution docs](https://github.com/nf-core/rangeland/tree/master/.github/CONTRIBUTING.md) - [ ] If necessary, also make a PR on the nf-core/rangeland _branch_ on the [nf-core/test-datasets](https://github.com/nf-core/test-datasets) repository. -- [ ] Make sure your code lints (`nf-core lint`). +- [ ] Make sure your code lints (`nf-core pipelines lint`). - [ ] Ensure the test suite passes (`nextflow run . -profile test,docker --outdir `). +- [ ] Check for unexpected warnings in debug mode (`nextflow run . -profile debug,test,docker --outdir `). - [ ] Usage Documentation in `docs/usage.md` is updated. - [ ] Output Documentation in `docs/output.md` is updated. - [ ] `CHANGELOG.md` is updated. diff --git a/.github/workflows/awsfulltest.yml b/.github/workflows/awsfulltest.yml index 280ca65..fb7ee45 100644 --- a/.github/workflows/awsfulltest.yml +++ b/.github/workflows/awsfulltest.yml @@ -1,19 +1,36 @@ name: nf-core AWS full size tests -# This workflow is triggered on published releases. +# This workflow is triggered on PRs opened against the master branch. # It can be additionally triggered manually with GitHub actions workflow dispatch button. # It runs the -profile 'test_full' on AWS batch on: - release: - types: [published] + pull_request: + branches: + - master workflow_dispatch: + pull_request_review: + types: [submitted] + jobs: - run-tower: + run-platform: name: Run AWS full tests - if: github.repository == 'nf-core/rangeland' + # run only if the PR is approved by at least 2 reviewers and against the master branch or manually triggered + if: github.repository == 'nf-core/rangeland' && github.event.review.state == 'approved' && github.event.pull_request.base.ref == 'master' || github.event_name == 'workflow_dispatch' runs-on: ubuntu-latest steps: - - name: Launch workflow via tower + - uses: octokit/request-action@v2.x + id: check_approvals + with: + route: GET /repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/reviews + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - id: test_variables + if: github.event_name != 'workflow_dispatch' + run: | + JSON_RESPONSE='${{ steps.check_approvals.outputs.data }}' + CURRENT_APPROVALS_COUNT=$(echo $JSON_RESPONSE | jq -c '[.[] | select(.state | contains("APPROVED")) ] | length') + test $CURRENT_APPROVALS_COUNT -ge 2 || exit 1 # At least 2 approvals are required + - name: Launch workflow via Seqera Platform uses: seqeralabs/action-tower-launch@v2 # Add full size test data (but still relatively small datasets for few samples) # on the `test_full.config` test runs with only one set of parameters @@ -30,9 +47,9 @@ jobs: } profiles: test_full - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: - name: Tower debug log file + name: Seqera Platform debug log file path: | - tower_action_*.log - tower_action_*.json + seqera_platform_action_*.log + seqera_platform_action_*.json diff --git a/.github/workflows/awstest.yml b/.github/workflows/awstest.yml index ad95e39..250e14b 100644 --- a/.github/workflows/awstest.yml +++ b/.github/workflows/awstest.yml @@ -5,13 +5,13 @@ name: nf-core AWS test on: workflow_dispatch: jobs: - run-tower: + run-platform: name: Run AWS tests if: github.repository == 'nf-core/rangeland' runs-on: ubuntu-latest steps: - # Launch workflow using Tower CLI tool action - - name: Launch workflow via tower + # Launch workflow using Seqera Platform CLI tool action + - name: Launch workflow via Seqera Platform uses: seqeralabs/action-tower-launch@v2 with: workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }} @@ -25,9 +25,9 @@ jobs: } profiles: test - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: - name: Tower debug log file + name: Seqera Platform debug log file path: | - tower_action_*.log - tower_action_*.json + seqera_platform_action_*.log + seqera_platform_action_*.json diff --git a/.github/workflows/branch.yml b/.github/workflows/branch.yml index 32eae92..7ea70c0 100644 --- a/.github/workflows/branch.yml +++ b/.github/workflows/branch.yml @@ -19,7 +19,7 @@ jobs: # NOTE - this doesn't currently work if the PR is coming from a fork, due to limitations in GitHub actions secrets - name: Post PR comment if: failure() - uses: mshick/add-pr-comment@v1 + uses: mshick/add-pr-comment@b8f338c590a895d50bcbfa6c5859251edc8952fc # v2 with: message: | ## This PR is against the `master` branch :x: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6266c1f..bded152 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,9 +7,12 @@ on: pull_request: release: types: [published] + workflow_dispatch: env: NXF_ANSI_LOG: false + NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity + NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity concurrency: group: "${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}" @@ -17,26 +20,66 @@ concurrency: jobs: test: - name: Run pipeline with test data + name: "Run pipeline with test data (${{ matrix.NXF_VER }} | ${{ matrix.test_name }} | ${{ matrix.profile }})" # Only run on push if this is the nf-core dev branch (merged PRs) if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/rangeland') }}" runs-on: ubuntu-latest strategy: matrix: NXF_VER: - - "23.04.0" + - "24.04.2" - "latest-everything" + profile: + - "conda" + - "docker" + - "singularity" + test_name: + - "test" + isMaster: + - ${{ github.base_ref == 'master' }} + # Exclude conda and singularity on dev + exclude: + - isMaster: false + profile: "conda" + - isMaster: false + profile: "singularity" steps: - name: Check out pipeline code - uses: actions/checkout@v3 + uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 - - name: Install Nextflow - uses: nf-core/setup-nextflow@v1 + - name: Set up Nextflow + uses: nf-core/setup-nextflow@v2 with: version: "${{ matrix.NXF_VER }}" - - name: Run pipeline with test data - # For example: adding multiple test runs with different parameters - # Remember that you can parallelise this by using strategy.matrix + - name: Set up Apptainer + if: matrix.profile == 'singularity' + uses: eWaterCycle/setup-apptainer@main + + - name: Set up Singularity + if: matrix.profile == 'singularity' + run: | + mkdir -p $NXF_SINGULARITY_CACHEDIR + mkdir -p $NXF_SINGULARITY_LIBRARYDIR + + - name: Set up Miniconda + if: matrix.profile == 'conda' + uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3 + with: + miniconda-version: "latest" + auto-update-conda: true + conda-solver: libmamba + channels: conda-forge,bioconda + + - name: Set up Conda + if: matrix.profile == 'conda' + run: | + echo $(realpath $CONDA)/condabin >> $GITHUB_PATH + echo $(realpath python) >> $GITHUB_PATH + + - name: Clean up Disk space + uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 + + - name: "Run pipeline with test data ${{ matrix.NXF_VER }} | ${{ matrix.test_name }} | ${{ matrix.profile }}" run: | - nextflow run ${GITHUB_WORKSPACE} -profile test,docker --outdir ./results + nextflow run ${GITHUB_WORKSPACE} -profile ${{ matrix.test_name }},${{ matrix.profile }} --outdir ./results diff --git a/.github/workflows/clean-up.yml b/.github/workflows/clean-up.yml index 694e90e..0b6b1f2 100644 --- a/.github/workflows/clean-up.yml +++ b/.github/workflows/clean-up.yml @@ -10,7 +10,7 @@ jobs: issues: write pull-requests: write steps: - - uses: actions/stale@v7 + - uses: actions/stale@28ca1036281a5e5922ead5184a1bbf96e5fc984e # v9 with: stale-issue-message: "This issue has been tagged as awaiting-changes or awaiting-feedback by an nf-core contributor. Remove stale label or add a comment otherwise this issue will be closed in 20 days." stale-pr-message: "This PR has been tagged as awaiting-changes or awaiting-feedback by an nf-core contributor. Remove stale label or add a comment if it is still useful." diff --git a/.github/workflows/download_pipeline.yml b/.github/workflows/download_pipeline.yml new file mode 100644 index 0000000..713dc3e --- /dev/null +++ b/.github/workflows/download_pipeline.yml @@ -0,0 +1,119 @@ +name: Test successful pipeline download with 'nf-core pipelines download' + +# Run the workflow when: +# - dispatched manually +# - when a PR is opened or reopened to master branch +# - the head branch of the pull request is updated, i.e. if fixes for a release are pushed last minute to dev. +on: + workflow_dispatch: + inputs: + testbranch: + description: "The specific branch you wish to utilize for the test execution of nf-core pipelines download." + required: true + default: "dev" + pull_request: + types: + - opened + - edited + - synchronize + branches: + - master + pull_request_target: + branches: + - master + +env: + NXF_ANSI_LOG: false + +jobs: + download: + runs-on: ubuntu-latest + steps: + - name: Install Nextflow + uses: nf-core/setup-nextflow@v2 + + - name: Disk space cleanup + uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 + + - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 + with: + python-version: "3.12" + architecture: "x64" + + - name: Setup Apptainer + uses: eWaterCycle/setup-apptainer@4bb22c52d4f63406c49e94c804632975787312b3 # v2.0.0 + with: + apptainer-version: 1.3.4 + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install git+https://github.com/nf-core/tools.git@dev + + - name: Get the repository name and current branch set as environment variable + run: | + echo "REPO_LOWERCASE=${GITHUB_REPOSITORY,,}" >> ${GITHUB_ENV} + echo "REPOTITLE_LOWERCASE=$(basename ${GITHUB_REPOSITORY,,})" >> ${GITHUB_ENV} + echo "REPO_BRANCH=${{ github.event.inputs.testbranch || 'dev' }}" >> ${GITHUB_ENV} + + - name: Make a cache directory for the container images + run: | + mkdir -p ./singularity_container_images + + - name: Download the pipeline + env: + NXF_SINGULARITY_CACHEDIR: ./singularity_container_images + run: | + nf-core pipelines download ${{ env.REPO_LOWERCASE }} \ + --revision ${{ env.REPO_BRANCH }} \ + --outdir ./${{ env.REPOTITLE_LOWERCASE }} \ + --compress "none" \ + --container-system 'singularity' \ + --container-library "quay.io" -l "docker.io" -l "community.wave.seqera.io" \ + --container-cache-utilisation 'amend' \ + --download-configuration 'yes' + + - name: Inspect download + run: tree ./${{ env.REPOTITLE_LOWERCASE }} + + - name: Count the downloaded number of container images + id: count_initial + run: | + image_count=$(ls -1 ./singularity_container_images | wc -l | xargs) + echo "Initial container image count: $image_count" + echo "IMAGE_COUNT_INITIAL=$image_count" >> ${GITHUB_ENV} + + - name: Run the downloaded pipeline (stub) + id: stub_run_pipeline + continue-on-error: true + env: + NXF_SINGULARITY_CACHEDIR: ./singularity_container_images + NXF_SINGULARITY_HOME_MOUNT: true + run: nextflow run ./${{ env.REPOTITLE_LOWERCASE }}/$( sed 's/\W/_/g' <<< ${{ env.REPO_BRANCH }}) -stub -profile test,singularity --outdir ./results + - name: Run the downloaded pipeline (stub run not supported) + id: run_pipeline + if: ${{ job.steps.stub_run_pipeline.status == failure() }} + env: + NXF_SINGULARITY_CACHEDIR: ./singularity_container_images + NXF_SINGULARITY_HOME_MOUNT: true + run: nextflow run ./${{ env.REPOTITLE_LOWERCASE }}/$( sed 's/\W/_/g' <<< ${{ env.REPO_BRANCH }}) -profile test,singularity --outdir ./results + + - name: Count the downloaded number of container images + id: count_afterwards + run: | + image_count=$(ls -1 ./singularity_container_images | wc -l | xargs) + echo "Post-pipeline run container image count: $image_count" + echo "IMAGE_COUNT_AFTER=$image_count" >> ${GITHUB_ENV} + + - name: Compare container image counts + run: | + if [ "${{ env.IMAGE_COUNT_INITIAL }}" -ne "${{ env.IMAGE_COUNT_AFTER }}" ]; then + initial_count=${{ env.IMAGE_COUNT_INITIAL }} + final_count=${{ env.IMAGE_COUNT_AFTER }} + difference=$((final_count - initial_count)) + echo "$difference additional container images were \n downloaded at runtime . The pipeline has no support for offline runs!" + tree ./singularity_container_images + exit 1 + else + echo "The pipeline can be downloaded successfully!" + fi diff --git a/.github/workflows/fix-linting.yml b/.github/workflows/fix-linting.yml index c57b539..da19b1c 100644 --- a/.github/workflows/fix-linting.yml +++ b/.github/workflows/fix-linting.yml @@ -4,7 +4,7 @@ on: types: [created] jobs: - deploy: + fix-linting: # Only run if comment is on a PR with the main repo, and if it contains the magic keywords if: > contains(github.event.comment.html_url, '/pull/') && @@ -13,10 +13,17 @@ jobs: runs-on: ubuntu-latest steps: # Use the @nf-core-bot token to check out so we can push later - - uses: actions/checkout@v3 + - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 with: token: ${{ secrets.nf_core_bot_auth_token }} + # indication that the linting is being fixed + - name: React on comment + uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4 + with: + comment-id: ${{ github.event.comment.id }} + reactions: eyes + # Action runs on the issue comment, so we don't get the PR by default # Use the gh cli to check out the PR - name: Checkout Pull Request @@ -24,32 +31,59 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.nf_core_bot_auth_token }} - - uses: actions/setup-node@v3 + # Install and run pre-commit + - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 + with: + python-version: "3.12" - - name: Install Prettier - run: npm install -g prettier @prettier/plugin-php + - name: Install pre-commit + run: pip install pre-commit - # Check that we actually need to fix something - - name: Run 'prettier --check' - id: prettier_status - run: | - if prettier --check ${GITHUB_WORKSPACE}; then - echo "result=pass" >> $GITHUB_OUTPUT - else - echo "result=fail" >> $GITHUB_OUTPUT - fi + - name: Run pre-commit + id: pre-commit + run: pre-commit run --all-files + continue-on-error: true - - name: Run 'prettier --write' - if: steps.prettier_status.outputs.result == 'fail' - run: prettier --write ${GITHUB_WORKSPACE} + # indication that the linting has finished + - name: react if linting finished succesfully + if: steps.pre-commit.outcome == 'success' + uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4 + with: + comment-id: ${{ github.event.comment.id }} + reactions: "+1" - name: Commit & push changes - if: steps.prettier_status.outputs.result == 'fail' + id: commit-and-push + if: steps.pre-commit.outcome == 'failure' run: | git config user.email "core@nf-co.re" git config user.name "nf-core-bot" git config push.default upstream git add . git status - git commit -m "[automated] Fix linting with Prettier" + git commit -m "[automated] Fix code linting" git push + + - name: react if linting errors were fixed + id: react-if-fixed + if: steps.commit-and-push.outcome == 'success' + uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4 + with: + comment-id: ${{ github.event.comment.id }} + reactions: hooray + + - name: react if linting errors were not fixed + if: steps.commit-and-push.outcome == 'failure' + uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4 + with: + comment-id: ${{ github.event.comment.id }} + reactions: confused + + - name: react if linting errors were not fixed + if: steps.commit-and-push.outcome == 'failure' + uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4 + with: + issue-number: ${{ github.event.issue.number }} + body: | + @${{ github.actor }} I tried to fix the linting errors, but it didn't work. Please fix them manually. + See [CI log](https://github.com/nf-core/rangeland/actions/runs/${{ github.run_id }}) for more details. diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index b8bdd21..a502573 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -1,6 +1,6 @@ name: nf-core linting # This workflow is triggered on pushes and PRs to the repository. -# It runs the `nf-core lint` and markdown lint tests to ensure +# It runs the `nf-core pipelines lint` and markdown lint tests to ensure # that the code meets the nf-core guidelines. on: push: @@ -11,87 +11,62 @@ on: types: [published] jobs: - EditorConfig: + pre-commit: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 - - uses: actions/setup-node@v3 - - - name: Install editorconfig-checker - run: npm install -g editorconfig-checker - - - name: Run ECLint check - run: editorconfig-checker -exclude README.md $(find .* -type f | grep -v '.git\|.py\|.md\|json\|yml\|yaml\|html\|css\|work\|.nextflow\|build\|nf_core.egg-info\|log.txt\|Makefile') - - Prettier: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-node@v3 - - - name: Install Prettier - run: npm install -g prettier - - - name: Run Prettier --check - run: prettier --check ${GITHUB_WORKSPACE} - - PythonBlack: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - name: Check code lints with Black - uses: psf/black@stable - - # If the above check failed, post a comment on the PR explaining the failure - - name: Post PR comment - if: failure() - uses: mshick/add-pr-comment@v1 + - name: Set up Python 3.12 + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: - message: | - ## Python linting (`black`) is failing - - To keep the code consistent with lots of contributors, we run automated code consistency checks. - To fix this CI test, please run: - - * Install [`black`](https://black.readthedocs.io/en/stable/): `pip install black` - * Fix formatting errors in your pipeline: `black .` + python-version: "3.12" - Once you push these changes the test should pass, and you can hide this comment :+1: + - name: Install pre-commit + run: pip install pre-commit - We highly recommend setting up Black in your code editor so that this formatting is done automatically on save. Ask about it on Slack for help! - - Thanks again for your contribution! - repo-token: ${{ secrets.GITHUB_TOKEN }} - allow-repeats: false + - name: Run pre-commit + run: pre-commit run --all-files nf-core: runs-on: ubuntu-latest steps: - name: Check out pipeline code - uses: actions/checkout@v3 + uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 - name: Install Nextflow - uses: nf-core/setup-nextflow@v1 + uses: nf-core/setup-nextflow@v2 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: - python-version: "3.11" + python-version: "3.12" architecture: "x64" + - name: read .nf-core.yml + uses: pietrobolcato/action-read-yaml@1.1.0 + id: read_yml + with: + config: ${{ github.workspace }}/.nf-core.yml + - name: Install dependencies run: | python -m pip install --upgrade pip - pip install nf-core + pip install nf-core==${{ steps.read_yml.outputs['nf_core_version'] }} + + - name: Run nf-core pipelines lint + if: ${{ github.base_ref != 'master' }} + env: + GITHUB_COMMENTS_URL: ${{ github.event.pull_request.comments_url }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_PR_COMMIT: ${{ github.event.pull_request.head.sha }} + run: nf-core -l lint_log.txt pipelines lint --dir ${GITHUB_WORKSPACE} --markdown lint_results.md - - name: Run nf-core lint + - name: Run nf-core pipelines lint --release + if: ${{ github.base_ref == 'master' }} env: GITHUB_COMMENTS_URL: ${{ github.event.pull_request.comments_url }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_PR_COMMIT: ${{ github.event.pull_request.head.sha }} - run: nf-core -l lint_log.txt lint --dir ${GITHUB_WORKSPACE} --markdown lint_results.md + run: nf-core -l lint_log.txt pipelines lint --release --dir ${GITHUB_WORKSPACE} --markdown lint_results.md - name: Save PR number if: ${{ always() }} @@ -99,7 +74,7 @@ jobs: - name: Upload linting log file artifact if: ${{ always() }} - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4 with: name: linting-logs path: | diff --git a/.github/workflows/linting_comment.yml b/.github/workflows/linting_comment.yml index 0bbcd30..42e519b 100644 --- a/.github/workflows/linting_comment.yml +++ b/.github/workflows/linting_comment.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Download lint results - uses: dawidd6/action-download-artifact@v2 + uses: dawidd6/action-download-artifact@bf251b5aa9c2f7eeb574a96ee720e24f801b7c11 # v6 with: workflow: linting.yml workflow_conclusion: completed @@ -21,7 +21,7 @@ jobs: run: echo "pr_number=$(cat linting-logs/PR_number.txt)" >> $GITHUB_OUTPUT - name: Post PR comment - uses: marocchino/sticky-pull-request-comment@v2 + uses: marocchino/sticky-pull-request-comment@331f8f5b4215f0445d3c07b4967662a32a2d3e31 # v2 with: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} number: ${{ steps.pr_number.outputs.pr_number }} diff --git a/.github/workflows/release-announcments.yml b/.github/workflows/release-announcements.yml similarity index 80% rename from .github/workflows/release-announcments.yml rename to .github/workflows/release-announcements.yml index 6ad3392..c6ba35d 100644 --- a/.github/workflows/release-announcments.yml +++ b/.github/workflows/release-announcements.yml @@ -9,6 +9,11 @@ jobs: toot: runs-on: ubuntu-latest steps: + - name: get topics and convert to hashtags + id: get_topics + run: | + echo "topics=$(curl -s https://nf-co.re/pipelines.json | jq -r '.remote_workflows[] | select(.full_name == "${{ github.repository }}") | .topics[]' | awk '{print "#"$0}' | tr '\n' ' ')" | sed 's/-//g' >> $GITHUB_OUTPUT + - uses: rzr/fediverse-action@master with: access-token: ${{ secrets.MASTODON_ACCESS_TOKEN }} @@ -20,11 +25,13 @@ jobs: Please see the changelog: ${{ github.event.release.html_url }} + ${{ steps.get_topics.outputs.topics }} #nfcore #openscience #nextflow #bioinformatics + send-tweet: runs-on: ubuntu-latest steps: - - uses: actions/setup-python@v4 + - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: python-version: "3.10" - name: Install dependencies @@ -56,7 +63,7 @@ jobs: bsky-post: runs-on: ubuntu-latest steps: - - uses: zentered/bluesky-post-action@v0.0.2 + - uses: zentered/bluesky-post-action@80dbe0a7697de18c15ad22f4619919ceb5ccf597 # v0.1.0 with: post: | Pipeline release! ${{ github.repository }} v${{ github.event.release.tag_name }} - ${{ github.event.release.name }}! diff --git a/.github/workflows/template_version_comment.yml b/.github/workflows/template_version_comment.yml new file mode 100644 index 0000000..e8aafe4 --- /dev/null +++ b/.github/workflows/template_version_comment.yml @@ -0,0 +1,46 @@ +name: nf-core template version comment +# This workflow is triggered on PRs to check if the pipeline template version matches the latest nf-core version. +# It posts a comment to the PR, even if it comes from a fork. + +on: pull_request_target + +jobs: + template_version: + runs-on: ubuntu-latest + steps: + - name: Check out pipeline code + uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: Read template version from .nf-core.yml + uses: nichmor/minimal-read-yaml@v0.0.2 + id: read_yml + with: + config: ${{ github.workspace }}/.nf-core.yml + + - name: Install nf-core + run: | + python -m pip install --upgrade pip + pip install nf-core==${{ steps.read_yml.outputs['nf_core_version'] }} + + - name: Check nf-core outdated + id: nf_core_outdated + run: echo "OUTPUT=$(pip list --outdated | grep nf-core)" >> ${GITHUB_ENV} + + - name: Post nf-core template version comment + uses: mshick/add-pr-comment@b8f338c590a895d50bcbfa6c5859251edc8952fc # v2 + if: | + contains(env.OUTPUT, 'nf-core') + with: + repo-token: ${{ secrets.NF_CORE_BOT_AUTH_TOKEN }} + allow-repeats: false + message: | + > [!WARNING] + > Newer version of the nf-core template is available. + > + > Your pipeline is using an old version of the nf-core template: ${{ steps.read_yml.outputs['nf_core_version'] }}. + > Please update your pipeline to the latest version. + > + > For more documentation on how to update your pipeline, please see the [nf-core documentation](https://github.com/nf-core/tools?tab=readme-ov-file#sync-a-pipeline-with-the-template) and [Synchronisation documentation](https://nf-co.re/docs/contributing/sync). + # diff --git a/.gitignore b/.gitignore index 5124c9a..23b0c7d 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,5 @@ results/ testing/ testing* *.pyc +null/ +.nf-test* diff --git a/.gitpod.yml b/.gitpod.yml index 25488dc..4611863 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -7,13 +7,11 @@ tasks: vscode: extensions: # based on nf-core.nf-core-extensionpack - - codezombiech.gitignore # Language support for .gitignore files - # - cssho.vscode-svgviewer # SVG viewer - - esbenp.prettier-vscode # Markdown/CommonMark linting and style checking for Visual Studio Code - - eamodio.gitlens # Quickly glimpse into whom, why, and when a line or code block was changed + #- esbenp.prettier-vscode # Markdown/CommonMark linting and style checking for Visual Studio Code - EditorConfig.EditorConfig # override user/workspace settings with settings found in .editorconfig files - Gruntfuggly.todo-tree # Display TODO and FIXME in a tree view in the activity bar - mechatroner.rainbow-csv # Highlight columns in csv files in different colors - # - nextflow.nextflow # Nextflow syntax highlighting + - nextflow.nextflow # Nextflow syntax highlighting - oderwat.indent-rainbow # Highlight indentation level - streetsidesoftware.code-spell-checker # Spelling checker for source code + - charliermarsh.ruff # Code linter Ruff diff --git a/.nf-core.yml b/.nf-core.yml index 14eebe5..d29a63f 100644 --- a/.nf-core.yml +++ b/.nf-core.yml @@ -1,8 +1,21 @@ -repository_type: pipeline -template: - prefix: nf-core - skip: - - igenomes +bump_version: null lint: files_exist: - conf/igenomes.config + - conf/igenomes_ignored.config +nf_core_version: 3.0.2 +org_path: null +repository_type: pipeline +template: + author: Fabian Lehmann, David Frantz, Felix Kummer + description: Long-term vegetation trend analysis pipeline for rangeland systems + using satellite imagery. + force: false + is_nfcore: true + name: rangeland + org: nf-core + outdir: . + skip_features: + - igenomes + version: 1.0.0 +update: null diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0c31cdb..9e9f0e1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,5 +1,13 @@ repos: - repo: https://github.com/pre-commit/mirrors-prettier - rev: "v2.7.1" + rev: "v3.1.0" hooks: - id: prettier + additional_dependencies: + - prettier@3.2.5 + + - repo: https://github.com/editorconfig-checker/editorconfig-checker.python + rev: "3.0.3" + hooks: + - id: editorconfig-checker + alias: ec diff --git a/CHANGELOG.md b/CHANGELOG.md index 534b0a6..259eb81 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,9 +3,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## v1.0dev - [date] +## v1.0.0 - [date] -Initial release of nf-core/rangeland, created with the [nf-core](https://nf-co.re/) template. +First release of `nf-core/rangeland`. +This work is a continuation, and nf-core port, of the [original version of this pipeline](https://github.com/CRC-FONDA/FORCE2NXF-Rangeland). ### `Added` diff --git a/CITATIONS.md b/CITATIONS.md index 200d4c5..db0724d 100644 --- a/CITATIONS.md +++ b/CITATIONS.md @@ -22,7 +22,7 @@ This paper describes technical details of deploying the FORCE tool in a Nextflow - [MultiQC](https://pubmed.ncbi.nlm.nih.gov/27312411/) - > Ewels P, Magnusson M, Lundin S, Käller M. MultiQC: summarize analysis results for multiple tools and samples in a single report. Bioinformatics. 2016 Oct 1;32(19):3047-8. doi: 10.1093/bioinformatics/btw354. Epub 2016 Jun 16. PubMed PMID: 27312411; PubMed Central PMCID: PMC5039924. +> Ewels P, Magnusson M, Lundin S, Käller M. MultiQC: summarize analysis results for multiple tools and samples in a single report. Bioinformatics. 2016 Oct 1;32(19):3047-8. doi: 10.1093/bioinformatics/btw354. Epub 2016 Jun 16. PubMed PMID: 27312411; PubMed Central PMCID: PMC5039924. ## Software packaging/containerisation tools diff --git a/README.md b/README.md index 5f780f1..fb74ba8 100644 --- a/README.md +++ b/README.md @@ -1,19 +1,26 @@ -# ![nf-core/rangeland](docs/images/nf-core-rangeland_logo_light.png#gh-light-mode-only) ![nf-core/rangeland](docs/images/nf-core-rangeland_logo_dark.png#gh-dark-mode-only) - -[![GitHub Actions CI Status](https://github.com/nf-core/rangeland/workflows/nf-core%20CI/badge.svg)](https://github.com/nf-core/rangeland/actions?query=workflow%3A%22nf-core+CI%22) -[![GitHub Actions Linting Status](https://github.com/nf-core/rangeland/workflows/nf-core%20linting/badge.svg)](https://github.com/nf-core/rangeland/actions?query=workflow%3A%22nf-core+linting%22)[![AWS CI](https://img.shields.io/badge/CI%20tests-full%20size-FF9900?labelColor=000000&logo=Amazon%20AWS)](https://nf-co.re/rangeland/results)[![Cite with Zenodo](http://img.shields.io/badge/DOI-10.5281/zenodo.XXXXXXX-1073c8?labelColor=000000)](https://doi.org/10.5281/zenodo.XXXXXXX) - -[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A523.04.0-23aa62.svg)](https://www.nextflow.io/) +

+ + + nf-core/rangeland + +

+ +[![GitHub Actions CI Status](https://github.com/nf-core/rangeland/actions/workflows/ci.yml/badge.svg)](https://github.com/nf-core/rangeland/actions/workflows/ci.yml) +[![GitHub Actions Linting Status](https://github.com/nf-core/rangeland/actions/workflows/linting.yml/badge.svg)](https://github.com/nf-core/rangeland/actions/workflows/linting.yml)[![AWS CI](https://img.shields.io/badge/CI%20tests-full%20size-FF9900?labelColor=000000&logo=Amazon%20AWS)](https://nf-co.re/rangeland/results)[![Cite with Zenodo](http://img.shields.io/badge/DOI-10.5281/zenodo.XXXXXXX-1073c8?labelColor=000000)](https://doi.org/10.5281/zenodo.XXXXXXX) +[![nf-test](https://img.shields.io/badge/unit_tests-nf--test-337ab7.svg)](https://www.nf-test.com) + +[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A524.04.2-23aa62.svg)](https://www.nextflow.io/) [![run with conda](http://img.shields.io/badge/run%20with-conda-3EB049?labelColor=000000&logo=anaconda)](https://docs.conda.io/en/latest/) [![run with docker](https://img.shields.io/badge/run%20with-docker-0db7ed?labelColor=000000&logo=docker)](https://www.docker.com/) [![run with singularity](https://img.shields.io/badge/run%20with-singularity-1d355c.svg?labelColor=000000)](https://sylabs.io/docs/) -[![Launch on Nextflow Tower](https://img.shields.io/badge/Launch%20%F0%9F%9A%80-Nextflow%20Tower-%234256e7)](https://tower.nf/launch?pipeline=https://github.com/nf-core/rangeland) +[![Launch on Seqera Platform](https://img.shields.io/badge/Launch%20%F0%9F%9A%80-Seqera%20Platform-%234256e7)](https://cloud.seqera.io/launch?pipeline=https://github.com/nf-core/rangeland) [![Get help on Slack](http://img.shields.io/badge/slack-nf--core%20%23rangeland-4A154B?labelColor=000000&logo=slack)](https://nfcore.slack.com/channels/rangeland)[![Follow on Twitter](http://img.shields.io/badge/twitter-%40nf__core-1DA1F2?labelColor=000000&logo=twitter)](https://twitter.com/nf_core)[![Follow on Mastodon](https://img.shields.io/badge/mastodon-nf__core-6364ff?labelColor=FFFFFF&logo=mastodon)](https://mstdn.science/@nf_core)[![Watch on YouTube](http://img.shields.io/badge/youtube-nf--core-FF0000?labelColor=000000&logo=youtube)](https://www.youtube.com/c/nf-core) ## Introduction -**nf-core/rangeland** is a geographical best-practice analysis pipeline for remotely sensed imagery. The pipeline processes satellite imagery alongside auxiliary data in multiple steps to arrive at a set of trend files related to land-cover changes. The main pipeline steps are: +**nf-core/rangeland** is a geographical best-practice analysis pipeline for remotely sensed imagery. +The pipeline processes satellite imagery alongside auxiliary data in multiple steps to arrive at a set of trend files related to land-cover changes. The main pipeline steps are: 1. Read satellite imagery, digital elevation model, endmember definition, water vapor database and area of interest definition 2. Generate allow list and analysis mask to determine which pixels from the satellite data can be used @@ -22,23 +29,22 @@ 5. Time series analyses to obtain trends in vegetation dynamics 6. Create mosaic and pyramid visualizations of the results -7. Read QC ([`FastQC`](https://www.bioinformatics.babraham.ac.uk/projects/fastqc/)) -8. Present QC for raw reads ([`MultiQC`](http://multiqc.info/)) +7. Present QC results ([`MultiQC`](http://multiqc.info/)) ## Usage -:::note -If you are new to Nextflow and nf-core, please refer to [this page](https://nf-co.re/docs/usage/installation) on how -to set-up Nextflow. Make sure to [test your setup](https://nf-co.re/docs/usage/introduction#how-to-run-a-pipeline) -with `-profile test` before running the workflow on actual data. -::: +> [!NOTE] +> If you are new to Nextflow and nf-core, please refer to [this page](https://nf-co.re/docs/usage/installation) on how to set-up Nextflow. +> Make sure to [test your setup](https://nf-co.re/docs/usage/introduction#how-to-run-a-pipeline) with `-profile test` before running the workflow on actual data. -To run the pipeline on real data, input data needs to be acquired. Concretely, satellite imagery, water vapor data, a digital elevation model, endmember definitions, a datacube specification, and a area-of-interest specification are required. Please refer to the [usage documentation](https://nf-co.re/rangeland/usage) for details on the input structure. +To run the pipeline on real data, input data needs to be acquired. +Concretely, satellite imagery, water vapor data, a digital elevation model, endmember definitions, a datacube specification, and a area-of-interest specification are required. +Please refer to the [usage documentation](https://nf-co.re/rangeland/usage) for details on the input structure. Now, you can run the pipeline using: ```bash -nextflow run nf-core/rangeland/main.nf \ +nextflow run nf-core/rangeland \ -profile \ --input \ --dem \ @@ -49,11 +55,8 @@ nextflow run nf-core/rangeland/main.nf \ --outdir ``` -:::warning -Please provide pipeline parameters via the CLI or Nextflow `-params-file` option. Custom config files including those -provided by the `-c` Nextflow option can be used to provide any configuration _**except for parameters**_; -see [docs](https://nf-co.re/usage/configuration#custom-configuration-files). -::: +> [!WARNING] +> Please provide pipeline parameters via the CLI or Nextflow `-params-file` option. Custom config files including those provided by the `-c` Nextflow option can be used to provide any configuration _**except for parameters**_; see [docs](https://nf-co.re/docs/usage/getting_started/configuration#custom-configuration-files). For more details and further functionality, please refer to the [usage documentation](https://nf-co.re/rangeland/usage) and the [parameter documentation](https://nf-co.re/rangeland/parameters). @@ -72,7 +75,8 @@ The rangeland workflow was originally written by: The original workflow can be found on [github](https://github.com/CRC-FONDA/FORCE2NXF-Rangeland). -Transformation to nf-core/rangeland was conducted by [Felix Kummer](https://github.com/Felix-Kummer). nf-core alignment started on the [nf-core branch of the original repository](https://github.com/CRC-FONDA/FORCE2NXF-Rangeland/tree/nf-core). +Transformation to nf-core/rangeland was conducted by [Felix Kummer](https://github.com/Felix-Kummer). +nf-core alignment started on the [nf-core branch of the original repository](https://github.com/CRC-FONDA/FORCE2NXF-Rangeland/tree/nf-core). We thank the following people for their extensive assistance in the development of this pipeline: @@ -102,7 +106,7 @@ For further information or help, don't hesitate to get in touch on the [Slack `# ## Citations - + An extensive list of references for the tools used by the pipeline can be found in the [`CITATIONS.md`](CITATIONS.md) file. @@ -114,7 +118,8 @@ You can cite the `nf-core` publication as follows: > > _Nat Biotechnol._ 2020 Feb 13. doi: [10.1038/s41587-020-0439-x](https://dx.doi.org/10.1038/s41587-020-0439-x). -This pipeline is based one the publication listed below. The publication can be cited as follows: +This pipeline is based one the publication listed below. +The publication can be cited as follows: > **FORCE on Nextflow: Scalable Analysis of Earth Observation Data on Commodity Clusters** > diff --git a/assets/email_template.html b/assets/email_template.html index 53f8090..f719cb5 100644 --- a/assets/email_template.html +++ b/assets/email_template.html @@ -4,7 +4,7 @@ - + nf-core/rangeland Pipeline Report @@ -12,7 +12,7 @@ -

nf-core/rangeland v${version}

+

nf-core/rangeland ${version}

Run Name: $runName

<% if (!success){ diff --git a/assets/email_template.txt b/assets/email_template.txt index 575a487..cb9cb22 100644 --- a/assets/email_template.txt +++ b/assets/email_template.txt @@ -4,7 +4,7 @@ |\\ | |__ __ / ` / \\ |__) |__ } { | \\| | \\__, \\__/ | \\ |___ \\`-._,-`-, `._,._,' - nf-core/rangeland v${version} + nf-core/rangeland ${version} ---------------------------------------------------- Run Name: $runName diff --git a/assets/multiqc_config.yml b/assets/multiqc_config.yml index 3bf0482..13dd96a 100644 --- a/assets/multiqc_config.yml +++ b/assets/multiqc_config.yml @@ -1,7 +1,7 @@ report_comment: > - This report has been generated by the nf-core/rangeland + This report has been generated by the nf-core/rangeland analysis pipeline. For information about how to interpret these results, please see the - documentation. + documentation. report_section_order: "nf-core-rangeland-methods-description": order: -1000 @@ -11,3 +11,5 @@ report_section_order: order: -1002 export_plots: true + +disable_version_detection: true diff --git a/assets/nf-core-rangeland_logo_light.png b/assets/nf-core-rangeland_logo_light.png index 118b325..bb5b76e 100644 Binary files a/assets/nf-core-rangeland_logo_light.png and b/assets/nf-core-rangeland_logo_light.png differ diff --git a/assets/schema_input.json b/assets/schema_input.json index 655729c..e46c2ee 100644 --- a/assets/schema_input.json +++ b/assets/schema_input.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://raw.githubusercontent.com/nf-core/rangeland/master/assets/schema_input.json", "title": "nf-core/rangeland pipeline - params.input schema", "description": "Schema for the file provided with params.input", @@ -10,25 +10,22 @@ "sample": { "type": "string", "pattern": "^\\S+$", - "errorMessage": "Sample name must be provided and cannot contain spaces" + "errorMessage": "Sample name must be provided and cannot contain spaces", + "meta": ["id"] }, "fastq_1": { "type": "string", + "format": "file-path", + "exists": true, "pattern": "^\\S+\\.f(ast)?q\\.gz$", "errorMessage": "FastQ file for reads 1 must be provided, cannot contain spaces and must have extension '.fq.gz' or '.fastq.gz'" }, "fastq_2": { - "errorMessage": "FastQ file for reads 2 cannot contain spaces and must have extension '.fq.gz' or '.fastq.gz'", - "anyOf": [ - { - "type": "string", - "pattern": "^\\S+\\.f(ast)?q\\.gz$" - }, - { - "type": "string", - "maxLength": 0 - } - ] + "type": "string", + "format": "file-path", + "exists": true, + "pattern": "^\\S+\\.f(ast)?q\\.gz$", + "errorMessage": "FastQ file for reads 2 cannot contain spaces and must have extension '.fq.gz' or '.fastq.gz'" } }, "required": ["sample", "fastq_1"] diff --git a/assets/slackreport.json b/assets/slackreport.json index af53fb9..50d914f 100644 --- a/assets/slackreport.json +++ b/assets/slackreport.json @@ -3,7 +3,7 @@ { "fallback": "Plain-text summary of the attachment.", "color": "<% if (success) { %>good<% } else { %>danger<%} %>", - "author_name": "nf-core/rangeland v${version} - ${runName}", + "author_name": "nf-core/rangeland ${version} - ${runName}", "author_icon": "https://www.nextflow.io/docs/latest/_static/favicon.ico", "text": "<% if (success) { %>Pipeline completed successfully!<% } else { %>Pipeline completed with errors<% } %>", "fields": [ diff --git a/bin/check_samplesheet.py b/bin/check_samplesheet.py deleted file mode 100755 index 4a758fe..0000000 --- a/bin/check_samplesheet.py +++ /dev/null @@ -1,259 +0,0 @@ -#!/usr/bin/env python - - -"""Provide a command line tool to validate and transform tabular samplesheets.""" - - -import argparse -import csv -import logging -import sys -from collections import Counter -from pathlib import Path - -logger = logging.getLogger() - - -class RowChecker: - """ - Define a service that can validate and transform each given row. - - Attributes: - modified (list): A list of dicts, where each dict corresponds to a previously - validated and transformed row. The order of rows is maintained. - - """ - - VALID_FORMATS = ( - ".fq.gz", - ".fastq.gz", - ) - - def __init__( - self, - sample_col="sample", - first_col="fastq_1", - second_col="fastq_2", - single_col="single_end", - **kwargs, - ): - """ - Initialize the row checker with the expected column names. - - Args: - sample_col (str): The name of the column that contains the sample name - (default "sample"). - first_col (str): The name of the column that contains the first (or only) - FASTQ file path (default "fastq_1"). - second_col (str): The name of the column that contains the second (if any) - FASTQ file path (default "fastq_2"). - single_col (str): The name of the new column that will be inserted and - records whether the sample contains single- or paired-end sequencing - reads (default "single_end"). - - """ - super().__init__(**kwargs) - self._sample_col = sample_col - self._first_col = first_col - self._second_col = second_col - self._single_col = single_col - self._seen = set() - self.modified = [] - - def validate_and_transform(self, row): - """ - Perform all validations on the given row and insert the read pairing status. - - Args: - row (dict): A mapping from column headers (keys) to elements of that row - (values). - - """ - self._validate_sample(row) - self._validate_first(row) - self._validate_second(row) - self._validate_pair(row) - self._seen.add((row[self._sample_col], row[self._first_col])) - self.modified.append(row) - - def _validate_sample(self, row): - """Assert that the sample name exists and convert spaces to underscores.""" - if len(row[self._sample_col]) <= 0: - raise AssertionError("Sample input is required.") - # Sanitize samples slightly. - row[self._sample_col] = row[self._sample_col].replace(" ", "_") - - def _validate_first(self, row): - """Assert that the first FASTQ entry is non-empty and has the right format.""" - if len(row[self._first_col]) <= 0: - raise AssertionError("At least the first FASTQ file is required.") - self._validate_fastq_format(row[self._first_col]) - - def _validate_second(self, row): - """Assert that the second FASTQ entry has the right format if it exists.""" - if len(row[self._second_col]) > 0: - self._validate_fastq_format(row[self._second_col]) - - def _validate_pair(self, row): - """Assert that read pairs have the same file extension. Report pair status.""" - if row[self._first_col] and row[self._second_col]: - row[self._single_col] = False - first_col_suffix = Path(row[self._first_col]).suffixes[-2:] - second_col_suffix = Path(row[self._second_col]).suffixes[-2:] - if first_col_suffix != second_col_suffix: - raise AssertionError("FASTQ pairs must have the same file extensions.") - else: - row[self._single_col] = True - - def _validate_fastq_format(self, filename): - """Assert that a given filename has one of the expected FASTQ extensions.""" - if not any(filename.endswith(extension) for extension in self.VALID_FORMATS): - raise AssertionError( - f"The FASTQ file has an unrecognized extension: {filename}\n" - f"It should be one of: {', '.join(self.VALID_FORMATS)}" - ) - - def validate_unique_samples(self): - """ - Assert that the combination of sample name and FASTQ filename is unique. - - In addition to the validation, also rename all samples to have a suffix of _T{n}, where n is the - number of times the same sample exist, but with different FASTQ files, e.g., multiple runs per experiment. - - """ - if len(self._seen) != len(self.modified): - raise AssertionError("The pair of sample name and FASTQ must be unique.") - seen = Counter() - for row in self.modified: - sample = row[self._sample_col] - seen[sample] += 1 - row[self._sample_col] = f"{sample}_T{seen[sample]}" - - -def read_head(handle, num_lines=10): - """Read the specified number of lines from the current position in the file.""" - lines = [] - for idx, line in enumerate(handle): - if idx == num_lines: - break - lines.append(line) - return "".join(lines) - - -def sniff_format(handle): - """ - Detect the tabular format. - - Args: - handle (text file): A handle to a `text file`_ object. The read position is - expected to be at the beginning (index 0). - - Returns: - csv.Dialect: The detected tabular format. - - .. _text file: - https://docs.python.org/3/glossary.html#term-text-file - - """ - peek = read_head(handle) - handle.seek(0) - sniffer = csv.Sniffer() - dialect = sniffer.sniff(peek) - return dialect - - -def check_samplesheet(file_in, file_out): - """ - Check that the tabular samplesheet has the structure expected by nf-core pipelines. - - Validate the general shape of the table, expected columns, and each row. Also add - an additional column which records whether one or two FASTQ reads were found. - - Args: - file_in (pathlib.Path): The given tabular samplesheet. The format can be either - CSV, TSV, or any other format automatically recognized by ``csv.Sniffer``. - file_out (pathlib.Path): Where the validated and transformed samplesheet should - be created; always in CSV format. - - Example: - This function checks that the samplesheet follows the following structure, - see also the `viral recon samplesheet`_:: - - sample,fastq_1,fastq_2 - SAMPLE_PE,SAMPLE_PE_RUN1_1.fastq.gz,SAMPLE_PE_RUN1_2.fastq.gz - SAMPLE_PE,SAMPLE_PE_RUN2_1.fastq.gz,SAMPLE_PE_RUN2_2.fastq.gz - SAMPLE_SE,SAMPLE_SE_RUN1_1.fastq.gz, - - .. _viral recon samplesheet: - https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/samplesheet/samplesheet_test_illumina_amplicon.csv - - """ - required_columns = {"sample", "fastq_1", "fastq_2"} - # See https://docs.python.org/3.9/library/csv.html#id3 to read up on `newline=""`. - with file_in.open(newline="") as in_handle: - reader = csv.DictReader(in_handle, dialect=sniff_format(in_handle)) - # Validate the existence of the expected header columns. - if not required_columns.issubset(reader.fieldnames): - req_cols = ", ".join(required_columns) - logger.critical(f"The sample sheet **must** contain these column headers: {req_cols}.") - sys.exit(1) - # Validate each row. - checker = RowChecker() - for i, row in enumerate(reader): - try: - checker.validate_and_transform(row) - except AssertionError as error: - logger.critical(f"{str(error)} On line {i + 2}.") - sys.exit(1) - checker.validate_unique_samples() - header = list(reader.fieldnames) - header.insert(1, "single_end") - # See https://docs.python.org/3.9/library/csv.html#id3 to read up on `newline=""`. - with file_out.open(mode="w", newline="") as out_handle: - writer = csv.DictWriter(out_handle, header, delimiter=",") - writer.writeheader() - for row in checker.modified: - writer.writerow(row) - - -def parse_args(argv=None): - """Define and immediately parse command line arguments.""" - parser = argparse.ArgumentParser( - description="Validate and transform a tabular samplesheet.", - epilog="Example: python check_samplesheet.py samplesheet.csv samplesheet.valid.csv", - ) - parser.add_argument( - "file_in", - metavar="FILE_IN", - type=Path, - help="Tabular input samplesheet in CSV or TSV format.", - ) - parser.add_argument( - "file_out", - metavar="FILE_OUT", - type=Path, - help="Transformed output samplesheet in CSV format.", - ) - parser.add_argument( - "-l", - "--log-level", - help="The desired log level (default WARNING).", - choices=("CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG"), - default="WARNING", - ) - return parser.parse_args(argv) - - -def main(argv=None): - """Coordinate argument parsing and program execution.""" - args = parse_args(argv) - logging.basicConfig(level=args.log_level, format="[%(levelname)s] %(message)s") - if not args.file_in.is_file(): - logger.error(f"The given input file {args.file_in} was not found!") - sys.exit(2) - args.file_out.parent.mkdir(parents=True, exist_ok=True) - check_samplesheet(args.file_in, args.file_out) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/bin/merge_boa.r b/bin/merge_boa.r index 98a0afd..318164f 100755 --- a/bin/merge_boa.r +++ b/bin/merge_boa.r @@ -1,44 +1,46 @@ #!/usr/bin/env Rscript -args = commandArgs(trailingOnly=TRUE) +## Originally written by Felix Kummer and released under the MIT license. +## See git repository (https://github.com/nf-core/rangeland) for full license text. +# Script for merging bottom of atmosphere (boa) .tif raster files. +# This can improve the performance of downstream tasks. -if (length(args) < 3) { - stop("\nthis program needs at least 3 inputs\n1: output filename\n2-*: input files", call.=FALSE) -} - -fout <- args[1] -finp <- args[2:length(args)] -nf <- length(finp) - -require(raster) - - -img <- brick(finp[1]) -nc <- ncell(img) -nb <- nbands(img) +require(terra) +args <- commandArgs(trailingOnly = TRUE) -sum <- matrix(0, nc, nb) -num <- matrix(0, nc, nb) - -for (i in 1:nf){ - - data <- brick(finp[i])[] - - num <- num + !is.na(data) - - data[is.na(data)] <- 0 - sum <- sum + data +if (length(args) < 3) { + stop("\nError: this program needs at least 3 inputs\n1: output filename\n2-*: input files", call.=FALSE) } -mean <- sum/num -img[] <- mean - +fout <- args[1] +finp <- args[2:length(args)] -writeRaster(img, filename = fout, format = "GTiff", datatype = "INT2S", - options = c("INTERLEAVE=BAND", "COMPRESS=LZW", "PREDICTOR=2", - "NUM_THREADS=ALL_CPUS", "BIGTIFF=YES", - sprintf("BLOCKXSIZE=%s", img@file@blockcols[1]), - sprintf("BLOCKYSIZE=%s", img@file@blockrows[1]))) +# Load input rasters +rasters <- lapply(finp, rast) + +# Calculate the sum of non-NA values across all rasters +sum_rasters <- Reduce("+", lapply(rasters, function(x) { + x[is.na(x)] <- 0 + return(x) +})) + +# Calculate the number of values non-NA values for each cell +count_rasters <- Reduce("+", lapply(rasters, function(x) { + return(!is.na(x)) +})) + +# Calculate the mean raster +mean_raster <- sum_rasters / count_rasters + +# Write the mean raster +writeRaster(mean_raster, + filename = fout, + datatype = "INT2S", + filetype = "GTiff", + gdal = c("COMPRESS=LZW", "PREDICTOR=2", + "NUM_THREADS=ALL_CPUS", "BIGTIFF=YES", + sprintf("BLOCKXSIZE=%s", ncol(mean_raster)), + sprintf("BLOCKYSIZE=%s", nrow(mean_raster)))) diff --git a/bin/merge_qai.r b/bin/merge_qai.r index 9a31421..8515063 100755 --- a/bin/merge_qai.r +++ b/bin/merge_qai.r @@ -1,38 +1,41 @@ #!/usr/bin/env Rscript -args = commandArgs(trailingOnly=TRUE) +## Originally written by Felix Kummer and released under the MIT license. +## See git repository (https://github.com/nf-core/rangeland) for full license text. +# Script for merging quality information (qai) .tif raster files. +# This can improve the performance of downstream tasks. -if (length(args) < 3) { - stop("\nthis program needs at least 3 inputs\n1: output filename\n2-*: input files", call.=FALSE) -} - -fout <- args[1] -finp <- args[2:length(args)] -nf <- length(finp) - -require(raster) - - -img <- raster(finp[1]) -nc <- ncell(img) +require(terra) +args <- commandArgs(trailingOnly = TRUE) -last <- rep(1, nc) - -for (i in 1:nf){ - - data <- raster(finp[i])[] - - last[!is.na(data)] <- data[!is.na(data)] +if (length(args) < 3) { + stop("\nError: this program needs at least 3 inputs\n1: output filename\n2-*: input files", call.=FALSE) } -img[] <- last - +fout <- args[1] +finp <- args[2:length(args)] -writeRaster(img, filename = fout, format = "GTiff", datatype = "INT2S", - options = c("INTERLEAVE=BAND", "COMPRESS=LZW", "PREDICTOR=2", - "NUM_THREADS=ALL_CPUS", "BIGTIFF=YES", - sprintf("BLOCKXSIZE=%s", img@file@blockcols[1]), - sprintf("BLOCKYSIZE=%s", img@file@blockrows[1]))) +# load raster files into single SpatRaster +rasters <- rast(finp) + +# Merge rasters by maintaining the last non-NA value +merged_raster <- app(rasters, function(x) { + non_na_values <- na.omit(x) + if (length(non_na_values) == 0) { + return(1) + } + return(tail(non_na_values, 1)[1]) +}) + +# Write merged raster +writeRaster(merged_raster, + filename = fout, + filetype = "GTiff", + datatype = "INT2S", + gdal = c("INTERLEAVE=BAND", "COMPRESS=LZW", "PREDICTOR=2", + "NUM_THREADS=ALL_CPUS", "BIGTIFF=YES", + sprintf("BLOCKXSIZE=%s", ncol(merged_raster)), + sprintf("BLOCKYSIZE=%s", nrow(merged_raster)))) diff --git a/bin/test.R b/bin/test.R index 7274a98..298b023 100755 --- a/bin/test.R +++ b/bin/test.R @@ -1,37 +1,92 @@ #!/usr/bin/env Rscript +## Originally written by David Frantz and Felix Kummer and released under the MIT license. +## See git repository (https://github.com/nf-core/rangeland) for full license text. + +# Script to verify pipeline results from test and test_full profiles. + args = commandArgs(trailingOnly=TRUE) -if (length(args) != 7) { - stop("\ngive input directory (mosaic) as 1st arg\ngive reference rasters (*.tif) as 2nd-7th args in order: +if (length(args) != 7 && length(args) != 2) { + stop("\n Error: wrong number of parameters. Usage: \n 1st arg: workflow results directory (mosaic) + \n 2nd-7th args: reference rasters (*.tif) in order: woody cover change, woody cover year of change, herbaceous cover change, herbaceous cover year of change, - peak change, peak year of change, ", call.=FALSE) + peak change, peak year of change + \nor \n + 1st arg: workflow results directory (mosaic) \n 2nd arg: reference directory + ", call.=FALSE) } -dinp <- args[1] - # load package require(terra) +# function to compare change directions +compare_direction <- function(r1, r2, threshold = 0.95) { + + # get signs + s1 <- sign(r1) + s2 <- sign(r2) + + # replace na's + vals1 <- subst(s1, NA, -9999) + vals2 <- subst(s2, NA, -9999) + + # Compare the signs + matches <- vals1 == vals2 + match_count <- sum(values(matches)) + total_count <- sum(!is.na(values(vals1))) + + # Calculate the percentage of matches + match_percentage <- match_count / total_count + + if (match_percentage >= threshold) { + return(TRUE) + } else { + return(paste("Change directions not matching. Match percentage:", match_percentage)) + } +} # LOAD REFERENCE ####################################################################### -woody_cover_changes_ref <- rast(args[2]) -woody_cover_year_of_change_ref <- rast(args[3]) -herbaceous_cover_changes_ref <- rast(args[4]) -herbaceous_cover_year_of_change_ref <- rast(args[5]) +if (length(args) == 7 ){ + woody_cover_changes_ref <- rast(args[2]) + woody_cover_year_of_change_ref <- rast(args[3]) -peak_changes_ref <- rast(args[6]) -peak_year_of_change_ref <- rast(args[7]) + herbaceous_cover_changes_ref <- rast(args[4]) + herbaceous_cover_year_of_change_ref <- rast(args[5]) + peak_changes_ref <- rast(args[6]) + peak_year_of_change_ref <- rast(args[7]) +} else { + # reference parent dir + ref_dir <- args[2] + + vrt_file <- list.files(ref_dir, pattern = "VBL-CAO\\.vrt$", recursive = TRUE, full.names = TRUE) + woody_ref <- rast(vrt_file) + woody_cover_changes_ref <- woody_ref$CHANGE + woody_cover_year_of_change_ref <- woody_ref["YEAR-OF-CHANGE"] + + vrt_file <- list.files(ref_dir, pattern = "VSA-CAO\\.vrt$", recursive = TRUE, full.names = TRUE) + herbaceous_ref <- rast(vrt_file) + herbaceous_cover_changes_ref <- herbaceous_ref$CHANGE + herbaceous_cover_year_of_change_ref <- herbaceous_ref["YEAR-OF-CHANGE"] + + vrt_file <- list.files(ref_dir, pattern = "VPS-CAO\\.vrt$", recursive = TRUE, full.names = TRUE) + peak_ref <- rast(vrt_file) + peak_changes_ref <- peak_ref$CHANGE + peak_year_of_change_ref <- peak_ref["YEAR-OF-CHANGE"] +} # WOODY COVER CHANGE (VALUE OF BASE LEVEL) ####################################################################### +# input data dir +dinp <- args[1] + fname <- dir(dinp, ".*HL_TSA_LNDLG_SMA_VBL-CAO.vrt$", full.names=TRUE) woody_cover_rast <- rast(fname) @@ -66,67 +121,63 @@ peak_year_of_change <- peak_rast["YEAR-OF-CHANGE"] -# FOR REFERENCE: SAVE RASTERS -####################################################################### - -#writeRaster(woody_cover_changes, "woody_cover_chg_ref.tif") -#writeRaster(woody_cover_year_of_change, "woody_cover_yoc_ref.tif") - -#writeRaster(herbaceous_cover_changes, "herbaceous_cover_chg_ref.tif") -#writeRaster(herbaceous_cover_year_of_change, "herbaceous_cover_yoc_ref.tif") - -#writeRaster(peak_changes, "peak_chg_ref.tif") -#writeRaster(peak_year_of_change, "peak_yoc_ref.tif") - - - - # COMPARE TESTRUN WITH REFERENCE EXECUTION ####################################################################### +failure <- FALSE -woody_cover_changes_result <- all.equal(woody_cover_changes, woody_cover_changes_ref) -if (is.character(woody_cover_changes_result)){ - stop("Error: ", paste0(woody_cover_changes_result, " for woody cover changes.")) +woody_cover_changes_result <- compare_direction(woody_cover_changes, woody_cover_changes_ref) +if (is.character(woody_cover_changes_result)) { + print(paste0("Error: ", woody_cover_changes_result, " for woody cover changes.")) + failure <- TRUE } else { print("Woody cover change check passed.") } -woody_cover_year_of_change_result <- all.equal(woody_cover_year_of_change, woody_cover_year_of_change_ref) -if (is.character(woody_cover_year_of_change_result)){ - stop("Error: ", paste0(woody_cover_year_of_change_result, " for woody cover year of change.")) +woody_cover_year_of_change_result <- all.equal(woody_cover_year_of_change, woody_cover_year_of_change_ref, tolerance=1e-3) +if (is.character(woody_cover_year_of_change_result)) { + print(paste0("Error: ", woody_cover_year_of_change_result, " for woody cover year of change.")) + failure <- TRUE } else { print("Woody cover year of change check passed.") } -herbaceous_cover_changes_result <- all.equal(herbaceous_cover_changes, herbaceous_cover_changes_ref) -if (is.character(herbaceous_cover_changes_result)){ - stop("Error: ", paste0(herbaceous_cover_changes_result, " for herbaceous cover changes.")) +herbaceous_cover_changes_result <- compare_direction(herbaceous_cover_changes, herbaceous_cover_changes_ref) +if (is.character(herbaceous_cover_changes_result)) { + print(paste0("Error: ",herbaceous_cover_changes_result, " for herbaceous cover changes.")) + failure <- TRUE } else { print("Herbaceous cover change check passed.") } -herbaceous_cover_year_of_change_result <- all.equal(herbaceous_cover_year_of_change, herbaceous_cover_year_of_change_ref) -if (is.character(herbaceous_cover_year_of_change_result)){ - stop("Error: ", paste0(herbaceous_cover_year_of_change_result, " for herbaceous cover year of change.")) +herbaceous_cover_year_of_change_result <- all.equal(herbaceous_cover_year_of_change, herbaceous_cover_year_of_change_ref, tolerance=1e-3) +if (is.character(herbaceous_cover_year_of_change_result)) { + print(paste0("Error: ", herbaceous_cover_year_of_change_result, " for herbaceous cover year of change.")) + failure <- TRUE } else { print("Herbaceous cover year of change check passed.") } -peak_changes_result <- all.equal(peak_changes, peak_changes_ref) -if (is.character(peak_changes_result)){ - stop("Error: ", paste0(peak_changes_result, " for peak changes.")) +peak_changes_result <- compare_direction(peak_changes, peak_changes_ref) +if (is.character(peak_changes_result)) { + print(paste0("Error: ", peak_changes_result, " for peak changes.")) + failure <- TRUE } else { print("Peak change check passed.") } -peak_year_of_change_result <- all.equal(peak_year_of_change, peak_year_of_change_ref) -if (is.character(peak_year_of_change_result)){ - stop("Error: ", paste0(peak_year_of_change_result, " for peak year of change.")) +peak_year_of_change_result <- all.equal(peak_year_of_change, peak_year_of_change_ref, tolerance=1e-3) +if (is.character(peak_year_of_change_result)) { + print(paste0("Error: ", peak_year_of_change_result, " for peak year of change.")) + failure <- TRUE } else { print("Peak year of change check passed.") } -print("All checks passed.") +if (failure) { + stop("Some test failed.") +} else { + print("All checks passed.") +} diff --git a/conf/base.config b/conf/base.config index 2d99930..cf3d991 100644 --- a/conf/base.config +++ b/conf/base.config @@ -10,9 +10,9 @@ process { - cpus = { check_max( 1 * task.attempt, 'cpus' ) } - memory = { check_max( 6.GB * task.attempt, 'memory' ) } - time = { check_max( 4.h * task.attempt, 'time' ) } + cpus = { 1 * task.attempt } + memory = { 6.GB * task.attempt } + time = { 4.h * task.attempt } errorStrategy = { task.exitStatus in ((130..145) + 104) ? 'retry' : 'finish' } maxRetries = 1 @@ -25,30 +25,30 @@ process { // adding in your local modules too. // See https://www.nextflow.io/docs/latest/config.html#config-process-selectors withLabel:process_single { - cpus = { check_max( 1 , 'cpus' ) } - memory = { check_max( 6.GB * task.attempt, 'memory' ) } - time = { check_max( 4.h * task.attempt, 'time' ) } + cpus = { 1 } + memory = { 6.GB * task.attempt } + time = { 4.h * task.attempt } } withLabel:process_low { - cpus = { check_max( 2 * task.attempt, 'cpus' ) } - memory = { check_max( 12.GB * task.attempt, 'memory' ) } - time = { check_max( 4.h * task.attempt, 'time' ) } + cpus = { 2 * task.attempt } + memory = { 12.GB * task.attempt } + time = { 4.h * task.attempt } } withLabel:process_medium { - cpus = { check_max( 6 * task.attempt, 'cpus' ) } - memory = { check_max( 36.GB * task.attempt, 'memory' ) } - time = { check_max( 8.h * task.attempt, 'time' ) } + cpus = { 6 * task.attempt } + memory = { 36.GB * task.attempt } + time = { 8.h * task.attempt } } withLabel:process_high { - cpus = { check_max( 12 * task.attempt, 'cpus' ) } - memory = { check_max( 72.GB * task.attempt, 'memory' ) } - time = { check_max( 16.h * task.attempt, 'time' ) } + cpus = { 12 * task.attempt } + memory = { 72.GB * task.attempt } + time = { 16.h * task.attempt } } withLabel:process_long { - time = { check_max( 20.h * task.attempt, 'time' ) } + time = { 20.h * task.attempt } } withLabel:process_high_memory { - memory = { check_max( 200.GB * task.attempt, 'memory' ) } + memory = { 200.GB * task.attempt } } withLabel:error_ignore { errorStrategy = 'ignore' @@ -57,7 +57,4 @@ process { errorStrategy = 'retry' maxRetries = 2 } - withName:CUSTOM_DUMPSOFTWAREVERSIONS { - cache = false - } } diff --git a/conf/modules.config b/conf/modules.config index 95b8962..9eece2d 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -13,154 +13,133 @@ process { publishDir = [ - path: { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename.equals('versions.yml') ? null : filename } + path: { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename.equals('versions.yml') ? null : filename }, + enabled: params.publish_dir_enabled ] - withName: SAMPLESHEET_CHECK { - publishDir = [ - path: { "${params.outdir}/pipeline_info" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename.equals('versions.yml') ? null : filename } - ] - } - - withName: CUSTOM_DUMPSOFTWAREVERSIONS { - publishDir = [ - path: { "${params.outdir}/pipeline_info" }, - mode: params.publish_dir_mode, - pattern: '*_versions.yml' - ] - } - withName: "FORCE_GENERATE_ANALYSIS_MASK" { - memory = { 500.MB * task.attempt } - time = { 20.minute * task.attempt } publishDir = [ path: { "${params.outdir}/preparation/" }, mode: params.publish_dir_mode, - pattern: '**.tif' + pattern: '**.tif', + enabled: params.publish_dir_enabled ] } withName: "FORCE_GENERATE_TILE_ALLOW_LIST" { - memory = { 500.MB * task.attempt } - time = { 20.minute * task.attempt } publishDir = [ path: { "${params.outdir}/preparation/" }, pattern: '*.txt', - mode: params.publish_dir_mode + mode: params.publish_dir_mode, + enabled: params.publish_dir_enabled ] } withName: "FORCE_PREPROCESS" { - cpus = 4 - memory = { 4500.MB * task.attempt } - time = { 2.h * task.attempt } - errorStrategy = 'retry' - maxRetries = 5 publishDir = [ [ path: { "${params.outdir}/preprocess/${task.tag}/logs" }, mode: params.publish_dir_mode, - pattern: '*.log' + pattern: '**.log', + enabled: params.publish_dir_enabled ], [ path: { "${params.outdir}/preprocess/${task.tag}" }, - mode: 'symlink', - pattern: 'level2_ard/**/*' + mode: params.publish_dir_mode, + pattern: 'level2_ard/**/*', + saveAs: { params.save_ard ? it : null }, + enabled: params.publish_dir_enabled ] ] } - withName: "MERGE_BOA" { - memory = { 2000.MB * task.attempt } - time = { 2.h * task.attempt } - } - - withName: "MERGE_QAI" { - memory = { 2000.MB * task.attempt } - time = { 2.h * task.attempt } - } withName: "HIGHER_LEVEL_CONFIG" { - errorStrategy = 'retry' - maxRetries = 5 publishDir = [ path: { "${params.outdir}/higher-level/${task.tag}/param_files" }, mode: params.publish_dir_mode, - pattern: '*.prm' + pattern: '*.prm', + enabled: params.publish_dir_enabled ] } withName: "FORCE_HIGHER_LEVEL" { - cpus = 6 - memory = { (params.only_tile ? 12000.MB : 3300.MB ) * task.attempt } - time = { 2.h * task.attempt } publishDir = [ path: { "${params.outdir}/higher-level/${task.tag}" }, - mode: 'symlink', + mode: params.publish_dir_mode, pattern: 'trend/*.tif', - saveAs: { "trend_files/${it.tokenize('/')[-1]}" } + saveAs: { params.save_ard ? "trend_files/${it.tokenize('/')[-1]}" : null }, + enabled: params.publish_dir_enabled ] } withName: "FORCE_PYRAMID" { - memory = { 1500.MB * task.attempt } - time = { 20.minute * task.attempt } stageInMode = 'copy' publishDir = [ - path: "${params.outdir}/trend/pyramid/", - saveAs: { "${it.substring(12,it.indexOf("."))}/trend/${it.substring(0,11)}/$it" }, - pattern: '*.tif*', - mode: params.publish_dir_mode + [ + path: { "${params.outdir}/trend/pyramid/" }, + saveAs: { "${it.substring(12,it.indexOf("."))}/trend/${it.substring(0,11)}/$it" }, + pattern: '*.tif*', + mode: params.publish_dir_mode, + enabled: params.publish_dir_enabled + ], + [ + path: { "${params.outdir}/trend/pyramid/" }, + pattern: 'versions.yml', + mode: params.publish_dir_mode, + enabled: params.publish_dir_enabled + ] ] } withName: "FORCE_MOSAIC" { - memory = { 1500.MB * task.attempt } - time = { 20.minute * task.attempt } publishDir = [ - path: "${params.outdir}/trend/mosaic/", + path: { "${params.outdir}/trend/mosaic/" }, mode: params.publish_dir_mode, - saveAs: { it.equals('versions.yml') ? null : "${params.outdir}/trend/mosaic/${task.tag}/${it.replaceAll("trend/","")}"} + saveAs: {"${params.outdir}/trend/mosaic/${task.tag}/${it.replaceAll("trend/","")}"}, + enabled: params.publish_dir_enabled ] } withName: "CHECK_RESULTS" { - errorStrategy = { task.exitStatus == 143 ? 'retry' : 'ignore' } - memory = { 14000.MB * task.attempt } - time = { 2.h * task.attempt } + publishDir = [ + enabled: false + ] + } + + withName: "CHECK_RESULTS_FULL" { publishDir = [ enabled: false ] } withName: "PREPROCESS_CONFIG" { - errorStrategy = 'retry' - maxRetries = 5 publishDir = [ path: { "${params.outdir}/preprocess/${task.tag}/param_files" }, mode: params.publish_dir_mode, - pattern: '*.prm' + pattern: '*.prm', + enabled: params.publish_dir_enabled ] } withName: 'MULTIQC' { ext.args = params.multiqc_title ? "--title \"$params.multiqc_title\"" : '' publishDir = [ - path: { "${params.outdir}/multiqc" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename.equals('versions.yml') ? null : filename } + path: { "${params.outdir}/multiqc" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename.equals('versions.yml') ? null : filename }, + enabled: params.publish_dir_enabled ] } withName: "UNTAR_*" { publishDir = [ - path: { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" }, - mode: 'symlink', - saveAs: { filename -> filename.equals('versions.yml') ? null : filename } + path: { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" }, + mode: 'symlink', + saveAs: { filename -> filename.equals('versions.yml') ? null : filename }, + enabled: params.publish_dir_enabled ] } diff --git a/conf/test.config b/conf/test.config index f6bd6ee..c874060 100644 --- a/conf/test.config +++ b/conf/test.config @@ -10,24 +10,23 @@ ---------------------------------------------------------------------------------------- */ +process { + resourceLimits = [ + cpus: 4, + memory: '15.GB', + time: '1.h' + ] +} + params { config_profile_name = 'Test profile' config_profile_description = 'Minimal test dataset to check pipeline function' - // Limit resources so that this can run on GitHub Actions - max_cpus = 2 - max_memory = '6.GB' - max_time = '6.h' - // Input data input = 'https://github.com/nf-core/test-datasets/raw/rangeland/Landsat_collection2/Landsat_data.tar.gz' dem = 'https://github.com/nf-core/test-datasets/raw/rangeland/dem/dem.tar.gz' wvdb = 'https://github.com/nf-core/test-datasets/raw/rangeland/wvp/wvdb.tar.gz' - input_tar = true - dem_tar = true - wvdb_tar = true - data_cube = 'https://github.com/nf-core/test-datasets/raw/rangeland/datacube/datacube-definition.prj' aoi = 'https://github.com/nf-core/test-datasets/raw/rangeland/vector/aoi.gpkg' endmember = 'https://github.com/nf-core/test-datasets/raw/rangeland/endmember/hostert-2003.txt' @@ -36,28 +35,17 @@ params { start_date = '1987-01-01' end_date = '1989-12-31' - sensors_level1 = 'LT04,LT05' sensors_level2 = 'LND04 LND05' - // Reference data - woody_change_ref = 'https://github.com/nf-core/test-datasets/raw/rangeland/reference/woody_cover_chg_ref.tif' - woody_yoc_ref = 'https://github.com/nf-core/test-datasets/raw/rangeland/reference/woody_cover_yoc_ref.tif' - - herbaceous_change_ref = 'https://github.com/nf-core/test-datasets/raw/rangeland/reference/herbaceous_cover_chg_ref.tif' - herbaceous_yoc_ref = 'https://github.com/nf-core/test-datasets/raw/rangeland/reference/herbaceous_cover_yoc_ref.tif' - - peak_change_ref = 'https://github.com/nf-core/test-datasets/raw/rangeland/reference/peak_chg_ref.tif' - peak_yoc_ref = 'https://github.com/nf-core/test-datasets/raw/rangeland/reference/peak_yoc_ref.tif' - // Other parameters group_size = 10 - validationSchemaIgnoreParams = "peak_yoc_ref,peak_change_ref,herbaceous_yoc_ref,herbaceous_change_ref,woody_yoc_ref,woody_change_ref,config_profile_description,config_profile_name" + // enable mosaic visualization + mosaic_visualization = true } process { withName: "UNTAR_*" { - container = 'docker.io/ubuntu:23.10' ext.args2 = "--strip-components=0" } } diff --git a/conf/test_full.config b/conf/test_full.config index 21fc4b4..b2df818 100644 --- a/conf/test_full.config +++ b/conf/test_full.config @@ -14,11 +14,32 @@ params { config_profile_name = 'Full test profile' config_profile_description = 'Full test dataset to check pipeline function' - // Input data for full size test - // TODO nf-core: Specify the paths to your full test data ( on nf-core/test-datasets or directly in repositories, e.g. SRA) - // TODO nf-core: Give any required params for the test so that command line flags are not needed - input = 'https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/samplesheet/samplesheet_full_illumina_amplicon.csv' + // Input data + input = 's3://ngi-igenomes/test-data/rangeland/landsat.tar' + dem = 's3://ngi-igenomes/test-data/rangeland/dem.tar' + wvdb = 's3://ngi-igenomes/test-data/rangeland/wvdb.tar' - // Fasta references - fasta = 'https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/NC_045512.2/GCF_009858895.2_ASM985889v3_genomic.200409.fna.gz' + data_cube = 's3://ngi-igenomes/test-data/rangeland/datacube-definition.prj' + aoi = 's3://ngi-igenomes/test-data/rangeland/aoi.gpkg' + endmember = 's3://ngi-igenomes/test-data/rangeland/hostert-2003.txt' + + + // Remote sensing imagery parameters + start_date = '1986-01-01' + end_date = '1989-12-31' + + sensors_level2 = 'LND04 LND05' + + // Enable time series stack output + return_tss = true + + // Enable mosaic visualization + mosaic_visualization = true +} + +process { + // This is required because of the internal structure of the input tarballs + withName: "UNTAR_*" { + ext.args2 = "--strip-components=1" + } } diff --git a/docs/images/mqc_fastqc_adapter.png b/docs/images/mqc_fastqc_adapter.png deleted file mode 100755 index 361d0e4..0000000 Binary files a/docs/images/mqc_fastqc_adapter.png and /dev/null differ diff --git a/docs/images/mqc_fastqc_counts.png b/docs/images/mqc_fastqc_counts.png deleted file mode 100755 index cb39ebb..0000000 Binary files a/docs/images/mqc_fastqc_counts.png and /dev/null differ diff --git a/docs/images/mqc_fastqc_quality.png b/docs/images/mqc_fastqc_quality.png deleted file mode 100755 index a4b89bf..0000000 Binary files a/docs/images/mqc_fastqc_quality.png and /dev/null differ diff --git a/docs/images/nf-core-rangeland_logo_dark.png b/docs/images/nf-core-rangeland_logo_dark.png index 3c32b0d..ea231fc 100644 Binary files a/docs/images/nf-core-rangeland_logo_dark.png and b/docs/images/nf-core-rangeland_logo_dark.png differ diff --git a/docs/images/nf-core-rangeland_logo_light.png b/docs/images/nf-core-rangeland_logo_light.png index 118b325..65cbbb8 100644 Binary files a/docs/images/nf-core-rangeland_logo_light.png and b/docs/images/nf-core-rangeland_logo_light.png differ diff --git a/docs/output.md b/docs/output.md index 294a88f..8769c5a 100644 --- a/docs/output.md +++ b/docs/output.md @@ -4,17 +4,20 @@ This document describes the output produced by the pipeline. -The directories listed below will be created in the results directory after the pipeline has finished. All paths are relative to the top-level results directory. +The directories listed below will be created in the results directory after the pipeline has finished. +All paths are relative to the top-level results directory. + +Note that running this pipeline with `--publish_dir_enabled false` will prevent any module from publishing its output. See [Usage](usage.md#module-output-publishing) for details ## Pipeline overview The pipeline is built using [Nextflow](https://www.nextflow.io/) and processes data using the following steps: -- [untar](#untar) - Optionally extract input files -- [Preparation](#preparation)- Create a masks and boundaries for further analyses. +- [Untar](#untar) - Optionally extract input files +- [Preparation](#preparation) - Create a masks and boundaries for further analyses. - [Preprocessing](#preprocessing) - Preprocessing of satellite imagery. - [Higher-level-Processing](#higher-level-processing) - Classify preprocessed imagery and perform time series analyses. -- [Visualization](#visualization). Create two visualizations of the results. +- [Visualization](#visualization) - Create two visualizations of the results. - [MultiQC](#multiqc) - Aggregate report describing results and QC from the whole pipeline - [Pipeline information](#pipeline-information) - Report metrics generated during the workflow execution @@ -24,15 +27,23 @@ The pipeline is built using [Nextflow](https://www.nextflow.io/) and processes d Output files - `untar/` - - ``: directory containing symlinks to decompressed digital elevation input data. Only present if a tar archive was provided for the digital elevation model. Name of the directory derived from archive contents. - - ``: directory containing symlinks to decompressed water vapor input data. Only present if a tar archive was provided for water vapor data. Name of the directory derived from archive contents. - - ``: directory containing symlinks to decompressed satellite imagery input data. Only present if a tar archive was provided for satellite data. Name of the directory derived from archive contents. + - ``: directory containing symlinks to decompressed digital elevation input data. + Only present if a tar archive was provided for the digital elevation model. + Name of the directory derived from archive contents. + - ``: directory containing symlinks to decompressed water vapor input data. + Only present if a tar archive was provided for water vapor data. + Name of the directory derived from archive contents. + - ``: directory containing symlinks to decompressed satellite imagery input data. + Only present if a tar archive was provided for satellite data. + Name of the directory derived from archive contents. [untar](https://nf-co.re/modules/untar) is a nf-core module used to extract files from tar archives. -Invokation of untar depends on certain parameters (i.e input_tar, dem_tar and wvdb_tar). Thus, the outputs files are only generated when these are set to true. +[untar](https://nf-co.re/modules/untar) is automatically executed when certain input parameters where given as `.tar` or `.tar.gz` files. +The parameters `--input`, `--dem` and `--wvdb` are supported. +See [Usage](usage.md) for details. ### Preparation @@ -40,16 +51,23 @@ Invokation of untar depends on certain parameters (i.e input_tar, dem_tar and wv Output files - `preparation/` - - `tile_allow.txt`: File containing all [FORCE](https://force-eo.readthedocs.io/en/latest/index.html) notation tiles of the earths surface that should be used further in the pipeline. The first line contains the number of tiles. Following lines contain tile identifiers. - - `mask/`: Directory containing a subdirectory for every [FORCE](https://force-eo.readthedocs.io/en/latest/index.html) tile. Each subdirectory contains the `aoi.tif` file. This file represents a binary mask layer that indicates which pixels are eligible for analyses. + - `tile_allow.txt`: File containing all [FORCE](https://force-eo.readthedocs.io/en/latest/index.html) notation tiles of the earths surface that should be used further in the pipeline. + The first line contains the number of tiles. + Following lines contain tile identifiers. + - `mask/`: Directory containing a subdirectory for every [FORCE](https://force-eo.readthedocs.io/en/latest/index.html) tile. + Each subdirectory contains the `aoi.tif` file. + This file represents a binary mask layer that indicates which pixels are eligible for analyses. In the preparation step, usable tiles and pixels per tile are identified. -[force-tile-extent](https://force-eo.readthedocs.io/en/latest/components/auxilliary/tile-extent.html#force-tile-extent) analyses the area of interest information and determines the tiles that can be used. These tiles are later used by other [FORCE](https://force-eo.readthedocs.io/en/latest/index.html) submodules. +[force-tile-extent](https://force-eo.readthedocs.io/en/latest/components/auxilliary/tile-extent.html#force-tile-extent) analyses the area of interest information and determines the tiles that can be used. +These tiles are later used by other [FORCE](https://force-eo.readthedocs.io/en/latest/index.html) submodules. -[force-cube](https://force-eo.readthedocs.io/en/latest/components/auxilliary/cube.html#force-cube) computes the usable pixels for each [FORCE](https://force-eo.readthedocs.io/en/latest/index.html) tile. This computation is based on the specified are of interest and the resolution. The resulting binary masks can be used to understand which pixels were discarded (e.g. because they only contain water). +[force-cube](https://force-eo.readthedocs.io/en/latest/components/auxilliary/cube.html#force-cube) computes the usable pixels for each [FORCE](https://force-eo.readthedocs.io/en/latest/index.html) tile. +This computation is based on the specified are of interest and the resolution. +The resulting binary masks can be used to understand which pixels were discarded (e.g. because they only contain water). ### Preprocessing @@ -58,7 +76,8 @@ In the preparation step, usable tiles and pixels per tile are identified. - `preprocess//` - `param_files/`: Directory containing parameter files for [FORCE](https://force-eo.readthedocs.io/en/latest/index.html) preprocessing modules. One file per satellite mission per tile. - - `level2_ard/`: Directory containing symlinks to analysis-ready-data. Subdirectories contain the .tif files that were generated during preprocessing. + - `level2_ard/`: Directory containing symlinks to analysis-ready-data. + Subdirectories contain the .tif files that were generated during preprocessing. - `logs/`: Logs from preprocessing. @@ -67,7 +86,11 @@ Preprocessing consist of two parts, generating parameter files and actual prepro The parameter files created through [force-parameter](https://force-eo.readthedocs.io/en/latest/components/auxilliary/parameter.html#force-parameter) can be viewed to understand concrete preprocessing techniques applied for a given tile. -Logs and analysis-ready-data (ARD) are generated using the [force-l2ps](https://force-eo.readthedocs.io/en/latest/components/lower-level/level2/l2ps.html) command. Logs can be consulted for debugging purposes. ARD may be collected as a basis for other remote sensing workflows. Moreover, ARD contains two .tif files per initial input image, a quality data file and the atmospherically corrected satellite data, that can be viewed using geographic information systems (GISs). Note that ARD data is only published as symbolic links due to the amount and size of the files. +Logs and analysis-ready-data (ARD) are generated using the [force-l2ps](https://force-eo.readthedocs.io/en/latest/components/lower-level/level2/l2ps.html) command. +Logs can be consulted for debugging purposes. +ARD may be collected as a basis for other remote sensing workflows. +The ARD in `level2_ard/` consist two `.tif` files per initial input image, a quality data file and the atmospherically corrected satellite data. +Note that the `.tif` files are only published when the `--save_ard` parameter is set to `true` to avoid bloating the storage. ### Higher-level-Processing @@ -76,15 +99,28 @@ Logs and analysis-ready-data (ARD) are generated using the [force-l2ps](https:// - `higher-level//` - `param_files/`: Parameter files used in [force-higher-level](https://force-eo.readthedocs.io/en/latest/components/higher-level/index.html). - - `trend_files`: Symlinks to trend files that are the result of higher-level processing. + - `trend_files/`: Symlinks to trend files that are the result of higher-level processing. + This may optionally contain the time series stack. Higher level processing consist of two parts, generating parameter files and performing various processing task as defined in the parameter files. -Parameter files may be consulted to derive information about the specific processing task performed for a given tile. In this workflow a classification, optionally using spectral unmixing, is conducted. Next, time series analysis for different characteristics is performed. +Parameter files may be consulted to derive information about the specific processing task performed for a given tile. +In this workflow, classification using spectral unmixing is performed. + +Spectral unmixing is a common technique to derive sub-pixel classification. +Concretely, a set of endmember (provided using `--endmember`) is exploited to determine fractions of different types of vegetation, soil, ... for each pixel. + +Next, time series analysis for different vegetation characteristics is performed. + +The resulting trend files in `trend_files/` can be investigated to view trends for individual tiles. +However, these files are only published if the `--save_tsa` parameter is set to `true`. -The resulting trend files can be consulted to view trends for individual tiles. They are saved as symlinks because of their large size. +If the `--return_tss` parameter was set to `true`, the pipeline will also output `.tif` files with the `TSS` in their name. +These files contain the time series stack(TSS) for the given tile and index or band. +Here, for each date of acquisition, an image is available that contains the values for that date. +TSS files will not be returned if `--save_tsa` is set to `false`. ### Visualization @@ -93,13 +129,17 @@ The resulting trend files can be consulted to view trends for individual tiles. - `trend/` - `mosaic//` - - ``: Auxiliary files for the mosaic visualization. - - `mosaic`: Contains a single virtual raster file that defines the mosaic visualization. + - `/`: .tif files that are part of the mosaic. + - `mosaic/`: Contains a single virtual raster file that combines the .tif files into the mosaic visualization. - `pyramid//trend//`: Contains tile-wise pyramid visualizations for every trend analyzed in the workflow. -Two types of common visualizations are generated in the last step of the pipeline. They are results of [force-mosaic](https://force-eo.readthedocs.io/en/latest/components/auxilliary/mosaic.html) and [force-pyramid](https://force-eo.readthedocs.io/en/latest/components/auxilliary/pyramid.html). Note that these visualizations do not add more logic to the workflow but rather rearrange the output files of higher-level-processing. +Two types of common visualizations are generated in the last step of the pipeline. +They are results of [force-mosaic](https://force-eo.readthedocs.io/en/latest/components/auxilliary/mosaic.html) and [force-pyramid](https://force-eo.readthedocs.io/en/latest/components/auxilliary/pyramid.html). +Note that these visualizations do not add more logic to the workflow but rather rearrange the output files of higher-level-processing. +Both visualizations are enabled by default but may be disabled in a certain configuration files. +Thus, these outputs are optional. ### MultiQC @@ -113,9 +153,11 @@ Two types of common visualizations are generated in the last step of the pipelin -[MultiQC](http://multiqc.info) is a visualization tool that generates a single HTML report summarising all samples in your project. Most of the pipeline QC results are visualised in the report and further statistics are available in the report data directory. +[MultiQC](http://multiqc.info) is a visualization tool that generates a single HTML report summarising all samples in your project. +Most of the pipeline QC results are visualised in the report and further statistics are available in the report data directory. -Results generated by MultiQC collate pipeline QC from supported tools e.g. FastQC. The pipeline has special steps which also allow the software versions to be reported in the MultiQC output for future traceability. For more information about how to use MultiQC reports, see . +Results generated by MultiQC collate pipeline QC from supported tools. The pipeline has special steps which also allow the software versions to be reported in the MultiQC output for future traceability. +For more information about how to use MultiQC reports, see . ### Pipeline information @@ -126,7 +168,9 @@ Results generated by MultiQC collate pipeline QC from supported tools e.g. FastQ - Reports generated by Nextflow: `execution_report.html`, `execution_timeline.html`, `execution_trace.txt` and `pipeline_dag.dot`/`pipeline_dag.svg`. - Reports generated by the pipeline: `pipeline_report.html`, `pipeline_report.txt` and `software_versions.yml`. The `pipeline_report*` files will only be present if the `--email` / `--email_on_fail` parameter's are used when running the pipeline. - Reformatted samplesheet files used as input to the pipeline: `samplesheet.valid.csv`. + - Parameters used by the pipeline run: `params.json`. -[Nextflow](https://www.nextflow.io/docs/latest/tracing.html) provides excellent functionality for generating various reports relevant to the running and execution of the pipeline. This will allow you to troubleshoot errors with the running of the pipeline, and also provide you with other information such as launch commands, run times and resource usage. +[Nextflow](https://www.nextflow.io/docs/latest/tracing.html) provides excellent functionality for generating various reports relevant to the running and execution of the pipeline. +This will allow you to troubleshoot errors with the running of the pipeline, and also provide you with other information such as launch commands, run times and resource usage. diff --git a/docs/usage.md b/docs/usage.md index f32c2aa..7f73f5f 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -4,24 +4,27 @@ > _Documentation of pipeline parameters is generated automatically from the pipeline schema and can no longer be found in markdown files._ -## Introduction - ## Input -As most remote sensing workflows, this pipeline relies on numerous sources of data. In the following we will describe the required data and corresponding formats. Mandatory input data consists of satellite data, a digital elevation model, a water vapor database, a data_cube, an area-of-interest specification and an endmember definition. +As most remote sensing workflows, this pipeline relies on numerous sources of data. +In the following we will describe the required data and corresponding formats. +Mandatory input data consists of satellite data, a digital elevation model, a water vapor database, a data_cube, an area-of-interest specification and an endmember definition. ### Satellite data -This pipeline operates on Landsat data. Landsat is a joint NASA/U.S. Geolical Survey satellite mission that provides continuous Earth obersvation data since 1984 at 30m spatial resolution with a temporal revisit frequency of 8-16 days. -Landsast carries multispectral optical instruments that observe the land surface in the visible to shortwave infrared spectrum. -For infos on Landsat, see [here](https://www.usgs.gov/core-science-systems/nli/landsat). +This pipeline operates on Landsat data. +Landsat is a joint NASA/U.S. Geolical Survey satellite mission that provides continuous Earth obersvation data since 1984 at 30m spatial resolution with a temporal revisit frequency of 8-16 days. +Landsat satellites carry multispectral optical instruments that observe the land surface in the visible to shortwave infrared spectrum. +For information on Landsat, see [here](https://www.usgs.gov/core-science-systems/nli/landsat). -Satellite data should be given as a path to a common root of all imagery. This is a common format used in geographic information systems, including FORCE, which is applied in this pipeline. The expected structure underneath the root directory should follow this example: +Satellite data should be given as a path to a common root of all imagery. +This is a common format used in geographic information systems, including FORCE, which is applied in this pipeline. +The expected structure underneath the root directory should follow this example: -``` +```tree root -├── 181035 -│ └── LE07_L1TP_181035_20061217_20170106_01_T1 +├── 181035 # path/row id +│ └── LE07_L1TP_181035_20061217_20170106_01_T1 # satellite, path/row id, date of acquisition, date of processing, product number, collection tier │ | ├── LE07_L1TP_181035_20061217_20170106_01_T1_ANG.txt │ | ├── LE07_L1TP_181035_20061217_20170106_01_T1_B1.TIF │ | ├── LE07_L1TP_181035_20061217_20170106_01_T1_B2.TIF @@ -36,8 +39,8 @@ root │ | ├── LE07_L1TP_181035_20061217_20170106_01_T1_GCP.txt │ | └── LE07_L1TP_181035_20061217_20170106_01_T1_MTL.txt | └── ... -├── 181036 -│ └── LE07_L1TP_181036_20061217_20170105_01_T1 +├── 181036 # path/row id +│ └── LE07_L1TP_181036_20061217_20170105_01_T1 # satellite, path/row id, date of acquisition, date of processing, product number, collection tier │ | ├── LE07_L1TP_181036_20061217_20170105_01_T1_ANG.txt │ | ├── LE07_L1TP_181036_20061217_20170105_01_T1_B1.TIF │ | ├── LE07_L1TP_181036_20061217_20170105_01_T1_B2.TIF @@ -55,9 +58,11 @@ root └── ... ``` -Subdirectories of root contain _path_ and _row_ information as commonly used for Landsat imagery. As an example, the sub directory `181036/` contains imagery for path 18 and row 1036. +Subdirectories of `root/` contain _path_ and _row_ information as commonly used for Landsat imagery. +For example, the sub directory `181036/` contains imagery for path 18 and row 1036. -The next level of subdirectories contains the data for a specific day and from a specific source. Lets look at the example `LE07_L1TP_181036_20061217_20170105_01_T1`: +The next level of subdirectories contains the data for a specific day and from a specific source. +Lets look at the example `LE07_L1TP_181036_20061217_20170105_01_T1`: - "LE07" corresponds to Landsat 7 Enhanced - "L1TP" corresponds to Level-1 Terrain Corrected imagery @@ -67,17 +72,23 @@ The next level of subdirectories contains the data for a specific day and from a - "01" corresponds to version number of the remote sensing product - "T1" corresponds to the Tier of the data collection, which indicates the Tier 1 landsat collection in this case -On the lowest level of the structure, the actual data is stored. Looking at the contents of `LE07_L1TP_181036_20061217_20170105_01_T1`, we see that all files share the same prefix, followed by a specification of the specific files contents. These suffixes include: +On the lowest level of the structure, the actual data is stored. +Looking at the contents of `LE07_L1TP_181036_20061217_20170105_01_T1`, we see that all files share the same prefix, followed by a specification of the specific files contents. +These suffixes include: - "B" followed by a number _i_ identifying the band of the satellite (band 6 has two files as Landsat 7 has two thermal bands) -- "BQA" identifying the quality information band +- "BQA" identifies the quality information band - "GCP" identifies ground control point information - "ANG" identifies angle of observation and other geometric information information - "MTL" identifies meta data -All files within the lowest level of structure belong to a single observation. Files containing imagery (prefix starts with "B") should be .tif files. Files containing auxiliary data are text files. +All files within the lowest level of structure belong to a single observation. +Files containing imagery (prefix starts with "B") should be `.tif` files. +Files containing auxiliary data are text files. -This structure is automatically generated when [using force to download the data](https://force-eo.readthedocs.io/en/latest/components/lower-level/level1/level1-csd.html?). We strongly suggest users to download data using FORCE (e.g.). For example, executing the following code (e.g. with [FORCE in docker](https://force-eo.readthedocs.io/en/latest/setup/docker.html)) will download data for Landsat 4,5 and 7, in the time range from 1st January 1984 until 31st December 2006, including pictures with up to 70 percent of cloud coverage: +This structure is automatically generated when [using FORCE to download the data](https://force-eo.readthedocs.io/en/latest/components/lower-level/level1/level1-csd.html?). +We strongly suggest users to download data using FORCE. +For example, executing the following code (e.g. with [FORCE in docker](https://force-eo.readthedocs.io/en/latest/setup/docker.html)) will download data for Landsat 4,5 and 7, in the time range from 1st January 1984 until 31st December 2006, including pictures with up to 70 percent of cloud coverage: ```bash mkdir -p meta @@ -86,7 +97,8 @@ mkdir -p data force-level1-csd -s "LT04,LT05,LE07" -d "19840101,20061231" -c 0,70 meta/ data/ queue.txt vector/aoi.gpkg ``` -Note that you need to pass an area-of-interest file, see the area of interest section [Area of interest](#aoi) for details. +Note that an area-of-interest file has to be passed, see the area of interest section [Area of interest](#area-of-interest-aoi) for details. +In addition, downloading data using FORCE may require access the machine-to-machine interfaces. The satellite imagery can be given to the pipeline using: @@ -94,15 +106,21 @@ The satellite imagery can be given to the pipeline using: --input '[path to imagery root]' ``` -The satellite imagery can also be provide as a tar archive. In this case it is mandatory to set `--input_tar` to true. Moreover, within the tar archive, the structure explained above has to be in place. In the example above `181036/` and `181035/` would need to be in the top level of the archive. +The satellite imagery can also be provide as a tarball (`.tar` or `.tar.gz` files). +These files will be automatically extracted. +Providing tarballs can be specifically helpful when using foreign files as inputs. +In this case, it is mandatory to have the structure explained above in place. +In the example above `181036/` and `181035/` would need to be in the top level of the archive. ### Digital Elevation Model (DEM) -A DEM is necessary for topographic correction of Landsat data, and helps to distinguish between cloud, shadows and water surfaces. Common sources for digital elevation models are [Copernicus](https://www.copernicus.eu/en),[Shuttle Radar Topography Mission](https://www2.jpl.nasa.gov/srtm/) (SRTM), or [Advanced Spaceborne Thermal Emission and Reflection Radiometer](https://asterweb.jpl.nasa.gov/) (ASTER). +A DEM is necessary for topographic correction of Landsat data, and helps to distinguish between cloud, shadows and water surfaces. +Common sources for digital elevation models are [Copernicus](https://www.copernicus.eu/en),[Shuttle Radar Topography Mission](https://www2.jpl.nasa.gov/srtm/) (SRTM), or [Advanced Spaceborne Thermal Emission and Reflection Radiometer](https://asterweb.jpl.nasa.gov/) (ASTER). -The pipeline expects a path to the Digital Elevation Model root directory as a parameter. Concretely, the expected structure would look like this: +The pipeline expects a path to the digital elevation model root directory as the `--dem` parameter. +Concretely, the expected structure would look like this: -``` +```tree dem ├── .vrt └── / @@ -117,7 +135,11 @@ The DEM can be given to the pipeline using: --dem '[path to dem root]' ``` -The digital elevation model can also be provide as a tar archive. In this case it is mandatory to set `--dem_tar` to true. Moreover, within the tar archive, the structure explained above has to be in place. In the example above `.vrt` and `/` would need to be in the top level of the archive. +The digital elevation model can also be provide as a tarball (`.tar` or `.tar.gz` files). +These files will be automatically extracted. +Providing tarballs can be specifically helpful when using foreign files as inputs. +In this case, it is mandatory to have the structure explained above in place. +In the example above `.vrt` and `/` would need to be in the top level of the archive. ### Water Vapor Database (WVDB) @@ -140,11 +162,16 @@ The WVDB can be given to the pipeline using: --wvdb '[path to wvdb dir]' ``` -The water vapor database can also be provide as a tar archive. In this case it is mandatory to set `--wvdb_tar` to true. All files of the wvdb would need to be in the top level of the archive. +The water vapor database can also be provide as a tarball (`.tar` or `.tar.gz` files). +These files will be automatically extracted. +Providing tarballs can be specifically helpful when using foreign files as inputs. +In this case, it is mandatory to have the structure explained above in place. +All files of the wvdb would need to be in the top level of the archive. ### Datacube -The datacube definition stores information about the projection and reference grid of the generated datacube. For details see the [FORCE main paper](https://www.mdpi.com/2072-4292/11/9/1124). +The datacube definition stores information about the projection and reference grid of the generated datacube. +For details see the [FORCE main paper](https://www.mdpi.com/2072-4292/11/9/1124). The datacube definition is passed as a single file using: @@ -154,11 +181,10 @@ The datacube definition is passed as a single file using: ### Area of interest (AOI) - - The area of interest is a geospatial vector dataset that holds the boundary of the targeted area. +The file must be a shapefile or geopackage vector file. -AOI is passed as a single using: +AOI is passed as a single file using: ```bash --aoi '[path to area of interest file]' @@ -170,7 +196,7 @@ For unmixing satellite-observed reflectance into sub-pixel fractions of land sur An example endmember definition (developed in [Hostert et al. 2003](https://www.sciencedirect.com/science/article/abs/pii/S0034425703001457)) looks like this: -``` +```tsv 320 730 2620 0 560 1450 3100 0 450 2240 3340 0 @@ -179,7 +205,8 @@ An example endmember definition (developed in [Hostert et al. 2003](https://www. 710 3220 5490 0 ``` -Each colum represents a different endmember. Columns represent Landsat bands (R,G,B, NIR, SWIR1, SWIR2). +Each colum represents a different endmember. +Columns represent Landsat bands (R,G,B, NIR, SWIR1, SWIR2). The endmembers can be passed in a single text-file using: @@ -193,17 +220,12 @@ Users can specify additional parameters to configure how the underlying workflow ### Sensor Levels -Data from different satellites can be processed within this workflow. Users may wish to include different satellites in preprocessing and in higher level processing. To control this behavior, two parameters can be set when the pipeline is launched. -The first parameter - `sensors_level1` - controls the selection of satellites for preprocessing. This parameter should follow the FORCE notation for level 1 processing of satellites. Concretely, a string containing comma-separated satellite identifiers has to be supplied (e.g. `"LT04,LT05"` to include Landsat 4 and 5). Available options for satellite identifiers are: - -- `"LT04"`: Landsat 4 TM -- `"LT05"`: Landsat 5 TM -- `"LE07"`: Landsat 7 ETM+ -- `"LC08"`: Landsat 8 OLI -- `"S2A"`: Sentinel-2A MSI -- `"S2B"`: Sentinel-2B MSI - -The second parameter - `sensors_level2` - controls the selection of satellites for the higher level processing steps. The parameter has to follow the FORCE notation for level 2 processing. In particular, a string containing space-separated satellite identifiers has to be supplied (e.g. `"LND04 LND05"` to include Landsat 4 and 5). Note that these identifiers differ from those used for the `sensors_level1` parameter. +Data from different satellites can be processed within this workflow. +Users may wish to include different satellites in preprocessing and in higher level processing. +All input imagery is preprocessed. +The `--sensors_level2` parameter controls the selection of satellites for the higher level processing steps. +The parameter has to follow the FORCE notation for level 2 processing. +In particular, a string containing space-separated satellite identifiers has to be supplied (e.g. `"LND04 LND05"` to include Landsat 4 and 5). More details on available satellite identifiers can be found [here](https://force-eo.readthedocs.io/en/latest/components/higher-level/tsa/param.html), some common options include: - `"LND04"`: 6-band Landsat 4 TM @@ -213,20 +235,27 @@ More details on available satellite identifiers can be found [here](https://forc - `"SEN2A"`: 10-band Sentinel-2A - `"SEN2B"`: 10-band Sentinel-2B -Note that the identifiers specified for both processing levels have to match the data made available to the workflow. In other words, satellite data for e.g. Landsat 5 can't be processed if it was not supplied using the `input` parameter. +Note that the specified identifiers have to match the data made available to the workflow. +In other words, satellite data for e.g. Landsat 5 can't be processed if it was not supplied using the `--input` parameter. -Both parameters can be passed as using: +The satellite identifiers can be passed as follows: ```bash ---sensors_level1 = '[preprocessing satellite identifier string]' --sensors_level2 = '[higher level processing satellite identifier string]' ``` -Note that both parameters are optional and are by default set to: `"LT04,LT05,LE07,S2A"` and `"LND04 LND05 LND07"`. Therefore, by default, the pipeline will use Landsat 4,5,7, and Sentinel 2 for preprocessing, while using Landsat 4,5 and 7 for higher level processing. +Note that the parameter is optional and the default value is: `"LND04 LND05 LND07"`. +Therefore, by default, the pipeline will use Landsat 4,5 and 7 imagery in higher level processing. ### Resolution -Resolution of satellite imagery defines the real size of a single pixel. As an example, a resolution of 30 meters indicates that a single pixel in the data covers a 30x30 meters square of the earths surface. Users can customize the resolution that FORCE should assume. This does not necessarily have to match the resolution of the supplied data. FORCE will treat imagery as having the specified resolution. However, passing a resolution not matching the satellite data might lead to unexpected results. Resolution is specified in meters. +Resolution of satellite imagery defines the real size of a single pixel. +For example, a resolution of 30 meters indicates that a single pixel in the data covers a 30x30 meters square of the earth's surface. +Users can customize the resolution that FORCE should assume. +This does not necessarily have to match the resolution of the supplied data. +FORCE will treat imagery as having the specified resolution. +However, passing a resolution not matching the satellite data might lead to unexpected results. +Resolution is specified in meters. A custom resolution can be passed using: @@ -234,11 +263,12 @@ A custom resolution can be passed using: --resolution '[integer]' ``` -The default value is 30, as most Landsat satellite natively provide this resolution. +The default value is `30`, as most Landsat satellite natively provide this resolution. ### Temporal extent -In some scenarios, user may be interested to limit the temporal extent of analysis. To enables this, users can specify both start and end date in a string with this syntax: `'YYYY-MM-DD'`. +In some scenarios, user may be interested to limit the temporal extent of analysis. +To enables this, users can specify both start and end date in a string with this syntax: `'YYYY-MM-DD'`. Start and end date can be passed using: @@ -247,46 +277,96 @@ Start and end date can be passed using: --end_date '[YYYY-MM-DD]' ``` -Default values are `'1984-01-01'` for the start date and `'2006-12-31'` for the end date. +### Group size + +The `--group_size` parameter can be ignored in most cases. +It defines how many satellite scenes are processed together. +The parameter is used to balance the tradeoff between I/O and computational capacities on individual compute nodes. +By default, `--group_size` is set to `100`. + +The group size can be passed using: + +```bash +--group_size '[integer]' +``` + +### Higher level processing configuration -### Spectral Unmixing +During the higher level processing stage, time series analyses of different satellite bands and indexes is performed. +The concrete bands and indexes can be defined using the `--indexes` parameter. +Spectral unmixing is performed in any case. +Thus, passing an empty `--indexes` parameter will restrict time series analyses to the results of spectral unmixing. +All available indexes can be found [here](https://force-eo.readthedocs.io/en/latest/components/higher-level/tsa/param.html) above the `INDEX` entry. +The band/index codes need to be passed in a space-separated string. +The default value, `--indexes = "NDVI BLUE GREEN RED NIR SWIR1 SWIR2"`, enables time series analyses for the NDVI index and the blue, green, red, near-infrared and both shortwave infrared bands. +Note that indexes are usually computed based on certain bands. +If these bands are not present in the preprocessed data, these indexes can not be computed. -Spectral unmixing is a common technique to derive sub-pixel classification. Concretely, a set of endmember (provided using `--endmember`) is exploited to determine fractions of different types of vegetation, soil, $\ldots$ for each pixel. In this workflow, we users can enable spectral unmixing-based classification using the `only_tile` parameter. To enable spectral unmixing, user have to set the parameters to `true`, as this feature is disabled by default. +The bands and indexes can be passed using: -Spectral unmixing can be enabled or disabled using: +```bash +--indexes '[index-string]' +``` + +In so cases, it may be desirable to analyze the the individual images in a time series. +To enable such analysis, the parameter `--return_tss` can be used. +If set to `true`, the pipeline will return time series stacks for each tile and band combination. +The option is disabled by default to reduce the output size. + +The time series stack output can be enabled using: ```bash ---endmember '[true|false]' +--return_tss true ``` -### Group size +### Visualization -The `group_size` parameters can be ignored in most cases. It defines how many satellite scenes are processed together. The parameters is used to balance the tradeoff between I/O and computational capacities on individual compute nodes. By default, `group_size` is set to 100. +The workflow provides two types of results visualization and aggregation. +The fine grained mosaic visualization contains all time series analyses results for all tiles in the original resolution. +Pyramid visualizations present a broad overview of the same data but at a lower resolution. +Both visualizations can be enabled or disabled using the parameters `--mosaic_visualization` and `--pyramid_visualization`. +By default, both visualization methods are enabled. +Note that the mosaic visualization is required to be enabled when using the `test` and `test_full` profiles to allow the pipeline to check the correctness of its results. -The group size can be passed using: +The visualizations can be enabled using: ```bash ---group_size '[integer]' +--mosaic_visualization = true +--pyramid_visualization = true ``` -### FORCE configuration +### Intermediate data publishing -FORCE supports parallel computations. Users can specify the number of threads FORCE can spawn for a single preprocessing, or higher level processing process. This is archived through the `force_threads` parameter. +By default, preprocessing and higher level processing steps do not publish the `.tif` files that they generate to avoid bloating the available storage. +However, analysis-ready-data (aka. ARD or level-2 data) and the results of time series analyses (aka. level-3 data) maybe be interesting to certain users. +The files for preprocessing and higher level processing can, therefore, be published by setting the `--save_ard` and `--save_tsa` to `true`, respectively. -The number of threads can be passed using: +Publishing of intermediate data can be enabled using: ```bash ---force_threads '[integer]' +--save_ard = true +--save_tsa = true ``` -The default value is 2. +### Module output publishing + +Certain users may not be interested in the pipeline's outputs. +To disable all modules from publishing their outputs, the `--publish_dir_enabled` can be set to `false`. +Note that this affects _all modules_. +By default all modules publish their outputs according to the other parameters. + +Publishing of all module's outputs can be disabled using: + +```bash +--publish_dir_enabled = false +``` ## Running the pipeline The typical command for running the pipeline is as follows: ```bash -nextflow run nf-core/rangeland --input --dem --wvdb --data_cube --aoi --endmember --outdir -profile docker +nextflow run nf-core/rangeland -profile docker --input --dem --wvdb --data_cube --aoi --endmember --outdir ``` This will launch the pipeline with the `docker` configuration profile. See below for more information about profiles. @@ -300,9 +380,41 @@ work # Directory containing the nextflow working files # Other nextflow hidden files, eg. history of pipeline runs and old logs. ``` +If you wish to repeatedly use the same parameters for multiple runs, rather than specifying each flag in the command, you can specify these in a params file. + +Pipeline settings can be provided in a `yaml` or `json` file via `-params-file `. + +:::warning +Do not use `-c ` to specify parameters as this will result in errors. +Custom config files specified with `-c` must only be used for [tuning process resource specifications](https://nf-co.re/docs/usage/configuration#tuning-workflow-resources), other infrastructural tweaks (such as output directories), or module arguments (args). +::: + +The above pipeline run specified with a params file in yaml format: + +```bash +nextflow run nf-core/rangeland -profile docker -params-file params.yaml +``` + +with: + +```yaml title="params.yaml" +input: '' +dem: '' +wvdb: '' +data_cube: '' +aoi: '' +endmember: '' +outdir: './results/' +<...> +``` + +You can also generate such `YAML`/`JSON` files via [nf-core/launch](https://nf-co.re/launch). + ### Updating the pipeline -When you run the above command, Nextflow automatically pulls the pipeline code from GitHub and stores it as a cached version. When running the pipeline after this, it will always use the cached version if available - even if the pipeline has been updated since. To make sure that you're running the latest version of the pipeline, make sure that you regularly update the cached version of the pipeline: +When you run the above command, Nextflow automatically pulls the pipeline code from GitHub and stores it as a cached version. +When running the pipeline after this, it will always use the cached version if available - even if the pipeline has been updated since. +To make sure that you're running the latest version of the pipeline, make sure that you regularly update the cached version of the pipeline: ```bash nextflow pull nf-core/rangeland @@ -310,28 +422,46 @@ nextflow pull nf-core/rangeland ### Reproducibility -It is a good idea to specify a pipeline version when running the pipeline on your data. This ensures that a specific version of the pipeline code and software are used when you run your pipeline. If you keep using the same tag, you'll be running the same version of the pipeline, even if there have been changes to the code since. +It is a good idea to specify a pipeline version when running the pipeline on your data. +This ensures that a specific version of the pipeline code and software are used when you run your pipeline. +If you keep using the same tag, you'll be running the same version of the pipeline, even if there have been changes to the code since. -First, go to the [nf-core/rangeland releases page](https://github.com/nf-core/rangeland/releases) and find the latest pipeline version - numeric only (eg. `1.3.1`). Then specify this when running the pipeline with `-r` (one hyphen) - eg. `-r 1.3.1`. Of course, you can switch to another version by changing the number after the `-r` flag. +First, go to the [nf-core/rangeland releases page](https://github.com/nf-core/rangeland/releases) and find the latest pipeline version - numeric only (eg. `1.0.0`). +Then specify this when running the pipeline with `-r` (one hyphen) - eg. `-r 1.3.1`. Of course, you can switch to another version by changing the number after the `-r` flag. This version number will be logged in reports when you run the pipeline, so that you'll know what you used when you look back in the future. For example, at the bottom of the MultiQC reports. +To further assist in reproducbility, you can use share and re-use [parameter files](#running-the-pipeline) to repeat pipeline runs with the same settings without having to write out a command with every single parameter. + +:::tip +If you wish to share such profile (such as upload as supplementary material for academic publications), make sure to NOT include cluster specific paths to files, nor institutional specific profiles. +::: + ## Core Nextflow arguments -> **NB:** These options are part of Nextflow and use a _single_ hyphen (pipeline parameters use a double-hyphen). +:::note +These options are part of Nextflow and use a _single_ hyphen (pipeline parameters use a double-hyphen). +::: ### `-profile` -Use this parameter to choose a configuration profile. Profiles can give configuration presets for different compute environments. +Use this parameter to choose a configuration profile. +Profiles can give configuration presets for different compute environments. + +Several generic profiles are bundled with the pipeline which instruct the pipeline to use software packaged using different methods (Docker, Singularity, Podman, Shifter, Charliecloud, Apptainer, Conda) - see below. -Several generic profiles are bundled with the pipeline which instruct the pipeline to use software packaged using different methods (Docker, Singularity, Podman, Shifter, Charliecloud, Conda) - see below. +:::info +We highly recommend the use of Docker or Singularity containers for full pipeline reproducibility, however when this is not possible, Conda is also supported. +::: -> We highly recommend the use of Docker or Singularity containers for full pipeline reproducibility, however when this is not possible, Conda is also supported. +The pipeline also dynamically loads configurations from [https://github.com/nf-core/configs](https://github.com/nf-core/configs) when it runs, making multiple config profiles for various institutional clusters available at run time. +For more information and to see if your system is available in these configs please see the [nf-core/configs documentation](https://github.com/nf-core/configs#documentation). Note that multiple profiles can be loaded, for example: `-profile test,docker` - the order of arguments is important! They are loaded in sequence, so later profiles can overwrite earlier profiles. -If `-profile` is not specified, the pipeline will run locally and expect all software to be installed and available on the `PATH`. This is _not_ recommended, since it can lead to different results on different machines dependent on the computer enviroment. +If `-profile` is not specified, the pipeline will run locally and expect all software to be installed and available on the `PATH`. +This is _not_ recommended, since it can lead to different results on different machines dependent on the computer enviroment. - `test` - A profile with a complete configuration for automated testing @@ -346,127 +476,73 @@ If `-profile` is not specified, the pipeline will run locally and expect all sof - A generic configuration profile to be used with [Shifter](https://nersc.gitlab.io/development/shifter/how-to-use/) - `charliecloud` - A generic configuration profile to be used with [Charliecloud](https://hpc.github.io/charliecloud/) +- `apptainer` + - A generic configuration profile to be used with [Apptainer](https://apptainer.org/) +- `wave` + - A generic configuration profile to enable [Wave](https://seqera.io/wave/) containers. + Use together with one of the above (requires Nextflow ` 24.03.0-edge` or later). - `conda` - - A generic configuration profile to be used with [Conda](https://conda.io/docs/). Please only use Conda as a last resort i.e. when it's not possible to run the pipeline with Docker, Singularity, Podman, Shifter or Charliecloud. + - A generic configuration profile to be used with [Conda](https://conda.io/docs/). + Please only use Conda as a last resort i.e. when it's not possible to run the pipeline with Docker, Singularity, Podman, Shifter, Charliecloud, or Apptainer. ### `-resume` -Specify this when restarting a pipeline. Nextflow will use cached results from any pipeline steps where the inputs are the same, continuing from where it got to previously. For input to be considered the same, not only the names must be identical but the files' contents as well. For more info about this parameter, see [this blog post](https://www.nextflow.io/blog/2019/demystifying-nextflow-resume.html). +Specify this when restarting a pipeline. +Nextflow will use cached results from any pipeline steps where the inputs are the same, continuing from where it got to previously. +For input to be considered the same, not only the names must be identical but the files' contents as well. +For more info about this parameter, see [this blog post](https://www.nextflow.io/blog/2019/demystifying-nextflow-resume.html). -You can also supply a run name to resume a specific run: `-resume [run-name]`. Use the `nextflow log` command to show previous run names. +You can also supply a run name to resume a specific run: `-resume [run-name]`. +Use the `nextflow log` command to show previous run names. ### `-c` -Specify the path to a specific config file (this is a core Nextflow command). See the [nf-core website documentation](https://nf-co.re/usage/configuration) for more information. +Specify the path to a specific config file (this is a core Nextflow command). +See the [nf-core website documentation](https://nf-co.re/usage/configuration) for more information. ## Custom configuration ### Resource requests -Whilst the default requirements set within the pipeline will hopefully work for most people and with most input data, you may find that you want to customise the compute resources that the pipeline requests. Each step in the pipeline has a default set of requirements for number of CPUs, memory and time. For most of the steps in the pipeline, if the job exits with any of the error codes specified [here](https://github.com/nf-core/rnaseq/blob/4c27ef5610c87db00c3c5a3eed10b1d161abf575/conf/base.config#L18) it will automatically be resubmitted with higher requests (2 x original, then 3 x original). If it still fails after the third attempt then the pipeline execution is stopped. - -For example, if the nf-core/rnaseq pipeline is failing after multiple re-submissions of the `STAR_ALIGN` process due to an exit code of `137` this would indicate that there is an out of memory issue: - -```console -[62/149eb0] NOTE: Process `NFCORE_RNASEQ:RNASEQ:ALIGN_STAR:STAR_ALIGN (WT_REP1)` terminated with an error exit status (137) -- Execution is retried (1) -Error executing process > 'NFCORE_RNASEQ:RNASEQ:ALIGN_STAR:STAR_ALIGN (WT_REP1)' - -Caused by: - Process `NFCORE_RNASEQ:RNASEQ:ALIGN_STAR:STAR_ALIGN (WT_REP1)` terminated with an error exit status (137) - -Command executed: - STAR \ - --genomeDir star \ - --readFilesIn WT_REP1_trimmed.fq.gz \ - --runThreadN 2 \ - --outFileNamePrefix WT_REP1. \ - - -Command exit status: - 137 - -Command output: - (empty) - -Command error: - .command.sh: line 9: 30 Killed STAR --genomeDir star --readFilesIn WT_REP1_trimmed.fq.gz --runThreadN 2 --outFileNamePrefix WT_REP1. -Work dir: - /home/pipelinetest/work/9d/172ca5881234073e8d76f2a19c88fb - -Tip: you can replicate the issue by changing to the process work dir and entering the command `bash .command.run` -``` - -#### For beginners - -A first step to bypass this error, you could try to increase the amount of CPUs, memory, and time for the whole pipeline. Therefor you can try to increase the resource for the parameters `--max_cpus`, `--max_memory`, and `--max_time`. Based on the error above, you have to increase the amount of memory. Therefore you can go to the [parameter documentation of rnaseq](https://nf-co.re/rnaseq/3.9/parameters) and scroll down to the `show hidden parameter` button to get the default value for `--max_memory`. In this case 128GB, you than can try to run your pipeline again with `--max_memory 200GB -resume` to skip all process, that were already calculated. If you can not increase the resource of the complete pipeline, you can try to adapt the resource for a single process as mentioned below. - -#### Advanced option on process level - -To bypass this error you would need to find exactly which resources are set by the `STAR_ALIGN` process. The quickest way is to search for `process STAR_ALIGN` in the [nf-core/rnaseq Github repo](https://github.com/nf-core/rnaseq/search?q=process+STAR_ALIGN). -We have standardised the structure of Nextflow DSL2 pipelines such that all module files will be present in the `modules/` directory and so, based on the search results, the file we want is `modules/nf-core/star/align/main.nf`. -If you click on the link to that file you will notice that there is a `label` directive at the top of the module that is set to [`label process_high`](https://github.com/nf-core/rnaseq/blob/4c27ef5610c87db00c3c5a3eed10b1d161abf575/modules/nf-core/software/star/align/main.nf#L9). -The [Nextflow `label`](https://www.nextflow.io/docs/latest/process.html#label) directive allows us to organise workflow processes in separate groups which can be referenced in a configuration file to select and configure subset of processes having similar computing requirements. -The default values for the `process_high` label are set in the pipeline's [`base.config`](https://github.com/nf-core/rnaseq/blob/4c27ef5610c87db00c3c5a3eed10b1d161abf575/conf/base.config#L33-L37) which in this case is defined as 72GB. -Providing you haven't set any other standard nf-core parameters to **cap** the [maximum resources](https://nf-co.re/usage/configuration#max-resources) used by the pipeline then we can try and bypass the `STAR_ALIGN` process failure by creating a custom config file that sets at least 72GB of memory, in this case increased to 100GB. -The custom config below can then be provided to the pipeline via the [`-c`](#-c) parameter as highlighted in previous sections. - -```nextflow -process { - withName: 'NFCORE_RNASEQ:RNASEQ:ALIGN_STAR:STAR_ALIGN' { - memory = 100.GB - } -} -``` +Whilst the default requirements set within the pipeline will hopefully work for most people and with most input data, you may find that you want to customise the compute resources that the pipeline requests. +Each step in the pipeline has a default set of requirements for number of CPUs, memory and time. +For most of the steps in the pipeline, if the job exits with any of the error codes specified [here](https://github.com/nf-core/rnaseq/blob/4c27ef5610c87db00c3c5a3eed10b1d161abf575/conf/base.config#L18) it will automatically be resubmitted with higher requests (2 x original, then 3 x original). +If it still fails after the third attempt then the pipeline execution is stopped. -> **NB:** We specify the full process name i.e. `NFCORE_RNASEQ:RNASEQ:ALIGN_STAR:STAR_ALIGN` in the config file because this takes priority over the short name (`STAR_ALIGN`) and allows existing configuration using the full process name to be correctly overridden. -> -> If you get a warning suggesting that the process selector isn't recognised check that the process name has been specified correctly. +To change the resource requests, please see the [max resources](https://nf-co.re/docs/usage/configuration#max-resources) and [tuning workflow resources](https://nf-co.re/docs/usage/configuration#tuning-workflow-resources) section of the nf-core website. -### Updating containers (advanced users) +### Custom Containers -The [Nextflow DSL2](https://www.nextflow.io/docs/latest/dsl2.html) implementation of this pipeline uses one container per process which makes it much easier to maintain and update software dependencies. If for some reason you need to use a different version of a particular tool with the pipeline then you just need to identify the `process` name and override the Nextflow `container` definition for that process using the `withName` declaration. For example, in the [nf-core/viralrecon](https://nf-co.re/viralrecon) pipeline a tool called [Pangolin](https://github.com/cov-lineages/pangolin) has been used during the COVID-19 pandemic to assign lineages to SARS-CoV-2 genome sequenced samples. Given that the lineage assignments change quite frequently it doesn't make sense to re-release the nf-core/viralrecon everytime a new version of Pangolin has been released. However, you can override the default container used by the pipeline by creating a custom config file and passing it as a command-line argument via `-c custom.config`. +In some cases you may wish to change which container or conda environment a step of the pipeline uses for a particular tool. +By default nf-core pipelines use containers and software from the [biocontainers](https://biocontainers.pro/) or [bioconda](https://bioconda.github.io/) projects. +However in some cases the pipeline specified version maybe out of date. -1. Check the default version used by the pipeline in the module file for [Pangolin](https://github.com/nf-core/viralrecon/blob/a85d5969f9025409e3618d6c280ef15ce417df65/modules/nf-core/software/pangolin/main.nf#L14-L19) -2. Find the latest version of the Biocontainer available on [Quay.io](https://quay.io/repository/biocontainers/pangolin?tag=latest&tab=tags) -3. Create the custom config accordingly: +To use a different container from the default container or conda environment specified in a pipeline, please see the [updating tool versions](https://nf-co.re/docs/usage/configuration#updating-tool-versions) section of the nf-core website. - - For Docker: +### Custom Tool Arguments - ```nextflow - process { - withName: PANGOLIN { - container = 'quay.io/biocontainers/pangolin:3.0.5--pyhdfd78af_0' - } - } - ``` +A pipeline might not always support every possible argument or option of a particular tool used in pipeline. +Fortunately, nf-core pipelines provide some freedom to users to insert additional parameters that the pipeline does not include by default. - - For Singularity: +To learn how to provide additional arguments to a particular tool of the pipeline, please see the [customising tool arguments](https://nf-co.re/docs/usage/configuration#customising-tool-arguments) section of the nf-core website. - ```nextflow - process { - withName: PANGOLIN { - container = 'https://depot.galaxyproject.org/singularity/pangolin:3.0.5--pyhdfd78af_0' - } - } - ``` +### nf-core/configs - - For Conda: +In most cases, you will only need to create a custom config as a one-off but if you and others within your organisation are likely to be running nf-core pipelines regularly and need to use the same settings regularly it may be a good idea to request that your custom config file is uploaded to the `nf-core/configs` git repository. +Before you do this please can you test that the config file works with your pipeline of choice using the `-c` parameter. +You can then create a pull request to the `nf-core/configs` repository with the addition of your config file, associated documentation file (see examples in [`nf-core/configs/docs`](https://github.com/nf-core/configs/tree/master/docs)), and amending [`nfcore_custom.config`](https://github.com/nf-core/configs/blob/master/nfcore_custom.config) to include your custom profile. - ```nextflow - process { - withName: PANGOLIN { - conda = 'bioconda::pangolin=3.0.5' - } - } - ``` +See the main [Nextflow documentation](https://www.nextflow.io/docs/latest/config.html) for more information about creating your own configuration files. -> **NB:** If you wish to periodically update individual tool-specific results (e.g. Pangolin) generated by the pipeline then you must ensure to keep the `work/` directory otherwise the `-resume` ability of the pipeline will be compromised and it will restart from scratch. +If you have any questions or issues please send us a message on [Slack](https://nf-co.re/join/slack) on the [`#configs` channel](https://nfcore.slack.com/channels/configs). ## Running in the background -Nextflow handles job submissions and supervises the running jobs. The Nextflow process must run until the pipeline is finished. +Nextflow handles job submissions and supervises the running jobs. +The Nextflow process must run until the pipeline is finished. -The Nextflow `-bg` flag launches Nextflow in the background, detached from your terminal so that the workflow does not stop if you log out of your session. The logs are saved to a file. +The Nextflow `-bg` flag launches Nextflow in the background, detached from your terminal so that the workflow does not stop if you log out of your session. +The logs are saved to a file. Alternatively, you can use `screen` / `tmux` or similar tool to create a detached session which you can log back into at a later time. Some HPC setups also allow you to run nextflow within a cluster job submitted your job scheduler (from where it submits more jobs). diff --git a/lib/NfcoreTemplate.groovy b/lib/NfcoreTemplate.groovy deleted file mode 100755 index 01b8653..0000000 --- a/lib/NfcoreTemplate.groovy +++ /dev/null @@ -1,352 +0,0 @@ -// -// This file holds several functions used within the nf-core pipeline template. -// - -import org.yaml.snakeyaml.Yaml -import groovy.json.JsonOutput - -class NfcoreTemplate { - - // - // Check AWS Batch related parameters have been specified correctly - // - public static void awsBatch(workflow, params) { - if (workflow.profile.contains('awsbatch')) { - // Check params.awsqueue and params.awsregion have been set if running on AWSBatch - assert (params.awsqueue && params.awsregion) : "Specify correct --awsqueue and --awsregion parameters on AWSBatch!" - // Check outdir paths to be S3 buckets if running on AWSBatch - assert params.outdir.startsWith('s3:') : "Outdir not on S3 - specify S3 Bucket to run on AWSBatch!" - } - } - - // - // Warn if a -profile or Nextflow config has not been provided to run the pipeline - // - public static void checkConfigProvided(workflow, log) { - if (workflow.profile == 'standard' && workflow.configFiles.size() <= 1) { - log.warn "[$workflow.manifest.name] You are attempting to run the pipeline without any custom configuration!\n\n" + - "This will be dependent on your local compute environment but can be achieved via one or more of the following:\n" + - " (1) Using an existing pipeline profile e.g. `-profile docker` or `-profile singularity`\n" + - " (2) Using an existing nf-core/configs for your Institution e.g. `-profile crick` or `-profile uppmax`\n" + - " (3) Using your own local custom config e.g. `-c /path/to/your/custom.config`\n\n" + - "Please refer to the quick start section and usage docs for the pipeline.\n " - } - } - - // - // Generate version string - // - public static String version(workflow) { - String version_string = "" - - if (workflow.manifest.version) { - def prefix_v = workflow.manifest.version[0] != 'v' ? 'v' : '' - version_string += "${prefix_v}${workflow.manifest.version}" - } - - if (workflow.commitId) { - def git_shortsha = workflow.commitId.substring(0, 7) - version_string += "-g${git_shortsha}" - } - - return version_string - } - - // - // Construct and send completion email - // - public static void email(workflow, params, summary_params, projectDir, log, multiqc_report=[]) { - - // Set up the e-mail variables - def subject = "[$workflow.manifest.name] Successful: $workflow.runName" - if (!workflow.success) { - subject = "[$workflow.manifest.name] FAILED: $workflow.runName" - } - - def summary = [:] - for (group in summary_params.keySet()) { - summary << summary_params[group] - } - - def misc_fields = [:] - misc_fields['Date Started'] = workflow.start - misc_fields['Date Completed'] = workflow.complete - misc_fields['Pipeline script file path'] = workflow.scriptFile - misc_fields['Pipeline script hash ID'] = workflow.scriptId - if (workflow.repository) misc_fields['Pipeline repository Git URL'] = workflow.repository - if (workflow.commitId) misc_fields['Pipeline repository Git Commit'] = workflow.commitId - if (workflow.revision) misc_fields['Pipeline Git branch/tag'] = workflow.revision - misc_fields['Nextflow Version'] = workflow.nextflow.version - misc_fields['Nextflow Build'] = workflow.nextflow.build - misc_fields['Nextflow Compile Timestamp'] = workflow.nextflow.timestamp - - def email_fields = [:] - email_fields['version'] = NfcoreTemplate.version(workflow) - email_fields['runName'] = workflow.runName - email_fields['success'] = workflow.success - email_fields['dateComplete'] = workflow.complete - email_fields['duration'] = workflow.duration - email_fields['exitStatus'] = workflow.exitStatus - email_fields['errorMessage'] = (workflow.errorMessage ?: 'None') - email_fields['errorReport'] = (workflow.errorReport ?: 'None') - email_fields['commandLine'] = workflow.commandLine - email_fields['projectDir'] = workflow.projectDir - email_fields['summary'] = summary << misc_fields - - // On success try attach the multiqc report - def mqc_report = null - try { - if (workflow.success) { - mqc_report = multiqc_report.getVal() - if (mqc_report.getClass() == ArrayList && mqc_report.size() >= 1) { - if (mqc_report.size() > 1) { - log.warn "[$workflow.manifest.name] Found multiple reports from process 'MULTIQC', will use only one" - } - mqc_report = mqc_report[0] - } - } - } catch (all) { - if (multiqc_report) { - log.warn "[$workflow.manifest.name] Could not attach MultiQC report to summary email" - } - } - - // Check if we are only sending emails on failure - def email_address = params.email - if (!params.email && params.email_on_fail && !workflow.success) { - email_address = params.email_on_fail - } - - // Render the TXT template - def engine = new groovy.text.GStringTemplateEngine() - def tf = new File("$projectDir/assets/email_template.txt") - def txt_template = engine.createTemplate(tf).make(email_fields) - def email_txt = txt_template.toString() - - // Render the HTML template - def hf = new File("$projectDir/assets/email_template.html") - def html_template = engine.createTemplate(hf).make(email_fields) - def email_html = html_template.toString() - - // Render the sendmail template - def max_multiqc_email_size = (params.containsKey('max_multiqc_email_size') ? params.max_multiqc_email_size : 0) as nextflow.util.MemoryUnit - def smail_fields = [ email: email_address, subject: subject, email_txt: email_txt, email_html: email_html, projectDir: "$projectDir", mqcFile: mqc_report, mqcMaxSize: max_multiqc_email_size.toBytes() ] - def sf = new File("$projectDir/assets/sendmail_template.txt") - def sendmail_template = engine.createTemplate(sf).make(smail_fields) - def sendmail_html = sendmail_template.toString() - - // Send the HTML e-mail - Map colors = logColours(params.monochrome_logs) - if (email_address) { - try { - if (params.plaintext_email) { throw GroovyException('Send plaintext e-mail, not HTML') } - // Try to send HTML e-mail using sendmail - [ 'sendmail', '-t' ].execute() << sendmail_html - log.info "-${colors.purple}[$workflow.manifest.name]${colors.green} Sent summary e-mail to $email_address (sendmail)-" - } catch (all) { - // Catch failures and try with plaintext - def mail_cmd = [ 'mail', '-s', subject, '--content-type=text/html', email_address ] - if ( mqc_report.size() <= max_multiqc_email_size.toBytes() ) { - mail_cmd += [ '-A', mqc_report ] - } - mail_cmd.execute() << email_html - log.info "-${colors.purple}[$workflow.manifest.name]${colors.green} Sent summary e-mail to $email_address (mail)-" - } - } - - // Write summary e-mail HTML to a file - def output_d = new File("${params.outdir}/pipeline_info/") - if (!output_d.exists()) { - output_d.mkdirs() - } - def output_hf = new File(output_d, "pipeline_report.html") - output_hf.withWriter { w -> w << email_html } - def output_tf = new File(output_d, "pipeline_report.txt") - output_tf.withWriter { w -> w << email_txt } - } - - // - // Construct and send a notification to a web server as JSON - // e.g. Microsoft Teams and Slack - // - public static void IM_notification(workflow, params, summary_params, projectDir, log) { - def hook_url = params.hook_url - - def summary = [:] - for (group in summary_params.keySet()) { - summary << summary_params[group] - } - - def misc_fields = [:] - misc_fields['start'] = workflow.start - misc_fields['complete'] = workflow.complete - misc_fields['scriptfile'] = workflow.scriptFile - misc_fields['scriptid'] = workflow.scriptId - if (workflow.repository) misc_fields['repository'] = workflow.repository - if (workflow.commitId) misc_fields['commitid'] = workflow.commitId - if (workflow.revision) misc_fields['revision'] = workflow.revision - misc_fields['nxf_version'] = workflow.nextflow.version - misc_fields['nxf_build'] = workflow.nextflow.build - misc_fields['nxf_timestamp'] = workflow.nextflow.timestamp - - def msg_fields = [:] - msg_fields['version'] = NfcoreTemplate.version(workflow) - msg_fields['runName'] = workflow.runName - msg_fields['success'] = workflow.success - msg_fields['dateComplete'] = workflow.complete - msg_fields['duration'] = workflow.duration - msg_fields['exitStatus'] = workflow.exitStatus - msg_fields['errorMessage'] = (workflow.errorMessage ?: 'None') - msg_fields['errorReport'] = (workflow.errorReport ?: 'None') - msg_fields['commandLine'] = workflow.commandLine.replaceFirst(/ +--hook_url +[^ ]+/, "") - msg_fields['projectDir'] = workflow.projectDir - msg_fields['summary'] = summary << misc_fields - - // Render the JSON template - def engine = new groovy.text.GStringTemplateEngine() - // Different JSON depending on the service provider - // Defaults to "Adaptive Cards" (https://adaptivecards.io), except Slack which has its own format - def json_path = hook_url.contains("hooks.slack.com") ? "slackreport.json" : "adaptivecard.json" - def hf = new File("$projectDir/assets/${json_path}") - def json_template = engine.createTemplate(hf).make(msg_fields) - def json_message = json_template.toString() - - // POST - def post = new URL(hook_url).openConnection(); - post.setRequestMethod("POST") - post.setDoOutput(true) - post.setRequestProperty("Content-Type", "application/json") - post.getOutputStream().write(json_message.getBytes("UTF-8")); - def postRC = post.getResponseCode(); - if (! postRC.equals(200)) { - log.warn(post.getErrorStream().getText()); - } - } - - // - // Dump pipeline parameters in a json file - // - public static void dump_parameters(workflow, params) { - def output_d = new File("${params.outdir}/pipeline_info/") - if (!output_d.exists()) { - output_d.mkdirs() - } - - def timestamp = new java.util.Date().format( 'yyyy-MM-dd_HH-mm-ss') - def output_pf = new File(output_d, "params_${timestamp}.json") - def jsonStr = JsonOutput.toJson(params) - output_pf.text = JsonOutput.prettyPrint(jsonStr) - } - - // - // Print pipeline summary on completion - // - public static void summary(workflow, params, log) { - Map colors = logColours(params.monochrome_logs) - if (workflow.success) { - if (workflow.stats.ignoredCount == 0) { - log.info "-${colors.purple}[$workflow.manifest.name]${colors.green} Pipeline completed successfully${colors.reset}-" - } else { - log.info "-${colors.purple}[$workflow.manifest.name]${colors.yellow} Pipeline completed successfully, but with errored process(es) ${colors.reset}-" - } - } else { - log.info "-${colors.purple}[$workflow.manifest.name]${colors.red} Pipeline completed with errors${colors.reset}-" - } - } - - // - // ANSII Colours used for terminal logging - // - public static Map logColours(Boolean monochrome_logs) { - Map colorcodes = [:] - - // Reset / Meta - colorcodes['reset'] = monochrome_logs ? '' : "\033[0m" - colorcodes['bold'] = monochrome_logs ? '' : "\033[1m" - colorcodes['dim'] = monochrome_logs ? '' : "\033[2m" - colorcodes['underlined'] = monochrome_logs ? '' : "\033[4m" - colorcodes['blink'] = monochrome_logs ? '' : "\033[5m" - colorcodes['reverse'] = monochrome_logs ? '' : "\033[7m" - colorcodes['hidden'] = monochrome_logs ? '' : "\033[8m" - - // Regular Colors - colorcodes['black'] = monochrome_logs ? '' : "\033[0;30m" - colorcodes['red'] = monochrome_logs ? '' : "\033[0;31m" - colorcodes['green'] = monochrome_logs ? '' : "\033[0;32m" - colorcodes['yellow'] = monochrome_logs ? '' : "\033[0;33m" - colorcodes['blue'] = monochrome_logs ? '' : "\033[0;34m" - colorcodes['purple'] = monochrome_logs ? '' : "\033[0;35m" - colorcodes['cyan'] = monochrome_logs ? '' : "\033[0;36m" - colorcodes['white'] = monochrome_logs ? '' : "\033[0;37m" - - // Bold - colorcodes['bblack'] = monochrome_logs ? '' : "\033[1;30m" - colorcodes['bred'] = monochrome_logs ? '' : "\033[1;31m" - colorcodes['bgreen'] = monochrome_logs ? '' : "\033[1;32m" - colorcodes['byellow'] = monochrome_logs ? '' : "\033[1;33m" - colorcodes['bblue'] = monochrome_logs ? '' : "\033[1;34m" - colorcodes['bpurple'] = monochrome_logs ? '' : "\033[1;35m" - colorcodes['bcyan'] = monochrome_logs ? '' : "\033[1;36m" - colorcodes['bwhite'] = monochrome_logs ? '' : "\033[1;37m" - - // Underline - colorcodes['ublack'] = monochrome_logs ? '' : "\033[4;30m" - colorcodes['ured'] = monochrome_logs ? '' : "\033[4;31m" - colorcodes['ugreen'] = monochrome_logs ? '' : "\033[4;32m" - colorcodes['uyellow'] = monochrome_logs ? '' : "\033[4;33m" - colorcodes['ublue'] = monochrome_logs ? '' : "\033[4;34m" - colorcodes['upurple'] = monochrome_logs ? '' : "\033[4;35m" - colorcodes['ucyan'] = monochrome_logs ? '' : "\033[4;36m" - colorcodes['uwhite'] = monochrome_logs ? '' : "\033[4;37m" - - // High Intensity - colorcodes['iblack'] = monochrome_logs ? '' : "\033[0;90m" - colorcodes['ired'] = monochrome_logs ? '' : "\033[0;91m" - colorcodes['igreen'] = monochrome_logs ? '' : "\033[0;92m" - colorcodes['iyellow'] = monochrome_logs ? '' : "\033[0;93m" - colorcodes['iblue'] = monochrome_logs ? '' : "\033[0;94m" - colorcodes['ipurple'] = monochrome_logs ? '' : "\033[0;95m" - colorcodes['icyan'] = monochrome_logs ? '' : "\033[0;96m" - colorcodes['iwhite'] = monochrome_logs ? '' : "\033[0;97m" - - // Bold High Intensity - colorcodes['biblack'] = monochrome_logs ? '' : "\033[1;90m" - colorcodes['bired'] = monochrome_logs ? '' : "\033[1;91m" - colorcodes['bigreen'] = monochrome_logs ? '' : "\033[1;92m" - colorcodes['biyellow'] = monochrome_logs ? '' : "\033[1;93m" - colorcodes['biblue'] = monochrome_logs ? '' : "\033[1;94m" - colorcodes['bipurple'] = monochrome_logs ? '' : "\033[1;95m" - colorcodes['bicyan'] = monochrome_logs ? '' : "\033[1;96m" - colorcodes['biwhite'] = monochrome_logs ? '' : "\033[1;97m" - - return colorcodes - } - - // - // Does what is says on the tin - // - public static String dashedLine(monochrome_logs) { - Map colors = logColours(monochrome_logs) - return "-${colors.dim}----------------------------------------------------${colors.reset}-" - } - - // - // nf-core logo - // - public static String logo(workflow, monochrome_logs) { - Map colors = logColours(monochrome_logs) - String workflow_version = NfcoreTemplate.version(workflow) - String.format( - """\n - ${dashedLine(monochrome_logs)} - ${colors.green},--.${colors.black}/${colors.green},-.${colors.reset} - ${colors.blue} ___ __ __ __ ___ ${colors.green}/,-._.--~\'${colors.reset} - ${colors.blue} |\\ | |__ __ / ` / \\ |__) |__ ${colors.yellow}} {${colors.reset} - ${colors.blue} | \\| | \\__, \\__/ | \\ |___ ${colors.green}\\`-._,-`-,${colors.reset} - ${colors.green}`._,._,\'${colors.reset} - ${colors.purple} ${workflow.manifest.name} ${workflow_version}${colors.reset} - ${dashedLine(monochrome_logs)} - """.stripIndent() - ) - } -} diff --git a/lib/Utils.groovy b/lib/Utils.groovy deleted file mode 100644 index 8d030f4..0000000 --- a/lib/Utils.groovy +++ /dev/null @@ -1,47 +0,0 @@ -// -// This file holds several Groovy functions that could be useful for any Nextflow pipeline -// - -import org.yaml.snakeyaml.Yaml - -class Utils { - - // - // When running with -profile conda, warn if channels have not been set-up appropriately - // - public static void checkCondaChannels(log) { - Yaml parser = new Yaml() - def channels = [] - try { - def config = parser.load("conda config --show channels".execute().text) - channels = config.channels - } catch(NullPointerException | IOException e) { - log.warn "Could not verify conda channel configuration." - return - } - - // Check that all channels are present - // This channel list is ordered by required channel priority. - def required_channels_in_order = ['conda-forge', 'bioconda', 'defaults'] - def channels_missing = ((required_channels_in_order as Set) - (channels as Set)) as Boolean - - // Check that they are in the right order - def channel_priority_violation = false - def n = required_channels_in_order.size() - for (int i = 0; i < n - 1; i++) { - channel_priority_violation |= !(channels.indexOf(required_channels_in_order[i]) < channels.indexOf(required_channels_in_order[i+1])) - } - - if (channels_missing | channel_priority_violation) { - log.warn "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n" + - " There is a problem with your Conda configuration!\n\n" + - " You will need to set-up the conda-forge and bioconda channels correctly.\n" + - " Please refer to https://bioconda.github.io/\n" + - " The observed channel order is \n" + - " ${channels}\n" + - " but the following channel order is required:\n" + - " ${required_channels_in_order}\n" + - "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - } - } -} diff --git a/lib/WorkflowMain.groovy b/lib/WorkflowMain.groovy deleted file mode 100755 index 3181574..0000000 --- a/lib/WorkflowMain.groovy +++ /dev/null @@ -1,52 +0,0 @@ -// -// This file holds several functions specific to the main.nf workflow in the nf-core/rangeland pipeline -// - -import nextflow.Nextflow - -class WorkflowMain { - - // - // Citation string for pipeline - // - public static String citation(workflow) { - return "If you use ${workflow.manifest.name} for your analysis please cite:\n\n" + - // TODO nf-core: Add Zenodo DOI for pipeline after first release - //"* The pipeline\n" + - //" https://doi.org/10.5281/zenodo.XXXXXXX\n\n" + - "* The nf-core framework\n" + - " https://doi.org/10.1038/s41587-020-0439-x\n\n" + - "* Software dependencies\n" + - " https://github.com/${workflow.manifest.name}/blob/master/CITATIONS.md" - } - - - // - // Validate parameters and print summary to screen - // - public static void initialise(workflow, params, log) { - - // Print workflow version and exit on --version - if (params.version) { - String workflow_version = NfcoreTemplate.version(workflow) - log.info "${workflow.manifest.name} ${workflow_version}" - System.exit(0) - } - - // Check that a -profile or Nextflow config has been provided to run the pipeline - NfcoreTemplate.checkConfigProvided(workflow, log) - - // Check that conda channels are set-up correctly - if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { - Utils.checkCondaChannels(log) - } - - // Check AWS batch settings - NfcoreTemplate.awsBatch(workflow, params) - - // Check input has been provided - if (!params.input) { - Nextflow.error("Please provide an input samplesheet to the pipeline e.g. '--input samplesheet.csv'") - } - } -} diff --git a/lib/WorkflowRangeland.groovy b/lib/WorkflowRangeland.groovy deleted file mode 100755 index dfa757d..0000000 --- a/lib/WorkflowRangeland.groovy +++ /dev/null @@ -1,103 +0,0 @@ -// -// This file holds several functions specific to the workflow/rangeland.nf in the nf-core/rangeland pipeline -// - -import nextflow.Nextflow -import groovy.text.SimpleTemplateEngine - -class WorkflowRangeland { - - // - // Check and validate parameters - // - public static void initialise(params, log) { - // Check mandatory parameters - if (!params.input) { Nextflow.error "Input satellite data not specified with e.g. --input or via detectable config file." } - if (!params.dem) { Nextflow.error "Input digital elevation model not specified with e.g. --dem or via detectable config file." } - if (!params.wvdb) { Nextflow.error "Input water vapor data not specified with e.g. --wvdb or via detectable config file." } - if (!params.data_cube) { Nextflow.error "Input datacube definition not specified with e.g. --data_cube datacube.gpkg or via detectable config file." } - if (!params.aoi) { Nextflow.error "Input area-of-interest specification not specified with e.g. --aoi aoi.gpkg or via detectable config file." } - if (!params.endmember) { Nextflow.error "Input endmember specification not specified with e.g. --endmember endmember.txt or via detectable config file." } - } - - // - // Get workflow summary for MultiQC - // - public static String paramsSummaryMultiqc(workflow, summary) { - String summary_section = '' - for (group in summary.keySet()) { - def group_params = summary.get(group) // This gets the parameters of that particular group - if (group_params) { - summary_section += "

$group

\n" - summary_section += "
\n" - for (param in group_params.keySet()) { - summary_section += "
$param
${group_params.get(param) ?: 'N/A'}
\n" - } - summary_section += "
\n" - } - } - - String yaml_file_text = "id: '${workflow.manifest.name.replace('/','-')}-summary'\n" - yaml_file_text += "description: ' - this information is collected when the pipeline is started.'\n" - yaml_file_text += "section_name: '${workflow.manifest.name} Workflow Summary'\n" - yaml_file_text += "section_href: 'https://github.com/${workflow.manifest.name}'\n" - yaml_file_text += "plot_type: 'html'\n" - yaml_file_text += "data: |\n" - yaml_file_text += "${summary_section}" - return yaml_file_text - } - - // - // Generate methods description for MultiQC - // - - public static String toolCitationText(params) { - - // Uncomment function in methodsDescriptionText to render in MultiQC report - def citation_text = [ - "Tools used in the workflow included:", - "MultiQC (Ewels et al. 2016)", - "FORCE (Frantz et al. 2019)", - "." - ].join(' ').trim() - - return citation_text - } - - public static String toolBibliographyText(params) { - - // Uncomment function in methodsDescriptionText to render in MultiQC report - def reference_text = [ - "
  • Ewels, P., Magnusson, M., Lundin, S., & Käller, M. (2016). MultiQC: summarize analysis results for multiple tools and samples in a single report. Bioinformatics , 32(19), 3047–3048. doi: /10.1093/bioinformatics/btw354
    • ", - "
  • Frantz, D. (2019). FORCE—Landsat + Sentinel-2 Analysis Ready Data and Beyond. Remote Sensing, 11, 1124
    • " - ].join(' ').trim() - - return reference_text - } - - public static String methodsDescriptionText(run_workflow, mqc_methods_yaml, params) { - // Convert to a named map so can be used as with familar NXF ${workflow} variable syntax in the MultiQC YML file - def meta = [:] - meta.workflow = run_workflow.toMap() - meta["manifest_map"] = run_workflow.manifest.toMap() - - // Pipeline DOI - meta["doi_text"] = meta.manifest_map.doi ? "(doi: ${meta.manifest_map.doi})" : "" - meta["nodoi_text"] = meta.manifest_map.doi ? "": "
  • If available, make sure to update the text to include the Zenodo DOI of version of the pipeline used.
    • " - - // Tool references - meta["tool_citations"] = "" - meta["tool_bibliography"] = "" - - // Only uncomment below if logic in toolCitationText/toolBibliographyText has been filled! - meta["tool_citations"] = toolCitationText(params).replaceAll(", \\.", ".").replaceAll("\\. \\.", ".").replaceAll(", \\.", ".") - meta["tool_bibliography"] = toolBibliographyText(params) - - - def methods_text = mqc_methods_yaml.text - - def engine = new SimpleTemplateEngine() - def description_html = engine.createTemplate(methods_text).make(meta) - - return description_html - }} diff --git a/lib/nfcore_external_java_deps.jar b/lib/nfcore_external_java_deps.jar deleted file mode 100644 index 805c8bb..0000000 Binary files a/lib/nfcore_external_java_deps.jar and /dev/null differ diff --git a/main.nf b/main.nf index dd39611..69bb9ce 100644 --- a/main.nf +++ b/main.nf @@ -9,59 +9,78 @@ ---------------------------------------------------------------------------------------- */ -nextflow.enable.dsl = 2 - /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - VALIDATE & PRINT PARAMETER SUMMARY + IMPORT FUNCTIONS / MODULES / SUBWORKFLOWS / WORKFLOWS ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ -include { validateParameters; paramsHelp } from 'plugin/nf-validation' - -// Print help message if needed -if (params.help) { - def logo = NfcoreTemplate.logo(workflow, params.monochrome_logs) - def citation = '\n' + WorkflowMain.citation(workflow) + '\n' - def String command = "nextflow run ${workflow.manifest.name} --input --dem --wvdb --data_cube --aoi --endmember --outdir -profile docker" - log.info logo + paramsHelp(command) + citation + NfcoreTemplate.dashedLine(params.monochrome_logs) - System.exit(0) -} - -// Validate input parameters -if (params.validate_params) { - validateParameters() -} - -WorkflowMain.initialise(workflow, params, log) - +include { RANGELAND } from './workflows/rangeland' +include { PIPELINE_INITIALISATION } from './subworkflows/local/utils_nfcore_rangeland_pipeline' +include { PIPELINE_COMPLETION } from './subworkflows/local/utils_nfcore_rangeland_pipeline' /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - NAMED WORKFLOW FOR PIPELINE + NAMED WORKFLOWS FOR PIPELINE ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ -include { RANGELAND } from './workflows/rangeland' - // -// WORKFLOW: Run main nf-core/rangeland analysis pipeline +// WORKFLOW: Run main analysis pipeline depending on type of input // workflow NFCORE_RANGELAND { + + main: + + // + // WORKFLOW: Run pipeline + // RANGELAND () -} + emit: + level2_ard = RANGELAND.out.level2_ard + mosaic = RANGELAND.out.mosaic + pyramid = RANGELAND.out.pyramid + trends = RANGELAND.out.trends + multiqc_report = RANGELAND.out.multiqc_report // channel: /path/to/multiqc_report.html +} /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - RUN ALL WORKFLOWS + RUN MAIN WORKFLOW ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ -// -// WORKFLOW: Execute a single named workflow for the pipeline -// See: https://github.com/nf-core/rnaseq/issues/619 -// workflow { + + main: + // + // SUBWORKFLOW: Run initialisation tasks + // + PIPELINE_INITIALISATION ( + params.version, + params.validate_params, + params.monochrome_logs, + args, + params.outdir, + params.input + ) + + // + // WORKFLOW: Run main workflow + // NFCORE_RANGELAND () + + // + // SUBWORKFLOW: Run completion tasks + // + PIPELINE_COMPLETION ( + params.email, + params.email_on_fail, + params.plaintext_email, + params.outdir, + params.monochrome_logs, + params.hook_url, + NFCORE_RANGELAND.out.multiqc_report + ) } /* diff --git a/modules.json b/modules.json index 58ff5cc..5281958 100644 --- a/modules.json +++ b/modules.json @@ -5,22 +5,36 @@ "https://github.com/nf-core/modules.git": { "modules": { "nf-core": { - "custom/dumpsoftwareversions": { - "branch": "master", - "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", - "installed_by": ["modules"] - }, "multiqc": { "branch": "master", - "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", + "git_sha": "cf17ca47590cc578dfb47db1c2a44ef86f89976d", "installed_by": ["modules"] }, "untar": { "branch": "master", - "git_sha": "cfd937a668919d948f6fcbf4218e79de50c2f36f", + "git_sha": "666652151335353eef2fcd58880bcef5bc2928e1", "installed_by": ["modules"] } } + }, + "subworkflows": { + "nf-core": { + "utils_nextflow_pipeline": { + "branch": "master", + "git_sha": "c2b22d85f30a706a3073387f30380704fcae013b", + "installed_by": ["subworkflows"] + }, + "utils_nfcore_pipeline": { + "branch": "master", + "git_sha": "1b89f75f1aa2021ec3360d0deccd0f6e97240551", + "installed_by": ["subworkflows"] + }, + "utils_nfschema_plugin": { + "branch": "master", + "git_sha": "2fd2cd6d0e7b273747f32e465fdc6bcc3ae0814e", + "installed_by": ["subworkflows"] + } + } } } } diff --git a/modules/local/check_results.nf b/modules/local/check_results/main.nf similarity index 97% rename from modules/local/check_results.nf rename to modules/local/check_results/main.nf index cb57366..def8337 100644 --- a/modules/local/check_results.nf +++ b/modules/local/check_results/main.nf @@ -1,7 +1,5 @@ -nextflow.enable.dsl = 2 - process CHECK_RESULTS { - + tag 'check' label 'process_low' container 'docker.io/rocker/geospatial:4.3.1' diff --git a/modules/local/check_results_full/main.nf b/modules/local/check_results_full/main.nf new file mode 100644 index 0000000..f1f094b --- /dev/null +++ b/modules/local/check_results_full/main.nf @@ -0,0 +1,34 @@ +process CHECK_RESULTS_FULL { + tag 'check' + label 'process_low' + + container 'docker.io/rocker/geospatial:4.3.1' + + input: + path{ "trend/?/*" } + path reference + + output: + path "versions.yml" , emit: versions + + when: + task.ext.when == null || task.ext.when + + script: + """ + files=`find ./trend/ -maxdepth 1 -mindepth 1 -type d` + for path in \$files; do + mkdir -p trend/\$(ls \$path) + cp \$path/*/* trend/\$(ls \$path)/ + rm \$path -r + done; + test.R trend/mosaic $reference + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + r-base: \$(echo \$(R --version 2>&1) | sed 's/^.*R version //; s/ .*\$//') + terra: \$(Rscript -e "library(terra); cat(as.character(packageVersion('terra')))") + END_VERSIONS + """ + +} diff --git a/modules/local/force-generate_analysis_mask.nf b/modules/local/force-generate_analysis_mask/main.nf similarity index 76% rename from modules/local/force-generate_analysis_mask.nf rename to modules/local/force-generate_analysis_mask/main.nf index ba8ce54..fdaa5a9 100644 --- a/modules/local/force-generate_analysis_mask.nf +++ b/modules/local/force-generate_analysis_mask/main.nf @@ -1,14 +1,13 @@ -nextflow.enable.dsl = 2 - process FORCE_GENERATE_ANALYSIS_MASK{ - + tag { aoi.simpleName } label 'process_single' - container "docker.io/davidfrantz/force:3.7.11" + container "docker.io/davidfrantz/force:3.7.10" input: path aoi path 'mask/datacube-definition.prj' + val resolution output: //Mask for whole region @@ -20,7 +19,7 @@ process FORCE_GENERATE_ANALYSIS_MASK{ script: """ - force-cube -o mask/ -s $params.resolution $aoi + force-cube -o mask/ -s $resolution $aoi cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/local/force-generate_tile_allow_list.nf b/modules/local/force-generate_tile_allow_list/main.nf similarity index 87% rename from modules/local/force-generate_tile_allow_list.nf rename to modules/local/force-generate_tile_allow_list/main.nf index 2a7bb70..6125e47 100644 --- a/modules/local/force-generate_tile_allow_list.nf +++ b/modules/local/force-generate_tile_allow_list/main.nf @@ -1,10 +1,8 @@ -nextflow.enable.dsl = 2 - process FORCE_GENERATE_TILE_ALLOW_LIST{ - + tag { aoi.simpleName } label 'process_single' - container "docker.io/davidfrantz/force:3.7.11" + container "docker.io/davidfrantz/force:3.7.10" input: path aoi diff --git a/modules/local/force-higher_level.nf b/modules/local/force-higher_level/main.nf similarity index 60% rename from modules/local/force-higher_level.nf rename to modules/local/force-higher_level/main.nf index 8a2a644..c042cb1 100644 --- a/modules/local/force-higher_level.nf +++ b/modules/local/force-higher_level/main.nf @@ -1,17 +1,15 @@ -nextflow.enable.dsl = 2 - process FORCE_HIGHER_LEVEL { - - label 'process_high' - - container "docker.io/davidfrantz/force:3.7.11" tag { tile } + label 'process_medium' + label 'error_retry' + + container "docker.io/davidfrantz/force:3.7.10" input: tuple val(tile), path(config), path(ard), path(aoi), path (datacube), path (endmember) output: - path 'trend/*.tif*', emit: trend_files + path 'trend/*.tif*', optional:true, emit: trend_files path "versions.yml", emit: versions when: @@ -23,18 +21,16 @@ process FORCE_HIGHER_LEVEL { mkdir trend - # set provenance + # set provenance directory mkdir prov sed -i "/^DIR_PROVENANCE /c\\DIR_PROVENANCE = prov/" \$PARAM - + # higher level processing force-higher-level \$PARAM - #Rename files: /trend// to _, otherwise we can not reextract the tile name later + # Rename files: /trend// to _, otherwise we can not reextract the tile name later results=`find trend -name '*.tif*'` - for path in \$results; do - mv \$path \${path%/*}_\${path##*/} - done; + parallel -j $task.cpus 'mv {} {//}_{/}' ::: \$results cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/local/force-mosaic.nf b/modules/local/force-mosaic/main.nf similarity index 70% rename from modules/local/force-mosaic.nf rename to modules/local/force-mosaic/main.nf index 519a5d0..a114805 100644 --- a/modules/local/force-mosaic.nf +++ b/modules/local/force-mosaic/main.nf @@ -1,11 +1,8 @@ -nextflow.enable.dsl = 2 - process FORCE_MOSAIC{ - + tag { product } label 'process_low' - tag { product } - container "docker.io/davidfrantz/force:3.7.11" + container "docker.io/davidfrantz/force:3.7.10" input: tuple val(product), path('trend/*') @@ -20,13 +17,18 @@ process FORCE_MOSAIC{ script: """ - #Move files from trend/_ to trend// - results=`find trend/*.tif*` - for path in \$results; do + move_file() { + path=\$1 mkdir -p \${path%_$product*} mv \$path \${path%_$product*}/${product}.\${path#*.} - done; + } + export -f move_file + + # Move files from trend/_ to trend// + results=`find trend/*.tif*` + parallel -j $task.cpus move_file ::: \$results + # start mosaic computation force-mosaic trend/ cat <<-END_VERSIONS > versions.yml diff --git a/modules/local/force-preprocess.nf b/modules/local/force-preprocess/main.nf similarity index 85% rename from modules/local/force-preprocess.nf rename to modules/local/force-preprocess/main.nf index 999d143..e694b6c 100644 --- a/modules/local/force-preprocess.nf +++ b/modules/local/force-preprocess/main.nf @@ -1,11 +1,9 @@ -nextflow.enable.dsl=2 - process FORCE_PREPROCESS { - - label 'process_medium' tag { data.simpleName } + label 'process_medium' + label 'error_retry' - container "docker.io/davidfrantz/force:3.7.11" + container "docker.io/davidfrantz/force:3.7.10" input: tuple path(conf), path(data), path(cube), path(tile), path(dem), path(wvdb) @@ -13,7 +11,7 @@ process FORCE_PREPROCESS { output: path "**/*BOA.tif", optional:true, emit: boa_tiles path "**/*QAI.tif", optional:true, emit: qai_tiles - path "*.log" , emit: log + path "**.log" , emit: log path "versions.yml" , emit: versions when: @@ -38,7 +36,7 @@ process FORCE_PREPROCESS { FILEPATH=$data BASE=\$(basename $data) - force-l2ps \$FILEPATH \$PARAM > level2_log\$BASE.log + force-l2ps \$FILEPATH \$PARAM > level2_log/\$BASE.log cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/local/force-pyramid.nf b/modules/local/force-pyramid/main.nf similarity index 71% rename from modules/local/force-pyramid.nf rename to modules/local/force-pyramid/main.nf index 98f6d5e..34bac8b 100644 --- a/modules/local/force-pyramid.nf +++ b/modules/local/force-pyramid/main.nf @@ -1,11 +1,8 @@ -nextflow.enable.dsl = 2 - process FORCE_PYRAMID { - - label 'process_low' tag { tile } + label 'process_low' - container "docker.io/davidfrantz/force:3.7.11" + container "docker.io/davidfrantz/force:3.7.10" input: tuple val(tile), path(image) @@ -19,10 +16,8 @@ process FORCE_PYRAMID { script: """ - files="*.tif" - for file in \$files; do - force-pyramid \$file - done; + file="*.tif" + force-pyramid \$file cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/local/higher_level_force_config.nf b/modules/local/higher_level_force_config/main.nf similarity index 68% rename from modules/local/higher_level_force_config.nf rename to modules/local/higher_level_force_config/main.nf index 5e9eac7..dbcc9fe 100644 --- a/modules/local/higher_level_force_config.nf +++ b/modules/local/higher_level_force_config/main.nf @@ -1,16 +1,20 @@ -nextflow.enable.dsl = 2 - process HIGHER_LEVEL_CONFIG { - - label 'process_single' tag { tile } + label 'process_single' + label 'error_retry' - container "docker.io/davidfrantz/force:3.7.11" + container "docker.io/davidfrantz/force:3.7.10" input: tuple val(tile), path("ard/${tile}/*"), path("ard/${tile}/*"), path("mask/${tile}/aoi.tif") path 'ard/datacube-definition.prj' path endmember + val resolution + val sensors_level2 + val start_date + val end_date + val indexes + val return_tss output: tuple val (tile), path("trend_${tile}.prm"), path("ard/", includeInputs: true), path("mask/", includeInputs: true), path('ard/datacube-definition.prj', includeInputs: true), path(endmember, includeInputs: true), emit: higher_level_configs_and_data @@ -27,19 +31,13 @@ process HIGHER_LEVEL_CONFIG { # set parameters - #Replace pathes + # Replace paths sed -i "/^DIR_LOWER /c\\DIR_LOWER = ard/" \$PARAM sed -i "/^DIR_HIGHER /c\\DIR_HIGHER = trend/" \$PARAM sed -i "/^DIR_MASK /c\\DIR_MASK = mask/" \$PARAM sed -i "/^BASE_MASK /c\\BASE_MASK = aoi.tif" \$PARAM sed -i "/^FILE_ENDMEM /c\\FILE_ENDMEM = $endmember" \$PARAM - # threading - sed -i "/^NTHREAD_READ /c\\NTHREAD_READ = 1" \$PARAM # might need some modification - sed -i "/^NTHREAD_COMPUTE /c\\NTHREAD_COMPUTE = $params.force_threads" \$PARAM # might need some modification - sed -i "/^NTHREAD_WRITE /c\\NTHREAD_WRITE = 1" \$PARAM # might need some modification - - # replace Tile to process TILE="$tile" X=\${TILE:1:4} @@ -48,20 +46,20 @@ process HIGHER_LEVEL_CONFIG { sed -i "/^Y_TILE_RANGE /c\\Y_TILE_RANGE = \$Y \$Y" \$PARAM # resolution - sed -i "/^RESOLUTION /c\\RESOLUTION = $params.resolution" \$PARAM + sed -i "/^RESOLUTION /c\\RESOLUTION = $resolution" \$PARAM # sensors - sed -i "/^SENSORS /c\\SENSORS = $params.sensors_level2" \$PARAM + sed -i "/^SENSORS /c\\SENSORS = $sensors_level2" \$PARAM # date range - sed -i "/^DATE_RANGE /c\\DATE_RANGE = $params.start_date $params.end_date" \$PARAM + sed -i "/^DATE_RANGE /c\\DATE_RANGE = $start_date $end_date" \$PARAM # spectral index - sed -i "/^INDEX /c\\INDEX = SMA${params.only_tile ? ' NDVI BLUE GREEN RED NIR SWIR1 SWIR2' : ''}" \$PARAM - ${ params.only_tile ? 'sed -i "/^OUTPUT_TSS /c\\OUTPUT_TSS = TRUE" \$PARAM' : ''} + sed -i "/^INDEX /c\\INDEX = SMA $indexes" \$PARAM + ${ return_tss ? 'sed -i "/^OUTPUT_TSS /c\\OUTPUT_TSS = TRUE" \$PARAM' : '' } # interpolation sed -i "/^INT_DAY /c\\INT_DAY = 8" \$PARAM diff --git a/modules/local/merge.nf b/modules/local/merge/main.nf similarity index 74% rename from modules/local/merge.nf rename to modules/local/merge/main.nf index fb7a1b7..f849269 100644 --- a/modules/local/merge.nf +++ b/modules/local/merge/main.nf @@ -1,11 +1,9 @@ -nextflow.enable.dsl = 2 - process MERGE { - - label 'process_low' tag { id } + label 'process_low' + label 'error_retry' - container 'docker.io/davidfrantz/force:dev' + container "docker.io/davidfrantz/force:3.7.10" input: val (data_type) // defines whether qai or boa is merged @@ -21,19 +19,19 @@ process MERGE { script: """ + # get files to merge files=`find -L input/ -type f -printf "%f\\n" | sort | uniq` numberFiles=`echo \$files | wc -w` - currentFile=0 - for file in \$files - do - currentFile=\$((currentFile+1)) - echo "Merging \$file (\$currentFile of \$numberFiles)" + # merge function + merge() { + file=\$1 + echo "Merging \$file (\$2 of \$numberFiles)" onefile=`ls -- */*/\${file} | head -1` - - #merge together matchingFiles=`ls -- */*/\${file}` + + # merge script execution depending on file type if [ "$data_type" = "boa" ]; then merge_boa.r \$file \${matchingFiles} elif [ "$data_type" = "qai" ]; then @@ -42,8 +40,12 @@ process MERGE { #apply meta force-mdcp \$onefile \$file + } + export -f merge + export numberFiles - done; + # start merging in parallel + parallel -j $task.cpus merge {} {#} ::: \$files cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/local/preprocess_force_config.nf b/modules/local/preprocess_force_config/main.nf similarity index 91% rename from modules/local/preprocess_force_config.nf rename to modules/local/preprocess_force_config/main.nf index 2830ea7..ff30e9c 100644 --- a/modules/local/preprocess_force_config.nf +++ b/modules/local/preprocess_force_config/main.nf @@ -1,11 +1,9 @@ -nextflow.enable.dsl = 2 - process PREPROCESS_CONFIG { - - label 'process_single' tag { data.simpleName } + label 'process_single' + label 'error_retry' - container "docker.io/davidfrantz/force:3.7.11" + container "docker.io/davidfrantz/force:3.7.10" input: path data @@ -49,7 +47,6 @@ process PREPROCESS_CONFIG { sed -i "/^ORIGIN_LON /c\\ORIGIN_LON = \$ORIGINX" \$PARAM sed -i "/^ORIGIN_LAT /c\\ORIGIN_LAT = \$ORIGINY" \$PARAM sed -i "/^PROJECTION /c\\PROJECTION = \$CRS" \$PARAM - sed -i "/^NTHREAD /c\\NTHREAD = $params.force_threads" \$PARAM cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/local/samplesheet_check.nf b/modules/local/samplesheet_check.nf deleted file mode 100644 index 44fe7f9..0000000 --- a/modules/local/samplesheet_check.nf +++ /dev/null @@ -1,31 +0,0 @@ -process SAMPLESHEET_CHECK { - tag "$samplesheet" - label 'process_single' - - conda "conda-forge::python=3.8.3" - container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/python:3.8.3' : - 'biocontainers/python:3.8.3' }" - - input: - path samplesheet - - output: - path '*.csv' , emit: csv - path "versions.yml", emit: versions - - when: - task.ext.when == null || task.ext.when - - script: // This script is bundled with the pipeline, in nf-core/rangeland/bin/ - """ - check_samplesheet.py \\ - $samplesheet \\ - samplesheet.valid.csv - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - python: \$(python --version | sed 's/Python //g') - END_VERSIONS - """ -} diff --git a/modules/nf-core/custom/dumpsoftwareversions/main.nf b/modules/nf-core/custom/dumpsoftwareversions/main.nf deleted file mode 100644 index ebc8727..0000000 --- a/modules/nf-core/custom/dumpsoftwareversions/main.nf +++ /dev/null @@ -1,24 +0,0 @@ -process CUSTOM_DUMPSOFTWAREVERSIONS { - label 'process_single' - - // Requires `pyyaml` which does not have a dedicated container but is in the MultiQC container - conda "bioconda::multiqc=1.14" - container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/multiqc:1.14--pyhdfd78af_0' : - 'biocontainers/multiqc:1.14--pyhdfd78af_0' }" - - input: - path versions - - output: - path "software_versions.yml" , emit: yml - path "software_versions_mqc.yml", emit: mqc_yml - path "versions.yml" , emit: versions - - when: - task.ext.when == null || task.ext.when - - script: - def args = task.ext.args ?: '' - template 'dumpsoftwareversions.py' -} diff --git a/modules/nf-core/custom/dumpsoftwareversions/meta.yml b/modules/nf-core/custom/dumpsoftwareversions/meta.yml deleted file mode 100644 index c32657d..0000000 --- a/modules/nf-core/custom/dumpsoftwareversions/meta.yml +++ /dev/null @@ -1,36 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/yaml-schema.json -name: custom_dumpsoftwareversions -description: Custom module used to dump software versions within the nf-core pipeline template -keywords: - - custom - - dump - - version -tools: - - custom: - description: Custom module used to dump software versions within the nf-core pipeline template - homepage: https://github.com/nf-core/tools - documentation: https://github.com/nf-core/tools - licence: ["MIT"] -input: - - versions: - type: file - description: YML file containing software versions - pattern: "*.yml" - -output: - - yml: - type: file - description: Standard YML file containing software versions - pattern: "software_versions.yml" - - mqc_yml: - type: file - description: MultiQC custom content YML file containing software versions - pattern: "software_versions_mqc.yml" - - versions: - type: file - description: File containing software versions - pattern: "versions.yml" - -authors: - - "@drpatelh" - - "@grst" diff --git a/modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py b/modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py deleted file mode 100755 index e55b8d4..0000000 --- a/modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py +++ /dev/null @@ -1,102 +0,0 @@ -#!/usr/bin/env python - - -"""Provide functions to merge multiple versions.yml files.""" - - -import platform -from textwrap import dedent - -import yaml - - -def _make_versions_html(versions): - """Generate a tabular HTML output of all versions for MultiQC.""" - html = [ - dedent( - """\\ - - - - - - - - - - """ - ) - ] - for process, tmp_versions in sorted(versions.items()): - html.append("") - for i, (tool, version) in enumerate(sorted(tmp_versions.items())): - html.append( - dedent( - f"""\\ - - - - - - """ - ) - ) - html.append("") - html.append("
    Process Name Software Version
    {process if (i == 0) else ''}{tool}{version}
    ") - return "\\n".join(html) - - -def main(): - """Load all version files and generate merged output.""" - versions_this_module = {} - versions_this_module["${task.process}"] = { - "python": platform.python_version(), - "yaml": yaml.__version__, - } - - with open("$versions") as f: - versions_by_process = yaml.load(f, Loader=yaml.BaseLoader) | versions_this_module - - # aggregate versions by the module name (derived from fully-qualified process name) - versions_by_module = {} - for process, process_versions in versions_by_process.items(): - module = process.split(":")[-1] - try: - if versions_by_module[module] != process_versions: - raise AssertionError( - "We assume that software versions are the same between all modules. " - "If you see this error-message it means you discovered an edge-case " - "and should open an issue in nf-core/tools. " - ) - except KeyError: - versions_by_module[module] = process_versions - - versions_by_module["Workflow"] = { - "Nextflow": "$workflow.nextflow.version", - "$workflow.manifest.name": "$workflow.manifest.version", - } - - versions_mqc = { - "id": "software_versions", - "section_name": "${workflow.manifest.name} Software Versions", - "section_href": "https://github.com/${workflow.manifest.name}", - "plot_type": "html", - "description": "are collected at run time from the software output.", - "data": _make_versions_html(versions_by_module), - } - - with open("software_versions.yml", "w") as f: - yaml.dump(versions_by_module, f, default_flow_style=False) - with open("software_versions_mqc.yml", "w") as f: - yaml.dump(versions_mqc, f, default_flow_style=False) - - with open("versions.yml", "w") as f: - yaml.dump(versions_this_module, f, default_flow_style=False) - - -if __name__ == "__main__": - main() diff --git a/modules/nf-core/multiqc/environment.yml b/modules/nf-core/multiqc/environment.yml new file mode 100644 index 0000000..6f5b867 --- /dev/null +++ b/modules/nf-core/multiqc/environment.yml @@ -0,0 +1,5 @@ +channels: + - conda-forge + - bioconda +dependencies: + - bioconda::multiqc=1.25.1 diff --git a/modules/nf-core/multiqc/main.nf b/modules/nf-core/multiqc/main.nf index 1fc387b..cc0643e 100644 --- a/modules/nf-core/multiqc/main.nf +++ b/modules/nf-core/multiqc/main.nf @@ -1,16 +1,18 @@ process MULTIQC { label 'process_single' - conda "bioconda::multiqc=1.14" + conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/multiqc:1.14--pyhdfd78af_0' : - 'biocontainers/multiqc:1.14--pyhdfd78af_0' }" + 'https://depot.galaxyproject.org/singularity/multiqc:1.25.1--pyhdfd78af_0' : + 'biocontainers/multiqc:1.25.1--pyhdfd78af_0' }" input: path multiqc_files, stageAs: "?/*" path(multiqc_config) path(extra_multiqc_config) path(multiqc_logo) + path(replace_names) + path(sample_names) output: path "*multiqc_report.html", emit: report @@ -23,14 +25,22 @@ process MULTIQC { script: def args = task.ext.args ?: '' + def prefix = task.ext.prefix ? "--filename ${task.ext.prefix}.html" : '' def config = multiqc_config ? "--config $multiqc_config" : '' def extra_config = extra_multiqc_config ? "--config $extra_multiqc_config" : '' + def logo = multiqc_logo ? "--cl-config 'custom_logo: \"${multiqc_logo}\"'" : '' + def replace = replace_names ? "--replace-names ${replace_names}" : '' + def samples = sample_names ? "--sample-names ${sample_names}" : '' """ multiqc \\ --force \\ $args \\ $config \\ + $prefix \\ $extra_config \\ + $logo \\ + $replace \\ + $samples \\ . cat <<-END_VERSIONS > versions.yml @@ -41,8 +51,8 @@ process MULTIQC { stub: """ - touch multiqc_data - touch multiqc_plots + mkdir multiqc_data + mkdir multiqc_plots touch multiqc_report.html cat <<-END_VERSIONS > versions.yml diff --git a/modules/nf-core/multiqc/meta.yml b/modules/nf-core/multiqc/meta.yml index f93b5ee..b16c187 100644 --- a/modules/nf-core/multiqc/meta.yml +++ b/modules/nf-core/multiqc/meta.yml @@ -1,6 +1,6 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/yaml-schema.json -name: MultiQC -description: Aggregate results from bioinformatics analyses across many samples into a single report +name: multiqc +description: Aggregate results from bioinformatics analyses across many samples into + a single report keywords: - QC - bioinformatics tools @@ -13,44 +13,66 @@ tools: homepage: https://multiqc.info/ documentation: https://multiqc.info/docs/ licence: ["GPL-3.0-or-later"] - + identifier: biotools:multiqc input: - - multiqc_files: - type: file - description: | - List of reports / files recognised by MultiQC, for example the html and zip output of FastQC - - multiqc_config: - type: file - description: Optional config yml for MultiQC - pattern: "*.{yml,yaml}" - - extra_multiqc_config: - type: file - description: Second optional config yml for MultiQC. Will override common sections in multiqc_config. - pattern: "*.{yml,yaml}" - - multiqc_logo: - type: file - description: Optional logo file for MultiQC - pattern: "*.{png}" - + - - multiqc_files: + type: file + description: | + List of reports / files recognised by MultiQC, for example the html and zip output of FastQC + - - multiqc_config: + type: file + description: Optional config yml for MultiQC + pattern: "*.{yml,yaml}" + - - extra_multiqc_config: + type: file + description: Second optional config yml for MultiQC. Will override common sections + in multiqc_config. + pattern: "*.{yml,yaml}" + - - multiqc_logo: + type: file + description: Optional logo file for MultiQC + pattern: "*.{png}" + - - replace_names: + type: file + description: | + Optional two-column sample renaming file. First column a set of + patterns, second column a set of corresponding replacements. Passed via + MultiQC's `--replace-names` option. + pattern: "*.{tsv}" + - - sample_names: + type: file + description: | + Optional TSV file with headers, passed to the MultiQC --sample_names + argument. + pattern: "*.{tsv}" output: - report: - type: file - description: MultiQC report file - pattern: "multiqc_report.html" + - "*multiqc_report.html": + type: file + description: MultiQC report file + pattern: "multiqc_report.html" - data: - type: directory - description: MultiQC data dir - pattern: "multiqc_data" + - "*_data": + type: directory + description: MultiQC data dir + pattern: "multiqc_data" - plots: - type: file - description: Plots created by MultiQC - pattern: "*_data" + - "*_plots": + type: file + description: Plots created by MultiQC + pattern: "*_data" - versions: - type: file - description: File containing software versions - pattern: "versions.yml" + - versions.yml: + type: file + description: File containing software versions + pattern: "versions.yml" authors: - "@abhi18av" - "@bunop" - "@drpatelh" - "@jfy133" +maintainers: + - "@abhi18av" + - "@bunop" + - "@drpatelh" + - "@jfy133" diff --git a/modules/nf-core/multiqc/tests/main.nf.test b/modules/nf-core/multiqc/tests/main.nf.test new file mode 100644 index 0000000..33316a7 --- /dev/null +++ b/modules/nf-core/multiqc/tests/main.nf.test @@ -0,0 +1,92 @@ +nextflow_process { + + name "Test Process MULTIQC" + script "../main.nf" + process "MULTIQC" + + tag "modules" + tag "modules_nfcore" + tag "multiqc" + + config "./nextflow.config" + + test("sarscov2 single-end [fastqc]") { + + when { + process { + """ + input[0] = Channel.of(file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastqc/test_fastqc.zip', checkIfExists: true)) + input[1] = [] + input[2] = [] + input[3] = [] + input[4] = [] + input[5] = [] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert process.out.report[0] ==~ ".*/multiqc_report.html" }, + { assert process.out.data[0] ==~ ".*/multiqc_data" }, + { assert snapshot(process.out.versions).match("multiqc_versions_single") } + ) + } + + } + + test("sarscov2 single-end [fastqc] [config]") { + + when { + process { + """ + input[0] = Channel.of(file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastqc/test_fastqc.zip', checkIfExists: true)) + input[1] = Channel.of(file("https://github.com/nf-core/tools/raw/dev/nf_core/pipeline-template/assets/multiqc_config.yml", checkIfExists: true)) + input[2] = [] + input[3] = [] + input[4] = [] + input[5] = [] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert process.out.report[0] ==~ ".*/multiqc_report.html" }, + { assert process.out.data[0] ==~ ".*/multiqc_data" }, + { assert snapshot(process.out.versions).match("multiqc_versions_config") } + ) + } + } + + test("sarscov2 single-end [fastqc] - stub") { + + options "-stub" + + when { + process { + """ + input[0] = Channel.of(file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastqc/test_fastqc.zip', checkIfExists: true)) + input[1] = [] + input[2] = [] + input[3] = [] + input[4] = [] + input[5] = [] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out.report.collect { file(it).getName() } + + process.out.data.collect { file(it).getName() } + + process.out.plots.collect { file(it).getName() } + + process.out.versions ).match("multiqc_stub") } + ) + } + + } +} diff --git a/modules/nf-core/multiqc/tests/main.nf.test.snap b/modules/nf-core/multiqc/tests/main.nf.test.snap new file mode 100644 index 0000000..2fcbb5f --- /dev/null +++ b/modules/nf-core/multiqc/tests/main.nf.test.snap @@ -0,0 +1,41 @@ +{ + "multiqc_versions_single": { + "content": [ + [ + "versions.yml:md5,41f391dcedce7f93ca188f3a3ffa0916" + ] + ], + "meta": { + "nf-test": "0.9.0", + "nextflow": "24.04.4" + }, + "timestamp": "2024-10-02T17:51:46.317523" + }, + "multiqc_stub": { + "content": [ + [ + "multiqc_report.html", + "multiqc_data", + "multiqc_plots", + "versions.yml:md5,41f391dcedce7f93ca188f3a3ffa0916" + ] + ], + "meta": { + "nf-test": "0.9.0", + "nextflow": "24.04.4" + }, + "timestamp": "2024-10-02T17:52:20.680978" + }, + "multiqc_versions_config": { + "content": [ + [ + "versions.yml:md5,41f391dcedce7f93ca188f3a3ffa0916" + ] + ], + "meta": { + "nf-test": "0.9.0", + "nextflow": "24.04.4" + }, + "timestamp": "2024-10-02T17:52:09.185842" + } +} \ No newline at end of file diff --git a/modules/nf-core/multiqc/tests/nextflow.config b/modules/nf-core/multiqc/tests/nextflow.config new file mode 100644 index 0000000..c537a6a --- /dev/null +++ b/modules/nf-core/multiqc/tests/nextflow.config @@ -0,0 +1,5 @@ +process { + withName: 'MULTIQC' { + ext.prefix = null + } +} diff --git a/modules/nf-core/multiqc/tests/tags.yml b/modules/nf-core/multiqc/tests/tags.yml new file mode 100644 index 0000000..bea6c0d --- /dev/null +++ b/modules/nf-core/multiqc/tests/tags.yml @@ -0,0 +1,2 @@ +multiqc: + - modules/nf-core/multiqc/** diff --git a/modules/nf-core/untar/environment.yml b/modules/nf-core/untar/environment.yml new file mode 100644 index 0000000..c779485 --- /dev/null +++ b/modules/nf-core/untar/environment.yml @@ -0,0 +1,7 @@ +channels: + - conda-forge + - bioconda +dependencies: + - conda-forge::grep=3.11 + - conda-forge::sed=4.8 + - conda-forge::tar=1.34 diff --git a/modules/nf-core/untar/main.nf b/modules/nf-core/untar/main.nf index 61461c3..9bd8f55 100644 --- a/modules/nf-core/untar/main.nf +++ b/modules/nf-core/untar/main.nf @@ -2,10 +2,10 @@ process UNTAR { tag "$archive" label 'process_single' - conda "conda-forge::sed=4.7 conda-forge::grep=3.11 conda-forge::tar=1.34" + conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/ubuntu:20.04' : - 'nf-core/ubuntu:20.04' }" + 'https://depot.galaxyproject.org/singularity/ubuntu:22.04' : + 'nf-core/ubuntu:22.04' }" input: tuple val(meta), path(archive) @@ -52,8 +52,29 @@ process UNTAR { stub: prefix = task.ext.prefix ?: ( meta.id ? "${meta.id}" : archive.toString().replaceFirst(/\.[^\.]+(.gz)?$/, "")) """ - mkdir $prefix - touch ${prefix}/file.txt + mkdir ${prefix} + ## Dry-run untaring the archive to get the files and place all in prefix + if [[ \$(tar -taf ${archive} | grep -o -P "^.*?\\/" | uniq | wc -l) -eq 1 ]]; then + for i in `tar -tf ${archive}`; + do + if [[ \$(echo "\${i}" | grep -E "/\$") == "" ]]; + then + touch \${i} + else + mkdir -p \${i} + fi + done + else + for i in `tar -tf ${archive}`; + do + if [[ \$(echo "\${i}" | grep -E "/\$") == "" ]]; + then + touch ${prefix}/\${i} + else + mkdir -p ${prefix}/\${i} + fi + done + fi cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/nf-core/untar/meta.yml b/modules/nf-core/untar/meta.yml index a9a2110..290346b 100644 --- a/modules/nf-core/untar/meta.yml +++ b/modules/nf-core/untar/meta.yml @@ -10,30 +10,33 @@ tools: Extract tar.gz files. documentation: https://www.gnu.org/software/tar/manual/ licence: ["GPL-3.0-or-later"] + identifier: "" input: - - meta: - type: map - description: | - Groovy Map containing sample information - e.g. [ id:'test', single_end:false ] - - archive: - type: file - description: File to be untar - pattern: "*.{tar}.{gz}" + - - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - archive: + type: file + description: File to be untar + pattern: "*.{tar}.{gz}" output: - - meta: - type: map - description: | - Groovy Map containing sample information - e.g. [ id:'test', single_end:false ] - untar: - type: directory - description: Directory containing contents of archive - pattern: "*/" + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - $prefix: + type: directory + description: Directory containing contents of archive + pattern: "*/" - versions: - type: file - description: File containing software versions - pattern: "versions.yml" + - versions.yml: + type: file + description: File containing software versions + pattern: "versions.yml" authors: - "@joseespinosa" - "@drpatelh" diff --git a/modules/nf-core/untar/tests/main.nf.test b/modules/nf-core/untar/tests/main.nf.test new file mode 100644 index 0000000..c957517 --- /dev/null +++ b/modules/nf-core/untar/tests/main.nf.test @@ -0,0 +1,85 @@ +nextflow_process { + + name "Test Process UNTAR" + script "../main.nf" + process "UNTAR" + tag "modules" + tag "modules_nfcore" + tag "untar" + + test("test_untar") { + + when { + process { + """ + input[0] = [ [], file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/db/kraken2.tar.gz', checkIfExists: true) ] + """ + } + } + + then { + assertAll ( + { assert process.success }, + { assert snapshot(process.out).match() }, + ) + } + } + + test("test_untar_onlyfiles") { + + when { + process { + """ + input[0] = [ [], file(params.modules_testdata_base_path + 'generic/tar/hello.tar.gz', checkIfExists: true) ] + """ + } + } + + then { + assertAll ( + { assert process.success }, + { assert snapshot(process.out).match() }, + ) + } + } + + test("test_untar - stub") { + + options "-stub" + + when { + process { + """ + input[0] = [ [], file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/db/kraken2.tar.gz', checkIfExists: true) ] + """ + } + } + + then { + assertAll ( + { assert process.success }, + { assert snapshot(process.out).match() }, + ) + } + } + + test("test_untar_onlyfiles - stub") { + + options "-stub" + + when { + process { + """ + input[0] = [ [], file(params.modules_testdata_base_path + 'generic/tar/hello.tar.gz', checkIfExists: true) ] + """ + } + } + + then { + assertAll ( + { assert process.success }, + { assert snapshot(process.out).match() }, + ) + } + } +} diff --git a/modules/nf-core/untar/tests/main.nf.test.snap b/modules/nf-core/untar/tests/main.nf.test.snap new file mode 100644 index 0000000..ceb91b7 --- /dev/null +++ b/modules/nf-core/untar/tests/main.nf.test.snap @@ -0,0 +1,158 @@ +{ + "test_untar_onlyfiles": { + "content": [ + { + "0": [ + [ + [ + + ], + [ + "hello.txt:md5,e59ff97941044f85df5297e1c302d260" + ] + ] + ], + "1": [ + "versions.yml:md5,6063247258c56fd271d076bb04dd7536" + ], + "untar": [ + [ + [ + + ], + [ + "hello.txt:md5,e59ff97941044f85df5297e1c302d260" + ] + ] + ], + "versions": [ + "versions.yml:md5,6063247258c56fd271d076bb04dd7536" + ] + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "24.04.3" + }, + "timestamp": "2024-07-10T12:04:28.231047" + }, + "test_untar_onlyfiles - stub": { + "content": [ + { + "0": [ + [ + [ + + ], + [ + "hello.txt:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ] + ], + "1": [ + "versions.yml:md5,6063247258c56fd271d076bb04dd7536" + ], + "untar": [ + [ + [ + + ], + [ + "hello.txt:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ] + ], + "versions": [ + "versions.yml:md5,6063247258c56fd271d076bb04dd7536" + ] + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "24.04.3" + }, + "timestamp": "2024-07-10T12:04:45.773103" + }, + "test_untar - stub": { + "content": [ + { + "0": [ + [ + [ + + ], + [ + "hash.k2d:md5,d41d8cd98f00b204e9800998ecf8427e", + "opts.k2d:md5,d41d8cd98f00b204e9800998ecf8427e", + "taxo.k2d:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ] + ], + "1": [ + "versions.yml:md5,6063247258c56fd271d076bb04dd7536" + ], + "untar": [ + [ + [ + + ], + [ + "hash.k2d:md5,d41d8cd98f00b204e9800998ecf8427e", + "opts.k2d:md5,d41d8cd98f00b204e9800998ecf8427e", + "taxo.k2d:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ] + ], + "versions": [ + "versions.yml:md5,6063247258c56fd271d076bb04dd7536" + ] + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "24.04.3" + }, + "timestamp": "2024-07-10T12:04:36.777441" + }, + "test_untar": { + "content": [ + { + "0": [ + [ + [ + + ], + [ + "hash.k2d:md5,8b8598468f54a7087c203ad0190555d9", + "opts.k2d:md5,a033d00cf6759407010b21700938f543", + "taxo.k2d:md5,094d5891cdccf2f1468088855c214b2c" + ] + ] + ], + "1": [ + "versions.yml:md5,6063247258c56fd271d076bb04dd7536" + ], + "untar": [ + [ + [ + + ], + [ + "hash.k2d:md5,8b8598468f54a7087c203ad0190555d9", + "opts.k2d:md5,a033d00cf6759407010b21700938f543", + "taxo.k2d:md5,094d5891cdccf2f1468088855c214b2c" + ] + ] + ], + "versions": [ + "versions.yml:md5,6063247258c56fd271d076bb04dd7536" + ] + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "24.04.3" + }, + "timestamp": "2024-07-10T12:04:19.377674" + } +} \ No newline at end of file diff --git a/modules/nf-core/untar/tests/tags.yml b/modules/nf-core/untar/tests/tags.yml new file mode 100644 index 0000000..feb6f15 --- /dev/null +++ b/modules/nf-core/untar/tests/tags.yml @@ -0,0 +1,2 @@ +untar: + - modules/nf-core/untar/** diff --git a/nextflow.config b/nextflow.config index 37b8933..8edbb7c 100644 --- a/nextflow.config +++ b/nextflow.config @@ -10,196 +10,191 @@ params { // Input options - input = null - dem = null - wvdb = null - data_cube = null - aoi = null - endmember = null - - input_tar = false - dem_tar = false - wvdb_tar = false + input = null + dem = null + wvdb = null + data_cube = null + aoi = null + endmember = null // Remote sensing imagery parameters - sensors_level1 = "LT04,LT05,LE07,S2A" sensors_level2 = "LND04 LND05 LND07" resolution = 30 start_date = "1984-01-01" end_date = "2006-12-31" // Workflow configuration - group_size = 100 - only_tile = false + group_size = 100 + save_ard = false + save_tsa = false + publish_dir_enabled = true + + // Higher Level processing configuration + indexes = "NDVI BLUE GREEN RED NIR SWIR1 SWIR2" + return_tss = false - // FORCE - force_threads = 2 + // Visualization + mosaic_visualization = true + pyramid_visualization = true // MultiQC options - multiqc_config = null - multiqc_title = null - multiqc_logo = null - max_multiqc_email_size = '25.MB' + multiqc_config = null + multiqc_title = null + multiqc_logo = null + max_multiqc_email_size = '25.MB' multiqc_methods_description = null // Boilerplate options - outdir = null - publish_dir_mode = 'copy' - email = null - email_on_fail = null - plaintext_email = false - monochrome_logs = false - hook_url = null - help = false - version = false + outdir = null + publish_dir_mode = 'copy' + email = null + email_on_fail = null + plaintext_email = false + monochrome_logs = false + hook_url = null + help = false + help_full = false + show_hidden = false + version = false + pipelines_testdata_base_path = 'https://raw.githubusercontent.com/nf-core/test-datasets/' // Config options config_profile_name = null config_profile_description = null + custom_config_version = 'master' custom_config_base = "https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}" config_profile_contact = null config_profile_url = null - - // Max resource options - // Defaults only, expecting to be overwritten - max_memory = '128.GB' - max_cpus = 16 - max_time = '240.h' - // Schema validation default options - validationFailUnrecognisedParams = false - validationLenientMode = false - validationSchemaIgnoreParams = 'genomes,igenomes_base' - validationShowHiddenParams = false - validate_params = true - + validate_params = true } // Load base.config by default for all pipelines includeConfig 'conf/base.config' -// Load nf-core custom profiles from different Institutions -try { - includeConfig "${params.custom_config_base}/nfcore_custom.config" -} catch (Exception e) { - System.err.println("WARNING: Could not load nf-core/config profiles: ${params.custom_config_base}/nfcore_custom.config") -} - -// Load nf-core/rangeland custom profiles from different institutions. -// Warning: Uncomment only if a pipeline-specific instititutional config already exists on nf-core/configs! -// try { -// includeConfig "${params.custom_config_base}/pipeline/rangeland.config" -// } catch (Exception e) { -// System.err.println("WARNING: Could not load nf-core/config/rangeland profiles: ${params.custom_config_base}/pipeline/rangeland.config") -// } profiles { debug { - dumpHashes = true - process.beforeScript = 'echo $HOSTNAME' - cleanup = false + dumpHashes = true + process.beforeScript = 'echo $HOSTNAME' + cleanup = false + nextflow.enable.configProcessNamesValidation = true } conda { - conda.enabled = true - docker.enabled = false - singularity.enabled = false - podman.enabled = false - shifter.enabled = false - charliecloud.enabled = false - apptainer.enabled = false + conda.enabled = true + docker.enabled = false + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + conda.channels = ['conda-forge', 'bioconda'] + apptainer.enabled = false } mamba { - conda.enabled = true - conda.useMamba = true - docker.enabled = false - singularity.enabled = false - podman.enabled = false - shifter.enabled = false - charliecloud.enabled = false - apptainer.enabled = false + conda.enabled = true + conda.useMamba = true + docker.enabled = false + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + apptainer.enabled = false } docker { docker.enabled = true - docker.userEmulation = true conda.enabled = false singularity.enabled = false podman.enabled = false shifter.enabled = false charliecloud.enabled = false apptainer.enabled = false + docker.fixOwnership = true + docker.runOptions = '-u $(id -u):$(id -g)' } arm { - docker.runOptions = '-u $(id -u):$(id -g) --platform=linux/amd64' + docker.runOptions = '-u $(id -u):$(id -g) --platform=linux/amd64' } singularity { - singularity.enabled = true - singularity.autoMounts = true - conda.enabled = false - docker.enabled = false - podman.enabled = false - shifter.enabled = false - charliecloud.enabled = false - apptainer.enabled = false + singularity.enabled = true + singularity.autoMounts = true + conda.enabled = false + docker.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + apptainer.enabled = false } podman { - podman.enabled = true - conda.enabled = false - docker.enabled = false - singularity.enabled = false - shifter.enabled = false - charliecloud.enabled = false - apptainer.enabled = false + podman.enabled = true + conda.enabled = false + docker.enabled = false + singularity.enabled = false + shifter.enabled = false + charliecloud.enabled = false + apptainer.enabled = false } shifter { - shifter.enabled = true - conda.enabled = false - docker.enabled = false - singularity.enabled = false - podman.enabled = false - charliecloud.enabled = false - apptainer.enabled = false + shifter.enabled = true + conda.enabled = false + docker.enabled = false + singularity.enabled = false + podman.enabled = false + charliecloud.enabled = false + apptainer.enabled = false } charliecloud { - charliecloud.enabled = true - conda.enabled = false - docker.enabled = false - singularity.enabled = false - podman.enabled = false - shifter.enabled = false - apptainer.enabled = false + charliecloud.enabled = true + conda.enabled = false + docker.enabled = false + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + apptainer.enabled = false } apptainer { - apptainer.enabled = true - apptainer.autoMounts = true - conda.enabled = false - docker.enabled = false - singularity.enabled = false - podman.enabled = false - shifter.enabled = false - charliecloud.enabled = false + apptainer.enabled = true + apptainer.autoMounts = true + conda.enabled = false + docker.enabled = false + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + } + wave { + apptainer.ociAutoPull = true + singularity.ociAutoPull = true + wave.enabled = true + wave.freeze = true + wave.strategy = 'conda,container' } gitpod { - executor.name = 'local' - executor.cpus = 4 - executor.memory = 8.GB + executor.name = 'local' + executor.cpus = 4 + executor.memory = 8.GB } test { includeConfig 'conf/test.config' } test_full { includeConfig 'conf/test_full.config' } } -// Set default registry for Apptainer, Docker, Podman and Singularity independent of -profile -// Will not be used unless Apptainer / Docker / Podman / Singularity are enabled +// Load nf-core custom profiles from different Institutions +includeConfig !System.getenv('NXF_OFFLINE') && params.custom_config_base ? "${params.custom_config_base}/nfcore_custom.config" : "/dev/null" + +// Load nf-core/rangeland custom profiles from different institutions. +// nf-core: Optionally, you can add a pipeline-specific nf-core config at https://github.com/nf-core/configs +includeConfig !System.getenv('NXF_OFFLINE') && params.custom_config_base ? "${params.custom_config_base}/pipeline/rangeland.config" : "/dev/null" + +// Set default registry for Apptainer, Docker, Podman, Charliecloud and Singularity independent of -profile +// Will not be used unless Apptainer / Docker / Podman / Charliecloud / Singularity are enabled // Set to your registry if you have a mirror of containers -apptainer.registry = 'quay.io' -docker.registry = 'quay.io' -podman.registry = 'quay.io' -singularity.registry = 'quay.io' +apptainer.registry = 'quay.io' +docker.registry = 'quay.io' +podman.registry = 'quay.io' +singularity.registry = 'quay.io' +charliecloud.registry = 'quay.io' + -// Nextflow plugins -plugins { - id 'nf-validation' // Validation of pipeline parameters and creation of an input channel from a sample sheet -} // Export these variables to prevent local Python/R libraries from conflicting with those in the container // The JULIA depot path has been adjusted to a fixed path `/usr/local/share/julia` that needs to be used for packages in the container. @@ -212,8 +207,18 @@ env { JULIA_DEPOT_PATH = "/usr/local/share/julia" } -// Capture exit codes from upstream processes when piping -process.shell = ['/bin/bash', '-euo', 'pipefail'] +// Set bash options +process.shell = """\ +bash + +set -e # Exit if a tool returns a non-zero status/exit code +set -u # Treat unset variables and parameters as an error +set -o pipefail # Returns the status of the last command to exit with a non-zero status or zero if all successfully execute +set -C # No clobber - prevent output redirection from overwriting files. +""" + +// Disable process selector warnings by default. Use debug profile to enable warnings. +nextflow.enable.configProcessNamesValidation = false def trace_timestamp = new java.util.Date().format( 'yyyy-MM-dd_HH-mm-ss') timeline { @@ -237,45 +242,48 @@ manifest { name = 'nf-core/rangeland' author = """Fabian Lehmann, David Frantz, Felix Kummer""" homePage = 'https://github.com/nf-core/rangeland' - description = """""" + description = """Long-term vegetation trend analysis pipeline for rangeland systems using satellite imagery.""" mainScript = 'main.nf' - nextflowVersion = '!>=23.04.0' - version = '1.0dev' + nextflowVersion = '!>=24.04.2' + version = '1.0.0' doi = '' } -// Load modules.config for DSL2 module specific options -includeConfig 'conf/modules.config' +// Nextflow plugins +plugins { + id 'nf-schema@2.1.1' // Validation of pipeline parameters and creation of an input channel from a sample sheet +} + +validation { + defaultIgnoreParams = ["genomes"] + help { + enabled = true + command = "nextflow run $manifest.name -profile --input samplesheet.csv --outdir " + fullParameter = "help_full" + showHiddenParameter = "show_hidden" + beforeText = """ +-\033[2m----------------------------------------------------\033[0m- + \033[0;32m,--.\033[0;30m/\033[0;32m,-.\033[0m +\033[0;34m ___ __ __ __ ___ \033[0;32m/,-._.--~\'\033[0m +\033[0;34m |\\ | |__ __ / ` / \\ |__) |__ \033[0;33m} {\033[0m +\033[0;34m | \\| | \\__, \\__/ | \\ |___ \033[0;32m\\`-._,-`-,\033[0m + \033[0;32m`._,._,\'\033[0m +\033[0;35m ${manifest.name} ${manifest.version}\033[0m +-\033[2m----------------------------------------------------\033[0m- +""" + afterText = """${manifest.doi ? "* The pipeline\n" : ""}${manifest.doi.tokenize(",").collect { " https://doi.org/${it.trim().replace('https://doi.org/','')}"}.join("\n")}${manifest.doi ? "\n" : ""} +* The nf-core framework + https://doi.org/10.1038/s41587-020-0439-x -// Function to ensure that resource requirements don't go beyond -// a maximum limit -def check_max(obj, type) { - if (type == 'memory') { - try { - if (obj.compareTo(params.max_memory as nextflow.util.MemoryUnit) == 1) - return params.max_memory as nextflow.util.MemoryUnit - else - return obj - } catch (all) { - println " ### ERROR ### Max memory '${params.max_memory}' is not valid! Using default value: $obj" - return obj - } - } else if (type == 'time') { - try { - if (obj.compareTo(params.max_time as nextflow.util.Duration) == 1) - return params.max_time as nextflow.util.Duration - else - return obj - } catch (all) { - println " ### ERROR ### Max time '${params.max_time}' is not valid! Using default value: $obj" - return obj - } - } else if (type == 'cpus') { - try { - return Math.min( obj, params.max_cpus as int ) - } catch (all) { - println " ### ERROR ### Max cpus '${params.max_cpus}' is not valid! Using default value: $obj" - return obj - } +* Software dependencies + https://github.com/${manifest.name}/blob/master/CITATIONS.md +""" + } + summary { + beforeText = validation.help.beforeText + afterText = validation.help.afterText } } + +// Load modules.config for DSL2 module specific options +includeConfig 'conf/modules.config' diff --git a/nextflow_schema.json b/nextflow_schema.json index 631007e..173a047 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -1,10 +1,10 @@ { - "$schema": "http://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://raw.githubusercontent.com/nf-core/rangeland/master/nextflow_schema.json", "title": "nf-core/rangeland pipeline parameters", - "description": "", + "description": "Long-term vegetation trend analysis pipeline for rangeland systems using satellite imagery.", "type": "object", - "definitions": { + "$defs": { "input_output_options": { "title": "Input/output options", "type": "object", @@ -15,63 +15,62 @@ "input": { "type": "string", "fa_icon": "fas fa-satellite", - "help_text": "Set this to the root directory of all satellite imagery. Directory structure should match format of data downloaded with '$ force-level1-csd\n' For concrete directory structure see ./docs/usage.md ", - "description": "Root directory of all sattelite imagery.", - "mimetype": "application/x-tar" - }, - "input_tar": { - "type": "boolean", - "fa_icon": "fas fa-archive", - "description": "Indicates whether input is a tar archive.", - "help_text": "Set to true if input is a tar archive. The workflow will extract it then." + "exists": true, + "help_text": "Set this to the root directory of all satellite imagery. Directory structure should match format of data downloaded with '$ force-level1-csd\n' For concrete directory structure see [usage documentation](https://nf-co.re/rangeland/docs/usage).\n\nAlternatively, a tarball can be supplied. The same directory structure needs to be in place in that tarball.", + "description": "Root directory or tarball of all satellite imagery.", + "mimetype": "image/tiff, application/x-tar", + "format": "path" }, "dem": { "type": "string", - "help_text": "Path to directory containing a subdirectory with tile-wise digital elevation files (.tif) and a virtual dataset file (.vrt file) comprising all tile-wise files from the subdirectory. \n\n\n", + "help_text": "Path to directory containing a subdirectory with tile-wise digital elevation files (`.tif`) and a virtual dataset file (`.vrt` file) comprising all tile-wise files from the subdirectory. See [usage documentation](https://nf-co.re/rangeland/docs/usage) for details regarding the required structure.\n\nAlternatively, a tarball can be supplied. The same directory structure needs to be in place in that tarball.\n\nAlternatively, a tarball can be supplied. The same directory structure needs to be in place in that tarball.", "fa_icon": "fas fa-mountain", - "description": "Digital elevation model." - }, - "dem_tar": { - "type": "boolean", - "fa_icon": "fas fa-archive", - "description": "Indicates whether dem is a tar archive.", - "help_text": "Set to true if dem is a tar archive. The workflow will extract it then." + "exists": true, + "description": "Digital elevation model.", + "format": "path" }, "wvdb": { "type": "string", "fa_icon": "fas fa-burn", + "exists": true, "description": "Water vapor dataset.", - "help_text": "Directory containg a number text files describing global water vapor data at different timestamps, and a coordinate order (.coo-)file containig the reference system of the water vapor data." - }, - "wvdb_tar": { - "type": "boolean", - "fa_icon": "fas fa-archive", - "description": "Indicates whether wvdb is a tar archive.", - "help_text": "Set to true if wvdb is a tar archive. The workflow will extract it then." + "help_text": "Directory containing a number text files describing global water vapor data at different timestamps, and a coordinate order (`.coo`-)file containing the reference system of the water vapor data. See [usage documentation](https://nf-co.re/rangeland/docs/usage) for details regarding the required structure.\n\nAlternatively, a tarball can be supplied. The same directory structure needs to be in place in that tarball.", + "format": "path" }, "data_cube": { "type": "string", "description": "Datacube definition.", "fa_icon": "fas fa-cube", - "help_text": "A single .prj file describing the projection and reference grid of the generated datacube." + "exists": true, + "help_text": "A single `.prj` file describing the projection and reference grid of the generated datacube.", + "pattern": "^\\S+\\.prj$", + "format": "file-path" }, "aoi": { "type": "string", "fa_icon": "fas fa-chart-area", + "exists": true, "description": "Area of interest.", - "help_text": "A single vector file specifying the spacial extend of the targeted area." + "help_text": "A single vector file specifying the spatial extend of the targeted area.", + "pattern": "^\\S+\\.(gpkg|shp)$", + "format": "file-path" }, "endmember": { "type": "string", "fa_icon": "fas fa-chart-line", + "exists": true, "description": "Endmember definition.", - "help_text": "A single text file specifying where lines correspond to satellite bands and columns correspond to endmembers. Values correspond to reflectance values and are separated by spaces." + "help_text": "A single text file specifying where rows correspond to satellite bands and columns correspond to endmembers. Values correspond to reflectance values and are separated by spaces.", + "pattern": "^\\S+\\.txt$", + "format": "file-path", + "mimetype": "text/plain" }, "outdir": { "type": "string", "format": "directory-path", "description": "The output directory where the results will be saved. You have to use absolute paths to storage on Cloud infrastructure.", - "fa_icon": "fas fa-folder-open" + "fa_icon": "fas fa-folder-open", + "exists": true }, "email": { "type": "string", @@ -94,81 +93,123 @@ "default": "", "help_text": "These parameters are used to tell pipeline tools which data is expected, how this data has to be incorporated, and which parts of the imagery can be excluded (e.g. due to temporal mismatch). These parameters should be closely aligned with the provided input data.", "properties": { - "sensors_level1": { - "type": "string", - "default": "LT04,LT05,LE07,S2A", - "fa_icon": "fas fa-satellite", - "description": "Satellites for which first level data should be processed.", - "help_text": "String containing comma-separated indicators of satellites in FORCE level 1format. Possible options: \n\"LT04\": Landsat 4 TM, \n\"LT05\": Landsat 5 TM, \n\"LE07\": Landsat 7 ETM+,\n\"LC08\": Landsat 8 OLI,\n\"S2A\": Sentinel-2A MSI,\n\"S2B\": Sentinel-2B MSI" - }, "sensors_level2": { "type": "string", "default": "LND04 LND05 LND07", "fa_icon": "fas fa-satellite", "description": "Satellites for which data should be incorporated into higher level processing.", - "help_text": "String containing space-separated indicators of satellites in FORCE level 2 format. Common options:\n\"LND04\": 6-band Landsat 4 TM, \n\"LND05\": 6-band Landsat 5 TM, \n\"LND07\": 6-band Landsat 7 ETM+,\n\"LND08/09\": 6-band Landsat 8-9 OLI, \n\"SEN2A\": 10-band Sentinel-2A, \n\"SEN2B\": 10-band Sentinel-2B,\nall options [here](https://force-eo.readthedocs.io/en/latest/components/higher-level/tsa/param.html) \n" + "help_text": "String containing space-separated indicators of satellites in FORCE level 2 format. Common options:\n\"LND04\": 6-band Landsat 4 TM, \n\"LND05\": 6-band Landsat 5 TM, \n\"LND07\": 6-band Landsat 7 ETM+,\n\"LND08/09\": 6-band Landsat 8-9 OLI, \n\"SEN2A\": 10-band Sentinel-2A, \n\"SEN2B\": 10-band Sentinel-2B,\nall options [here](https://force-eo.readthedocs.io/en/latest/components/higher-level/tsa/param.html) \n", + "pattern": "^((LND04|LND05|LND07|LND08|LND09|SEN2A|SEN2B|sen2a|sen2b|S1AIA|S1BIA|S1AID|S1BID|MOD01|MOD02|LNDLG|SEN2L|SEN2H|R-G-B|VVVHP|MODIS)(\\s|$))+$" }, "start_date": { "type": "string", "default": "1984-01-01", "fa_icon": "far fa-calendar-alt", "description": "First day of interest.", - "help_text": "String with format: \"YYYY-MM-DD\"." + "help_text": "String with format: \"YYYY-MM-DD\".", + "pattern": "^\\d{4}-\\d{2}-\\d{2}$", + "format": "date" }, "end_date": { "type": "string", "default": "2006-12-31", "fa_icon": "far fa-calendar-alt", "description": "Last day of interest.", - "help_text": "String with format: \"YYYY-MM-DD\"." + "help_text": "String with format: \"YYYY-MM-DD\".", + "pattern": "^\\d{4}-\\d{2}-\\d{2}$", + "format": "date" }, "resolution": { "type": "integer", "default": 30, "fa_icon": "fas fa-expand-arrows-alt", - "description": "Spatial resolution applied in analyses." + "description": "Spatial resolution applied in analyses.", + "minimum": 1 } }, "fa_icon": "fas fa-satellite" }, + "higher_level_processing_modification": { + "title": "Higher level processing modification", + "type": "object", + "default": "", + "properties": { + "indexes": { + "type": "string", + "default": "NDVI BLUE GREEN RED NIR SWIR1 SWIR2", + "help_text": "Space-separated list of indexes and bands that should be considered in time series analyses. They are indicated by using their established abbreviations. The full list of available indexes is available at https://force-eo.readthedocs.io/en/latest/components/higher-level/tsa/param.html under the 'INDEX' parameter. Spectral unmixing is a special index and always activated.", + "description": "Select which bands and indexes should be considered in time series analyses.", + "fa_icon": "fas fa-satellite", + "pattern": "((BLUE|GREEN|RED|NIR|SWIR1|SWIR2|RE1|RE2|RE3|BNIR|NDVI|EVI|NBR|NDTI|ARVI|SAVI|SARVI|TC-BRIGHT|TC-GREEN|TC-WET|TC-DI|NDBI|NDWI|MNDWI|NDMI|NDSI|SMA|kNDVI|NDRE1|NDRE2|CIre|NDVIre1|NDVIre2|NDVIre3|NDVIre1n|NDVIre2n|NDVIre3n|MSRre|MSRren,CCI)(\\s|$))+" + }, + "return_tss": { + "type": "boolean", + "description": "Should the full time series stack be returned.", + "help_text": "This parameter enables or disables the output of the time series stack for each tile and band/index. Here, each band shows the values of a single date.", + "fa_icon": "fas fa-layer-group" + } + }, + "description": "Modify higher level processing behaviour.", + "help_text": "These parameters can be used to modify the behaviour of the time series analyses process in higher level processing.", + "fa_icon": "fas fa-microchip" + }, + "visualization": { + "title": "Visualization", + "type": "object", + "description": "Enable or disable visualizations.", + "default": "", + "properties": { + "mosaic_visualization": { + "type": "boolean", + "default": true, + "description": "Whether mosaic visualization should be returned.", + "fa_icon": "fas fa-image" + }, + "pyramid_visualization": { + "type": "boolean", + "default": true, + "description": "Whether pyramid visualization should be returned.", + "fa_icon": "far fa-image" + } + }, + "fa_icon": "fas fa-images" + }, "workflow_configuration": { "title": "Workflow configuration", "type": "object", "description": "Parameters that configure workflow behavior not directly related to FORCE,", "default": "", "properties": { + "save_ard": { + "type": "boolean", + "fa_icon": "fas fa-database", + "description": "Whether analysis ready data should be published to the output directory.", + "help_text": "By default, the results of preprocessing (aka. analyses ready data or level-2 data) are not published. Setting this parameter to `true` enforces that these files are published to the output directory." + }, + "save_tsa": { + "type": "boolean", + "fa_icon": "fas fa-database", + "description": "Whether the results of time series analyses should be published to the output directory.", + "help_text": "By default, the results of higher level processing (aka. level-3 data) are not published. Setting this parameter to `true` enforces that these files are published to the output directory." + }, "group_size": { "type": "integer", "default": 100, - "description": "Batch size of tiles considered for merging.", + "description": "Batch size of tiles considered for merging.", "help_text": "Controls trade-off between parallelism and I/O load.\nHigher group_size -> More tiles merged in one process, less I/O load.\nLower group_size -> Less tiles merged in one process, more I/O load. ", "fa_icon": "fas fa-layer-group" }, - "only_tile": { + "publish_dir_enabled": { "type": "boolean", - "description": "Controls wheter spectral unmixing is applied in higher level processing..", - "fa_icon": "fas fa-project-diagram" + "default": true, + "description": "Publish pipeline outputs.", + "help_text": "Set to `false` to prevent *all* modules from publishing their results.", + "fa_icon": "fas fa-database" } }, "help_text": "You can savely use the defaults.", "fa_icon": "fas fa-project-diagram" }, - "force_parameters": { - "title": "FORCE parameters", - "type": "object", - "description": "Parameters specific to the FORCE tool.", - "default": "", - "properties": { - "force_threads": { - "type": "integer", - "default": 2, - "description": "Number of threads spawned by FORCE for each higher-level or preprocessing task.", - "fa_icon": "fas fa-microchip" - } - }, - "fa_icon": "fas fa-terminal", - "help_text": "" - }, "institutional_config_options": { "title": "Institutional config options", "type": "object", @@ -217,41 +258,6 @@ } } }, - "max_job_request_options": { - "title": "Max job request options", - "type": "object", - "fa_icon": "fab fa-acquisitions-incorporated", - "description": "Set the top limit for requested resources for any single job.", - "help_text": "If you are running on a smaller system, a pipeline step requesting more resources than are available may cause the Nextflow to stop the run with an error. These options allow you to cap the maximum resources requested by any single job so that the pipeline will run on your system.\n\nNote that you can not _increase_ the resources requested by any job using these options. For that you will need your own configuration file. See [the nf-core website](https://nf-co.re/usage/configuration) for details.", - "properties": { - "max_cpus": { - "type": "integer", - "description": "Maximum number of CPUs that can be requested for any single job.", - "default": 16, - "fa_icon": "fas fa-microchip", - "hidden": true, - "help_text": "Use to set an upper-limit for the CPU requirement for each process. Should be an integer e.g. `--max_cpus 1`" - }, - "max_memory": { - "type": "string", - "description": "Maximum amount of memory that can be requested for any single job.", - "default": "128.GB", - "fa_icon": "fas fa-memory", - "pattern": "^\\d+(\\.\\d+)?\\.?\\s*(K|M|G|T)?B$", - "hidden": true, - "help_text": "Use to set an upper-limit for the memory requirement for each process. Should be a string in the format integer-unit e.g. `--max_memory '8.GB'`" - }, - "max_time": { - "type": "string", - "description": "Maximum amount of time that can be requested for any single job.", - "default": "240.h", - "fa_icon": "far fa-clock", - "pattern": "^(\\d+\\.?\\s*(s|m|h|d|day)\\s*)+$", - "hidden": true, - "help_text": "Use to set an upper-limit for the time requirement for each process. Should be a string in the format integer-unit e.g. `--max_time '2.h'`" - } - } - }, "generic_options": { "title": "Generic options", "type": "object", @@ -259,12 +265,6 @@ "description": "Less common options for the pipeline, typically set in a config file.", "help_text": "These options are common to all nf-core pipelines and allow you to customise some of the core preferences for how the pipeline runs.\n\nTypically these options would be set in a Nextflow config file loaded for all pipeline runs, such as `~/.nextflow/config`.", "properties": { - "help": { - "type": "boolean", - "description": "Display help text.", - "fa_icon": "fas fa-question-circle", - "hidden": true - }, "version": { "type": "boolean", "description": "Display version and exit.", @@ -340,51 +340,37 @@ "fa_icon": "fas fa-check-square", "hidden": true }, - "validationShowHiddenParams": { - "type": "boolean", - "fa_icon": "far fa-eye-slash", - "description": "Show all params when using `--help`", - "hidden": true, - "help_text": "By default, parameters set as _hidden_ in the schema are not shown on the command line when a user runs with `--help`. Specifying this option will tell the pipeline to show all parameters." - }, - "validationFailUnrecognisedParams": { - "type": "boolean", - "fa_icon": "far fa-check-circle", - "description": "Validation of parameters fails when an unrecognised parameter is found.", - "hidden": true, - "help_text": "By default, when an unrecognised parameter is found, it returns a warinig." - }, - "validationLenientMode": { - "type": "boolean", + "pipelines_testdata_base_path": { + "type": "string", "fa_icon": "far fa-check-circle", - "description": "Validation of parameters in lenient more.", - "hidden": true, - "help_text": "Allows string values that are parseable as numbers or booleans. For further information see [JSONSchema docs](https://github.com/everit-org/json-schema#lenient-mode)." + "description": "Base URL or local path to location of pipeline test dataset files", + "default": "https://raw.githubusercontent.com/nf-core/test-datasets/", + "hidden": true } } } }, "allOf": [ { - "$ref": "#/definitions/input_output_options" + "$ref": "#/$defs/input_output_options" }, { - "$ref": "#/definitions/remote_sensing_image_options" + "$ref": "#/$defs/remote_sensing_image_options" }, { - "$ref": "#/definitions/workflow_configuration" + "$ref": "#/$defs/higher_level_processing_modification" }, { - "$ref": "#/definitions/force_parameters" + "$ref": "#/$defs/visualization" }, { - "$ref": "#/definitions/institutional_config_options" + "$ref": "#/$defs/workflow_configuration" }, { - "$ref": "#/definitions/max_job_request_options" + "$ref": "#/$defs/institutional_config_options" }, { - "$ref": "#/definitions/generic_options" + "$ref": "#/$defs/generic_options" } ] } diff --git a/nf-test.config b/nf-test.config new file mode 100644 index 0000000..843ed05 --- /dev/null +++ b/nf-test.config @@ -0,0 +1,11 @@ +config { + + testsDir "tests" + workDir ".nf-test" + configFile "tests/nextflow.config" + profile "docker,test" + + plugins { + load "nft-utils@0.0.3" + } +} diff --git a/pyproject.toml b/pyproject.toml deleted file mode 100644 index 0d62beb..0000000 --- a/pyproject.toml +++ /dev/null @@ -1,10 +0,0 @@ -# Config file for Python. Mostly used to configure linting of bin/check_samplesheet.py with Black. -# Should be kept the same as nf-core/tools to avoid fighting with template synchronisation. -[tool.black] -line-length = 120 -target_version = ["py37", "py38", "py39", "py310"] - -[tool.isort] -profile = "black" -known_first_party = ["nf_core"] -multi_line_output = 3 diff --git a/subworkflows/local/higher_level.nf b/subworkflows/local/higher_level.nf index fb7b838..1891ae0 100644 --- a/subworkflows/local/higher_level.nf +++ b/subworkflows/local/higher_level.nf @@ -1,10 +1,7 @@ -nextflow.enable.dsl = 2 - -//inputs -include { HIGHER_LEVEL_CONFIG } from '../../modules/local/higher_level_force_config.nf' -include { FORCE_HIGHER_LEVEL } from '../../modules/local/force-higher_level.nf' -include { FORCE_MOSAIC } from '../../modules/local/force-mosaic.nf' -include { FORCE_PYRAMID } from '../../modules/local/force-pyramid.nf' +include { HIGHER_LEVEL_CONFIG } from '../../modules/local/higher_level_force_config/main' +include { FORCE_HIGHER_LEVEL } from '../../modules/local/force-higher_level/main' +include { FORCE_MOSAIC } from '../../modules/local/force-mosaic/main' +include { FORCE_PYRAMID } from '../../modules/local/force-pyramid/main' workflow HIGHER_LEVEL { @@ -12,33 +9,59 @@ workflow HIGHER_LEVEL { tiles_and_masks cube_file endmember_file + mosaic_visualization + pyramid_visualization + resolution + sensors_level2 + start_date + end_date + indexes + return_tss main: ch_versions = Channel.empty() // create configuration file for higher level processing - HIGHER_LEVEL_CONFIG( tiles_and_masks, cube_file, endmember_file ) - ch_versions = ch_versions.mix(HIGHER_LEVEL_CONFIG.out.versions.first().ifEmpty(null)) + HIGHER_LEVEL_CONFIG ( + tiles_and_masks, + cube_file, + endmember_file, + resolution, + sensors_level2, + start_date, + end_date, + indexes, + return_tss + ) + ch_versions = ch_versions.mix(HIGHER_LEVEL_CONFIG.out.versions.first()) // main processing FORCE_HIGHER_LEVEL( HIGHER_LEVEL_CONFIG.out.higher_level_configs_and_data ) - ch_versions = ch_versions.mix(FORCE_HIGHER_LEVEL.out.versions.first().ifEmpty(null)) - + ch_versions = ch_versions.mix(FORCE_HIGHER_LEVEL.out.versions.first()) trend_files = FORCE_HIGHER_LEVEL.out.trend_files.flatten().map{ x -> [ x.simpleName.substring(12), x ] } trend_files_mosaic = trend_files.groupTuple() // visualizations - FORCE_MOSAIC( trend_files_mosaic, cube_file ) - ch_versions = ch_versions.mix(FORCE_MOSAIC.out.versions.first().ifEmpty(null)) + mosaic_files = Channel.empty() + if (mosaic_visualization) { + FORCE_MOSAIC( trend_files_mosaic, cube_file ) + mosaic_files = FORCE_MOSAIC.out.trend_files + ch_versions = ch_versions.mix(FORCE_MOSAIC.out.versions.first()) + } - FORCE_PYRAMID( trend_files.filter { it[1].name.endsWith('.tif') }.map { [ it[1].simpleName.substring(0,11), it[1] ] } .groupTuple() ) - ch_versions = ch_versions.mix(FORCE_PYRAMID.out.versions.first().ifEmpty(null)) + pyramid_files = Channel.empty() + if (pyramid_visualization) { + FORCE_PYRAMID( trend_files.filter { it[1].name.endsWith('.tif') }.map { [ it[1].simpleName.substring(0,11), it[1] ] } ) + pyramid_files = FORCE_PYRAMID.out.trends + ch_versions = ch_versions.mix(FORCE_PYRAMID.out.versions.first()) + } emit: - trend_files = FORCE_MOSAIC.out.trend_files - versions = ch_versions - + mosaic = mosaic_files + pyramid = pyramid_files + trends = FORCE_HIGHER_LEVEL.out.trend_files + versions = ch_versions } diff --git a/subworkflows/local/input_check.nf b/subworkflows/local/input_check.nf deleted file mode 100644 index 0aecf87..0000000 --- a/subworkflows/local/input_check.nf +++ /dev/null @@ -1,44 +0,0 @@ -// -// Check input samplesheet and get read channels -// - -include { SAMPLESHEET_CHECK } from '../../modules/local/samplesheet_check' - -workflow INPUT_CHECK { - take: - samplesheet // file: /path/to/samplesheet.csv - - main: - SAMPLESHEET_CHECK ( samplesheet ) - .csv - .splitCsv ( header:true, sep:',' ) - .map { create_fastq_channel(it) } - .set { reads } - - emit: - reads // channel: [ val(meta), [ reads ] ] - versions = SAMPLESHEET_CHECK.out.versions // channel: [ versions.yml ] -} - -// Function to get list of [ meta, [ fastq_1, fastq_2 ] ] -def create_fastq_channel(LinkedHashMap row) { - // create meta map - def meta = [:] - meta.id = row.sample - meta.single_end = row.single_end.toBoolean() - - // add path(s) of the fastq file(s) to the meta map - def fastq_meta = [] - if (!file(row.fastq_1).exists()) { - exit 1, "ERROR: Please check input samplesheet -> Read 1 FastQ file does not exist!\n${row.fastq_1}" - } - if (meta.single_end) { - fastq_meta = [ meta, [ file(row.fastq_1) ] ] - } else { - if (!file(row.fastq_2).exists()) { - exit 1, "ERROR: Please check input samplesheet -> Read 2 FastQ file does not exist!\n${row.fastq_2}" - } - fastq_meta = [ meta, [ file(row.fastq_1), file(row.fastq_2) ] ] - } - return fastq_meta -} diff --git a/subworkflows/local/preprocessing.nf b/subworkflows/local/preprocessing.nf index 82be52e..0ea00d2 100644 --- a/subworkflows/local/preprocessing.nf +++ b/subworkflows/local/preprocessing.nf @@ -1,13 +1,10 @@ -nextflow.enable.dsl = 2 +include { FORCE_GENERATE_TILE_ALLOW_LIST } from '../../modules/local/force-generate_tile_allow_list/main' +include { FORCE_GENERATE_ANALYSIS_MASK } from '../../modules/local/force-generate_analysis_mask/main' +include { PREPROCESS_CONFIG } from '../../modules/local/preprocess_force_config/main' +include { FORCE_PREPROCESS } from '../../modules/local/force-preprocess/main' +include { MERGE as MERGE_BOA; MERGE as MERGE_QAI } from '../../modules/local/merge/main' -//includes -include { FORCE_GENERATE_TILE_ALLOW_LIST } from '../../modules/local/force-generate_tile_allow_list' -include { FORCE_GENERATE_ANALYSIS_MASK } from '../../modules/local/force-generate_analysis_mask' -include { PREPROCESS_CONFIG } from '../../modules/local/preprocess_force_config' -include { FORCE_PREPROCESS } from '../../modules/local/force-preprocess' -include { MERGE as MERGE_BOA; MERGE as MERGE_QAI } from '../../modules/local/merge' - -//Closure to extract the parent directory of a file +// Closure to extract the parent directory of a file def extractDirectory = { it.parent.toString().substring(it.parent.toString().lastIndexOf('/') + 1 ) } workflow PREPROCESSING { @@ -18,6 +15,8 @@ workflow PREPROCESSING { wvdb cube_file aoi_file + group_size + resolution main: @@ -26,7 +25,7 @@ workflow PREPROCESSING { FORCE_GENERATE_TILE_ALLOW_LIST( aoi_file, cube_file ) ch_versions = ch_versions.mix(FORCE_GENERATE_TILE_ALLOW_LIST.out.versions) - FORCE_GENERATE_ANALYSIS_MASK( aoi_file, cube_file ) + FORCE_GENERATE_ANALYSIS_MASK( aoi_file, cube_file, resolution ) ch_versions = ch_versions.mix(FORCE_GENERATE_ANALYSIS_MASK.out.versions) //Group masks by tile @@ -34,11 +33,11 @@ workflow PREPROCESSING { // Preprocessing configuration PREPROCESS_CONFIG( data, cube_file, FORCE_GENERATE_TILE_ALLOW_LIST.out.tile_allow, dem, wvdb ) - ch_versions = ch_versions.mix(PREPROCESS_CONFIG.out.versions.first().ifEmpty(null)) + ch_versions = ch_versions.mix(PREPROCESS_CONFIG.out.versions.first()) // Main preprocessing FORCE_PREPROCESS( PREPROCESS_CONFIG.out.preprocess_config_and_data) - ch_versions = ch_versions.mix(FORCE_PREPROCESS.out.versions.first().ifEmpty(null)) + ch_versions = ch_versions.mix(FORCE_PREPROCESS.out.versions.first()) //Group by tile, date and sensor boa_tiles = FORCE_PREPROCESS.out.boa_tiles.flatten().map{ [ "${extractDirectory(it)}_${it.simpleName}", it ] }.groupTuple() @@ -50,23 +49,24 @@ workflow PREPROCESSING { //Sort to ensure the same groups if you use resume .toSortedList{ a,b -> a[1][0].simpleName <=> b[1][0].simpleName } .flatMap{it} - .groupTuple( remainder : true, size : params.group_size ).map{ [ it[0], it[1] .flatten() ] } + .groupTuple( remainder : true, size : group_size ).map{ [ it[0], it[1] .flatten() ] } + qai_tiles_to_merge = qai_tiles.filter{ x -> x[1].size() > 1 } .map{ [ it[0].substring( 0, 11 ), it[1] ] } //Sort to ensure the same groups if you use resume .toSortedList{ a,b -> a[1][0].simpleName <=> b[1][0].simpleName } .flatMap{it} - .groupTuple( remainder : true, size : params.group_size ).map{ [ it[0], it[1] .flatten() ] } + .groupTuple( remainder : true, size : group_size ).map{ [ it[0], it[1] .flatten() ] } //Find tiles with only one file boa_tiles_done = boa_tiles.filter{ x -> x[1].size() == 1 }.map{ x -> [ x[0] .substring( 0, 11 ), x[1][0] ] } qai_tiles_done = qai_tiles.filter{ x -> x[1].size() == 1 }.map{ x -> [ x[0] .substring( 0, 11 ), x[1][0] ] } MERGE_BOA( "boa", boa_tiles_to_merge, cube_file ) - ch_versions = ch_versions.mix(MERGE_BOA.out.versions.first().ifEmpty(null)) + ch_versions = ch_versions.mix(MERGE_BOA.out.versions.first()) MERGE_QAI( "qai", qai_tiles_to_merge, cube_file ) - ch_versions = ch_versions.mix(MERGE_QAI.out.versions.first().ifEmpty(null)) + ch_versions = ch_versions.mix(MERGE_QAI.out.versions.first()) //Concat merged list with single images, group by tile over time boa_tiles = MERGE_BOA.out.tiles_merged @@ -79,5 +79,4 @@ workflow PREPROCESSING { emit: tiles_and_masks = boa_tiles.join( qai_tiles ).join( masks ) versions = ch_versions - } diff --git a/subworkflows/local/utils_nfcore_rangeland_pipeline/main.nf b/subworkflows/local/utils_nfcore_rangeland_pipeline/main.nf new file mode 100644 index 0000000..f7bd710 --- /dev/null +++ b/subworkflows/local/utils_nfcore_rangeland_pipeline/main.nf @@ -0,0 +1,200 @@ +// +// Subworkflow with functionality specific to the nf-core/rangeland pipeline +// + +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + IMPORT FUNCTIONS / MODULES / SUBWORKFLOWS +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +*/ + +include { UTILS_NFSCHEMA_PLUGIN } from '../../nf-core/utils_nfschema_plugin' +include { paramsSummaryMap } from 'plugin/nf-schema' +include { samplesheetToList } from 'plugin/nf-schema' +include { completionEmail } from '../../nf-core/utils_nfcore_pipeline' +include { completionSummary } from '../../nf-core/utils_nfcore_pipeline' +include { imNotification } from '../../nf-core/utils_nfcore_pipeline' +include { UTILS_NFCORE_PIPELINE } from '../../nf-core/utils_nfcore_pipeline' +include { UTILS_NEXTFLOW_PIPELINE } from '../../nf-core/utils_nextflow_pipeline' + +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + SUBWORKFLOW TO INITIALISE PIPELINE +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +*/ + +workflow PIPELINE_INITIALISATION { + + take: + version // boolean: Display version and exit + validate_params // boolean: Boolean whether to validate parameters against the schema at runtime + monochrome_logs // boolean: Do not use coloured log outputs + nextflow_cli_args // array: List of positional nextflow CLI args + outdir // string: The output directory where the results will be saved + input // string: Path to input samplesheet + + main: + + ch_versions = Channel.empty() + + // + // Print version and exit if required and dump pipeline parameters to JSON file + // + UTILS_NEXTFLOW_PIPELINE ( + version, + true, + outdir, + workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1 + ) + + // + // Validate parameters and generate parameter summary to stdout + // + UTILS_NFSCHEMA_PLUGIN ( + workflow, + validate_params, + null + ) + + // + // Check config provided to the pipeline + // + UTILS_NFCORE_PIPELINE ( + nextflow_cli_args + ) + + + emit: + versions = ch_versions +} + +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + SUBWORKFLOW FOR PIPELINE COMPLETION +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +*/ + +workflow PIPELINE_COMPLETION { + + take: + email // string: email address + email_on_fail // string: email address sent on pipeline failure + plaintext_email // boolean: Send plain-text email instead of HTML + outdir // path: Path to output directory where results will be published + monochrome_logs // boolean: Disable ANSI colour codes in log output + hook_url // string: hook URL for notifications + multiqc_report // string: Path to MultiQC report + + main: + summary_params = paramsSummaryMap(workflow, parameters_schema: "nextflow_schema.json") + + // + // Completion email and summary + // + workflow.onComplete { + if (email || email_on_fail) { + completionEmail( + summary_params, + email, + email_on_fail, + plaintext_email, + outdir, + monochrome_logs, + multiqc_report.toList() + ) + } + + completionSummary(monochrome_logs) + if (hook_url) { + imNotification(summary_params, hook_url) + } + } + + workflow.onError { + log.error "Pipeline failed. Please refer to troubleshooting docs: https://nf-co.re/docs/usage/troubleshooting" + } +} + +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + FUNCTIONS +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +*/ + +// +// Validate channels from input samplesheet +// +def validateInputSamplesheet(input) { + def (metas, fastqs) = input[1..2] + + // Check that multiple runs of the same sample are of the same datatype i.e. single-end / paired-end + def endedness_ok = metas.collect{ meta -> meta.single_end }.unique().size == 1 + if (!endedness_ok) { + error("Please check input samplesheet -> Multiple runs of a sample must be of the same datatype i.e. single-end or paired-end: ${metas[0].id}") + } + + return [ metas[0], fastqs ] +} +// +// Generate methods description for MultiQC +// +def toolCitationText() { + // Can use ternary operators to dynamically construct based conditions, e.g. params["run_xyz"] ? "Tool (Foo et al. 2023)" : "", + // Uncomment function in methodsDescriptionText to render in MultiQC report + def citation_text = [ + "Tools used in the workflow included:", + "MultiQC (Ewels et al. 2016)", + "FORCE (Frantz 2019)", + "." + ].join(' ').trim() + + return citation_text +} + +def toolBibliographyText() { + // Can use ternary operators to dynamically construct based conditions, e.g. params["run_xyz"] ? "
  • Author (2023) Pub name, Journal, DOI
    • " : "", + // Uncomment function in methodsDescriptionText to render in MultiQC report + def reference_text = [ + "
  • Ewels, P., Magnusson, M., Lundin, S., & Käller, M. (2016). MultiQC: summarize analysis results for multiple tools and samples in a single report. Bioinformatics , 32(19), 3047–3048. doi: /10.1093/bioinformatics/btw354
    • ", + "
  • Frantz, D. (2019). FORCE—Landsat + Sentinel-2 Analysis Ready Data and Beyond. Remote Sensing, 11, 1124 doi: https://doi.org/10.3390/rs11091124
    • " + ].join(' ').trim() + + return reference_text +} + +def methodsDescriptionText(mqc_methods_yaml) { + // Convert to a named map so can be used as with familar NXF ${workflow} variable syntax in the MultiQC YML file + def meta = [:] + meta.workflow = workflow.toMap() + meta["manifest_map"] = workflow.manifest.toMap() + + // Pipeline DOI + if (meta.manifest_map.doi) { + // Using a loop to handle multiple DOIs + // Removing `https://doi.org/` to handle pipelines using DOIs vs DOI resolvers + // Removing ` ` since the manifest.doi is a string and not a proper list + def temp_doi_ref = "" + def manifest_doi = meta.manifest_map.doi.tokenize(",") + manifest_doi.each { doi_ref -> + temp_doi_ref += "(doi: ${doi_ref.replace("https://doi.org/", "").replace(" ", "")}), " + } + meta["doi_text"] = temp_doi_ref.substring(0, temp_doi_ref.length() - 2) + } else meta["doi_text"] = "" + meta["nodoi_text"] = meta.manifest_map.doi ? "" : "
  • If available, make sure to update the text to include the Zenodo DOI of version of the pipeline used.
    • " + + // Tool references + meta["tool_citations"] = "" + meta["tool_bibliography"] = "" + + meta["tool_citations"] = toolCitationText().replaceAll(", \\.", ".").replaceAll("\\. \\.", ".").replaceAll(", \\.", ".") + meta["tool_bibliography"] = toolBibliographyText() + + + def methods_text = mqc_methods_yaml.text + + def engine = new groovy.text.SimpleTemplateEngine() + def description_html = engine.createTemplate(methods_text).make(meta) + + return description_html.toString() +} + diff --git a/subworkflows/nf-core/utils_nextflow_pipeline/main.nf b/subworkflows/nf-core/utils_nextflow_pipeline/main.nf new file mode 100644 index 0000000..d6e593e --- /dev/null +++ b/subworkflows/nf-core/utils_nextflow_pipeline/main.nf @@ -0,0 +1,126 @@ +// +// Subworkflow with functionality that may be useful for any Nextflow pipeline +// + +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + SUBWORKFLOW DEFINITION +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +*/ + +workflow UTILS_NEXTFLOW_PIPELINE { + take: + print_version // boolean: print version + dump_parameters // boolean: dump parameters + outdir // path: base directory used to publish pipeline results + check_conda_channels // boolean: check conda channels + + main: + + // + // Print workflow version and exit on --version + // + if (print_version) { + log.info("${workflow.manifest.name} ${getWorkflowVersion()}") + System.exit(0) + } + + // + // Dump pipeline parameters to a JSON file + // + if (dump_parameters && outdir) { + dumpParametersToJSON(outdir) + } + + // + // When running with Conda, warn if channels have not been set-up appropriately + // + if (check_conda_channels) { + checkCondaChannels() + } + + emit: + dummy_emit = true +} + +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + FUNCTIONS +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +*/ + +// +// Generate version string +// +def getWorkflowVersion() { + def version_string = "" as String + if (workflow.manifest.version) { + def prefix_v = workflow.manifest.version[0] != 'v' ? 'v' : '' + version_string += "${prefix_v}${workflow.manifest.version}" + } + + if (workflow.commitId) { + def git_shortsha = workflow.commitId.substring(0, 7) + version_string += "-g${git_shortsha}" + } + + return version_string +} + +// +// Dump pipeline parameters to a JSON file +// +def dumpParametersToJSON(outdir) { + def timestamp = new java.util.Date().format('yyyy-MM-dd_HH-mm-ss') + def filename = "params_${timestamp}.json" + def temp_pf = new File(workflow.launchDir.toString(), ".${filename}") + def jsonStr = groovy.json.JsonOutput.toJson(params) + temp_pf.text = groovy.json.JsonOutput.prettyPrint(jsonStr) + + nextflow.extension.FilesEx.copyTo(temp_pf.toPath(), "${outdir}/pipeline_info/params_${timestamp}.json") + temp_pf.delete() +} + +// +// When running with -profile conda, warn if channels have not been set-up appropriately +// +def checkCondaChannels() { + def parser = new org.yaml.snakeyaml.Yaml() + def channels = [] + try { + def config = parser.load("conda config --show channels".execute().text) + channels = config.channels + } + catch (NullPointerException e) { + log.debug(e) + log.warn("Could not verify conda channel configuration.") + return null + } + catch (IOException e) { + log.debug(e) + log.warn("Could not verify conda channel configuration.") + return null + } + + // Check that all channels are present + // This channel list is ordered by required channel priority. + def required_channels_in_order = ['conda-forge', 'bioconda'] + def channels_missing = ((required_channels_in_order as Set) - (channels as Set)) as Boolean + + // Check that they are in the right order + def channel_priority_violation = required_channels_in_order != channels.findAll { ch -> ch in required_channels_in_order } + + if (channels_missing | channel_priority_violation) { + log.warn """\ + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + There is a problem with your Conda configuration! + You will need to set-up the conda-forge and bioconda channels correctly. + Please refer to https://bioconda.github.io/ + The observed channel order is + ${channels} + but the following channel order is required: + ${required_channels_in_order} + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + """.stripIndent(true) + } +} diff --git a/subworkflows/nf-core/utils_nextflow_pipeline/meta.yml b/subworkflows/nf-core/utils_nextflow_pipeline/meta.yml new file mode 100644 index 0000000..e5c3a0a --- /dev/null +++ b/subworkflows/nf-core/utils_nextflow_pipeline/meta.yml @@ -0,0 +1,38 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/subworkflows/yaml-schema.json +name: "UTILS_NEXTFLOW_PIPELINE" +description: Subworkflow with functionality that may be useful for any Nextflow pipeline +keywords: + - utility + - pipeline + - initialise + - version +components: [] +input: + - print_version: + type: boolean + description: | + Print the version of the pipeline and exit + - dump_parameters: + type: boolean + description: | + Dump the parameters of the pipeline to a JSON file + - output_directory: + type: directory + description: Path to output dir to write JSON file to. + pattern: "results/" + - check_conda_channel: + type: boolean + description: | + Check if the conda channel priority is correct. +output: + - dummy_emit: + type: boolean + description: | + Dummy emit to make nf-core subworkflows lint happy +authors: + - "@adamrtalbot" + - "@drpatelh" +maintainers: + - "@adamrtalbot" + - "@drpatelh" + - "@maxulysse" diff --git a/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.function.nf.test b/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.function.nf.test new file mode 100644 index 0000000..68718e4 --- /dev/null +++ b/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.function.nf.test @@ -0,0 +1,54 @@ + +nextflow_function { + + name "Test Functions" + script "subworkflows/nf-core/utils_nextflow_pipeline/main.nf" + config "subworkflows/nf-core/utils_nextflow_pipeline/tests/nextflow.config" + tag 'subworkflows' + tag 'utils_nextflow_pipeline' + tag 'subworkflows/utils_nextflow_pipeline' + + test("Test Function getWorkflowVersion") { + + function "getWorkflowVersion" + + then { + assertAll( + { assert function.success }, + { assert snapshot(function.result).match() } + ) + } + } + + test("Test Function dumpParametersToJSON") { + + function "dumpParametersToJSON" + + when { + function { + """ + // define inputs of the function here. Example: + input[0] = "$outputDir" + """.stripIndent() + } + } + + then { + assertAll( + { assert function.success } + ) + } + } + + test("Test Function checkCondaChannels") { + + function "checkCondaChannels" + + then { + assertAll( + { assert function.success }, + { assert snapshot(function.result).match() } + ) + } + } +} diff --git a/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.function.nf.test.snap b/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.function.nf.test.snap new file mode 100644 index 0000000..e3f0baf --- /dev/null +++ b/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.function.nf.test.snap @@ -0,0 +1,20 @@ +{ + "Test Function getWorkflowVersion": { + "content": [ + "v9.9.9" + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:02:05.308243" + }, + "Test Function checkCondaChannels": { + "content": null, + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:02:12.425833" + } +} \ No newline at end of file diff --git a/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.workflow.nf.test b/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.workflow.nf.test new file mode 100644 index 0000000..02dbf09 --- /dev/null +++ b/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.workflow.nf.test @@ -0,0 +1,113 @@ +nextflow_workflow { + + name "Test Workflow UTILS_NEXTFLOW_PIPELINE" + script "../main.nf" + config "subworkflows/nf-core/utils_nextflow_pipeline/tests/nextflow.config" + workflow "UTILS_NEXTFLOW_PIPELINE" + tag 'subworkflows' + tag 'utils_nextflow_pipeline' + tag 'subworkflows/utils_nextflow_pipeline' + + test("Should run no inputs") { + + when { + workflow { + """ + print_version = false + dump_parameters = false + outdir = null + check_conda_channels = false + + input[0] = print_version + input[1] = dump_parameters + input[2] = outdir + input[3] = check_conda_channels + """ + } + } + + then { + assertAll( + { assert workflow.success } + ) + } + } + + test("Should print version") { + + when { + workflow { + """ + print_version = true + dump_parameters = false + outdir = null + check_conda_channels = false + + input[0] = print_version + input[1] = dump_parameters + input[2] = outdir + input[3] = check_conda_channels + """ + } + } + + then { + expect { + with(workflow) { + assert success + assert "nextflow_workflow v9.9.9" in stdout + } + } + } + } + + test("Should dump params") { + + when { + workflow { + """ + print_version = false + dump_parameters = true + outdir = 'results' + check_conda_channels = false + + input[0] = false + input[1] = true + input[2] = outdir + input[3] = false + """ + } + } + + then { + assertAll( + { assert workflow.success } + ) + } + } + + test("Should not create params JSON if no output directory") { + + when { + workflow { + """ + print_version = false + dump_parameters = true + outdir = null + check_conda_channels = false + + input[0] = false + input[1] = true + input[2] = outdir + input[3] = false + """ + } + } + + then { + assertAll( + { assert workflow.success } + ) + } + } +} diff --git a/subworkflows/nf-core/utils_nextflow_pipeline/tests/nextflow.config b/subworkflows/nf-core/utils_nextflow_pipeline/tests/nextflow.config new file mode 100644 index 0000000..a09572e --- /dev/null +++ b/subworkflows/nf-core/utils_nextflow_pipeline/tests/nextflow.config @@ -0,0 +1,9 @@ +manifest { + name = 'nextflow_workflow' + author = """nf-core""" + homePage = 'https://127.0.0.1' + description = """Dummy pipeline""" + nextflowVersion = '!>=23.04.0' + version = '9.9.9' + doi = 'https://doi.org/10.5281/zenodo.5070524' +} diff --git a/subworkflows/nf-core/utils_nextflow_pipeline/tests/tags.yml b/subworkflows/nf-core/utils_nextflow_pipeline/tests/tags.yml new file mode 100644 index 0000000..f847611 --- /dev/null +++ b/subworkflows/nf-core/utils_nextflow_pipeline/tests/tags.yml @@ -0,0 +1,2 @@ +subworkflows/utils_nextflow_pipeline: + - subworkflows/nf-core/utils_nextflow_pipeline/** diff --git a/subworkflows/nf-core/utils_nfcore_pipeline/main.nf b/subworkflows/nf-core/utils_nfcore_pipeline/main.nf new file mode 100644 index 0000000..4cd3362 --- /dev/null +++ b/subworkflows/nf-core/utils_nfcore_pipeline/main.nf @@ -0,0 +1,465 @@ +// +// Subworkflow with utility functions specific to the nf-core pipeline template +// + +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + SUBWORKFLOW DEFINITION +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +*/ + +workflow UTILS_NFCORE_PIPELINE { + take: + nextflow_cli_args + + main: + valid_config = checkConfigProvided() + checkProfileProvided(nextflow_cli_args) + + emit: + valid_config +} + +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + FUNCTIONS +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +*/ + +// +// Warn if a -profile or Nextflow config has not been provided to run the pipeline +// +def checkConfigProvided() { + def valid_config = true as Boolean + if (workflow.profile == 'standard' && workflow.configFiles.size() <= 1) { + log.warn( + "[${workflow.manifest.name}] You are attempting to run the pipeline without any custom configuration!\n\n" + "This will be dependent on your local compute environment but can be achieved via one or more of the following:\n" + " (1) Using an existing pipeline profile e.g. `-profile docker` or `-profile singularity`\n" + " (2) Using an existing nf-core/configs for your Institution e.g. `-profile crick` or `-profile uppmax`\n" + " (3) Using your own local custom config e.g. `-c /path/to/your/custom.config`\n\n" + "Please refer to the quick start section and usage docs for the pipeline.\n " + ) + valid_config = false + } + return valid_config +} + +// +// Exit pipeline if --profile contains spaces +// +def checkProfileProvided(nextflow_cli_args) { + if (workflow.profile.endsWith(',')) { + error( + "The `-profile` option cannot end with a trailing comma, please remove it and re-run the pipeline!\n" + "HINT: A common mistake is to provide multiple values separated by spaces e.g. `-profile test, docker`.\n" + ) + } + if (nextflow_cli_args[0]) { + log.warn( + "nf-core pipelines do not accept positional arguments. The positional argument `${nextflow_cli_args[0]}` has been detected.\n" + "HINT: A common mistake is to provide multiple values separated by spaces e.g. `-profile test, docker`.\n" + ) + } +} + +// +// Citation string for pipeline +// +def workflowCitation() { + def temp_doi_ref = "" + def manifest_doi = workflow.manifest.doi.tokenize(",") + // Handling multiple DOIs + // Removing `https://doi.org/` to handle pipelines using DOIs vs DOI resolvers + // Removing ` ` since the manifest.doi is a string and not a proper list + manifest_doi.each { doi_ref -> + temp_doi_ref += " https://doi.org/${doi_ref.replace('https://doi.org/', '').replace(' ', '')}\n" + } + return "If you use ${workflow.manifest.name} for your analysis please cite:\n\n" + "* The pipeline\n" + temp_doi_ref + "\n" + "* The nf-core framework\n" + " https://doi.org/10.1038/s41587-020-0439-x\n\n" + "* Software dependencies\n" + " https://github.com/${workflow.manifest.name}/blob/master/CITATIONS.md" +} + +// +// Generate workflow version string +// +def getWorkflowVersion() { + def version_string = "" as String + if (workflow.manifest.version) { + def prefix_v = workflow.manifest.version[0] != 'v' ? 'v' : '' + version_string += "${prefix_v}${workflow.manifest.version}" + } + + if (workflow.commitId) { + def git_shortsha = workflow.commitId.substring(0, 7) + version_string += "-g${git_shortsha}" + } + + return version_string +} + +// +// Get software versions for pipeline +// +def processVersionsFromYAML(yaml_file) { + def yaml = new org.yaml.snakeyaml.Yaml() + def versions = yaml.load(yaml_file).collectEntries { k, v -> [k.tokenize(':')[-1], v] } + return yaml.dumpAsMap(versions).trim() +} + +// +// Get workflow version for pipeline +// +def workflowVersionToYAML() { + return """ + Workflow: + ${workflow.manifest.name}: ${getWorkflowVersion()} + Nextflow: ${workflow.nextflow.version} + """.stripIndent().trim() +} + +// +// Get channel of software versions used in pipeline in YAML format +// +def softwareVersionsToYAML(ch_versions) { + return ch_versions.unique().map { version -> processVersionsFromYAML(version) }.unique().mix(Channel.of(workflowVersionToYAML())) +} + +// +// Get workflow summary for MultiQC +// +def paramsSummaryMultiqc(summary_params) { + def summary_section = '' + summary_params + .keySet() + .each { group -> + def group_params = summary_params.get(group) + // This gets the parameters of that particular group + if (group_params) { + summary_section += "

    ${group}

    \n" + summary_section += "
    \n" + group_params + .keySet() + .sort() + .each { param -> + summary_section += "
    ${param}
    ${group_params.get(param) ?: 'N/A'}
    \n" + } + summary_section += "
    \n" + } + } + + def yaml_file_text = "id: '${workflow.manifest.name.replace('/', '-')}-summary'\n" as String + yaml_file_text += "description: ' - this information is collected when the pipeline is started.'\n" + yaml_file_text += "section_name: '${workflow.manifest.name} Workflow Summary'\n" + yaml_file_text += "section_href: 'https://github.com/${workflow.manifest.name}'\n" + yaml_file_text += "plot_type: 'html'\n" + yaml_file_text += "data: |\n" + yaml_file_text += "${summary_section}" + + return yaml_file_text +} + +// +// nf-core logo +// +def nfCoreLogo(monochrome_logs=true) { + def colors = logColours(monochrome_logs) as Map + String.format( + """\n + ${dashedLine(monochrome_logs)} + ${colors.green},--.${colors.black}/${colors.green},-.${colors.reset} + ${colors.blue} ___ __ __ __ ___ ${colors.green}/,-._.--~\'${colors.reset} + ${colors.blue} |\\ | |__ __ / ` / \\ |__) |__ ${colors.yellow}} {${colors.reset} + ${colors.blue} | \\| | \\__, \\__/ | \\ |___ ${colors.green}\\`-._,-`-,${colors.reset} + ${colors.green}`._,._,\'${colors.reset} + ${colors.purple} ${workflow.manifest.name} ${getWorkflowVersion()}${colors.reset} + ${dashedLine(monochrome_logs)} + """.stripIndent() + ) +} + +// +// Return dashed line +// +def dashedLine(monochrome_logs=true) { + def colors = logColours(monochrome_logs) as Map + return "-${colors.dim}----------------------------------------------------${colors.reset}-" +} + +// +// ANSII colours used for terminal logging +// +def logColours(monochrome_logs=true) { + def colorcodes = [:] as Map + + // Reset / Meta + colorcodes['reset'] = monochrome_logs ? '' : "\033[0m" + colorcodes['bold'] = monochrome_logs ? '' : "\033[1m" + colorcodes['dim'] = monochrome_logs ? '' : "\033[2m" + colorcodes['underlined'] = monochrome_logs ? '' : "\033[4m" + colorcodes['blink'] = monochrome_logs ? '' : "\033[5m" + colorcodes['reverse'] = monochrome_logs ? '' : "\033[7m" + colorcodes['hidden'] = monochrome_logs ? '' : "\033[8m" + + // Regular Colors + colorcodes['black'] = monochrome_logs ? '' : "\033[0;30m" + colorcodes['red'] = monochrome_logs ? '' : "\033[0;31m" + colorcodes['green'] = monochrome_logs ? '' : "\033[0;32m" + colorcodes['yellow'] = monochrome_logs ? '' : "\033[0;33m" + colorcodes['blue'] = monochrome_logs ? '' : "\033[0;34m" + colorcodes['purple'] = monochrome_logs ? '' : "\033[0;35m" + colorcodes['cyan'] = monochrome_logs ? '' : "\033[0;36m" + colorcodes['white'] = monochrome_logs ? '' : "\033[0;37m" + + // Bold + colorcodes['bblack'] = monochrome_logs ? '' : "\033[1;30m" + colorcodes['bred'] = monochrome_logs ? '' : "\033[1;31m" + colorcodes['bgreen'] = monochrome_logs ? '' : "\033[1;32m" + colorcodes['byellow'] = monochrome_logs ? '' : "\033[1;33m" + colorcodes['bblue'] = monochrome_logs ? '' : "\033[1;34m" + colorcodes['bpurple'] = monochrome_logs ? '' : "\033[1;35m" + colorcodes['bcyan'] = monochrome_logs ? '' : "\033[1;36m" + colorcodes['bwhite'] = monochrome_logs ? '' : "\033[1;37m" + + // Underline + colorcodes['ublack'] = monochrome_logs ? '' : "\033[4;30m" + colorcodes['ured'] = monochrome_logs ? '' : "\033[4;31m" + colorcodes['ugreen'] = monochrome_logs ? '' : "\033[4;32m" + colorcodes['uyellow'] = monochrome_logs ? '' : "\033[4;33m" + colorcodes['ublue'] = monochrome_logs ? '' : "\033[4;34m" + colorcodes['upurple'] = monochrome_logs ? '' : "\033[4;35m" + colorcodes['ucyan'] = monochrome_logs ? '' : "\033[4;36m" + colorcodes['uwhite'] = monochrome_logs ? '' : "\033[4;37m" + + // High Intensity + colorcodes['iblack'] = monochrome_logs ? '' : "\033[0;90m" + colorcodes['ired'] = monochrome_logs ? '' : "\033[0;91m" + colorcodes['igreen'] = monochrome_logs ? '' : "\033[0;92m" + colorcodes['iyellow'] = monochrome_logs ? '' : "\033[0;93m" + colorcodes['iblue'] = monochrome_logs ? '' : "\033[0;94m" + colorcodes['ipurple'] = monochrome_logs ? '' : "\033[0;95m" + colorcodes['icyan'] = monochrome_logs ? '' : "\033[0;96m" + colorcodes['iwhite'] = monochrome_logs ? '' : "\033[0;97m" + + // Bold High Intensity + colorcodes['biblack'] = monochrome_logs ? '' : "\033[1;90m" + colorcodes['bired'] = monochrome_logs ? '' : "\033[1;91m" + colorcodes['bigreen'] = monochrome_logs ? '' : "\033[1;92m" + colorcodes['biyellow'] = monochrome_logs ? '' : "\033[1;93m" + colorcodes['biblue'] = monochrome_logs ? '' : "\033[1;94m" + colorcodes['bipurple'] = monochrome_logs ? '' : "\033[1;95m" + colorcodes['bicyan'] = monochrome_logs ? '' : "\033[1;96m" + colorcodes['biwhite'] = monochrome_logs ? '' : "\033[1;97m" + + return colorcodes +} + +// +// Attach the multiqc report to email +// +def attachMultiqcReport(multiqc_report) { + def mqc_report = null + try { + if (workflow.success) { + mqc_report = multiqc_report.getVal() + if (mqc_report.getClass() == ArrayList && mqc_report.size() >= 1) { + if (mqc_report.size() > 1) { + log.warn("[${workflow.manifest.name}] Found multiple reports from process 'MULTIQC', will use only one") + } + mqc_report = mqc_report[0] + } + } + } + catch (Exception msg) { + log.debug(msg) + if (multiqc_report) { + log.warn("[${workflow.manifest.name}] Could not attach MultiQC report to summary email") + } + } + return mqc_report +} + +// +// Construct and send completion email +// +def completionEmail(summary_params, email, email_on_fail, plaintext_email, outdir, monochrome_logs=true, multiqc_report=null) { + + // Set up the e-mail variables + def subject = "[${workflow.manifest.name}] Successful: ${workflow.runName}" + if (!workflow.success) { + subject = "[${workflow.manifest.name}] FAILED: ${workflow.runName}" + } + + def summary = [:] + summary_params + .keySet() + .sort() + .each { group -> + summary << summary_params[group] + } + + def misc_fields = [:] + misc_fields['Date Started'] = workflow.start + misc_fields['Date Completed'] = workflow.complete + misc_fields['Pipeline script file path'] = workflow.scriptFile + misc_fields['Pipeline script hash ID'] = workflow.scriptId + if (workflow.repository) { + misc_fields['Pipeline repository Git URL'] = workflow.repository + } + if (workflow.commitId) { + misc_fields['Pipeline repository Git Commit'] = workflow.commitId + } + if (workflow.revision) { + misc_fields['Pipeline Git branch/tag'] = workflow.revision + } + misc_fields['Nextflow Version'] = workflow.nextflow.version + misc_fields['Nextflow Build'] = workflow.nextflow.build + misc_fields['Nextflow Compile Timestamp'] = workflow.nextflow.timestamp + + def email_fields = [:] + email_fields['version'] = getWorkflowVersion() + email_fields['runName'] = workflow.runName + email_fields['success'] = workflow.success + email_fields['dateComplete'] = workflow.complete + email_fields['duration'] = workflow.duration + email_fields['exitStatus'] = workflow.exitStatus + email_fields['errorMessage'] = (workflow.errorMessage ?: 'None') + email_fields['errorReport'] = (workflow.errorReport ?: 'None') + email_fields['commandLine'] = workflow.commandLine + email_fields['projectDir'] = workflow.projectDir + email_fields['summary'] = summary << misc_fields + + // On success try attach the multiqc report + def mqc_report = attachMultiqcReport(multiqc_report) + + // Check if we are only sending emails on failure + def email_address = email + if (!email && email_on_fail && !workflow.success) { + email_address = email_on_fail + } + + // Render the TXT template + def engine = new groovy.text.GStringTemplateEngine() + def tf = new File("${workflow.projectDir}/assets/email_template.txt") + def txt_template = engine.createTemplate(tf).make(email_fields) + def email_txt = txt_template.toString() + + // Render the HTML template + def hf = new File("${workflow.projectDir}/assets/email_template.html") + def html_template = engine.createTemplate(hf).make(email_fields) + def email_html = html_template.toString() + + // Render the sendmail template + def max_multiqc_email_size = (params.containsKey('max_multiqc_email_size') ? params.max_multiqc_email_size : 0) as nextflow.util.MemoryUnit + def smail_fields = [email: email_address, subject: subject, email_txt: email_txt, email_html: email_html, projectDir: "${workflow.projectDir}", mqcFile: mqc_report, mqcMaxSize: max_multiqc_email_size.toBytes()] + def sf = new File("${workflow.projectDir}/assets/sendmail_template.txt") + def sendmail_template = engine.createTemplate(sf).make(smail_fields) + def sendmail_html = sendmail_template.toString() + + // Send the HTML e-mail + def colors = logColours(monochrome_logs) as Map + if (email_address) { + try { + if (plaintext_email) { +new org.codehaus.groovy.GroovyException('Send plaintext e-mail, not HTML') } + // Try to send HTML e-mail using sendmail + def sendmail_tf = new File(workflow.launchDir.toString(), ".sendmail_tmp.html") + sendmail_tf.withWriter { w -> w << sendmail_html } + ['sendmail', '-t'].execute() << sendmail_html + log.info("-${colors.purple}[${workflow.manifest.name}]${colors.green} Sent summary e-mail to ${email_address} (sendmail)-") + } + catch (Exception msg) { + log.debug(msg) + log.debug("Trying with mail instead of sendmail") + // Catch failures and try with plaintext + def mail_cmd = ['mail', '-s', subject, '--content-type=text/html', email_address] + mail_cmd.execute() << email_html + log.info("-${colors.purple}[${workflow.manifest.name}]${colors.green} Sent summary e-mail to ${email_address} (mail)-") + } + } + + // Write summary e-mail HTML to a file + def output_hf = new File(workflow.launchDir.toString(), ".pipeline_report.html") + output_hf.withWriter { w -> w << email_html } + nextflow.extension.FilesEx.copyTo(output_hf.toPath(), "${outdir}/pipeline_info/pipeline_report.html") + output_hf.delete() + + // Write summary e-mail TXT to a file + def output_tf = new File(workflow.launchDir.toString(), ".pipeline_report.txt") + output_tf.withWriter { w -> w << email_txt } + nextflow.extension.FilesEx.copyTo(output_tf.toPath(), "${outdir}/pipeline_info/pipeline_report.txt") + output_tf.delete() +} + +// +// Print pipeline summary on completion +// +def completionSummary(monochrome_logs=true) { + def colors = logColours(monochrome_logs) as Map + if (workflow.success) { + if (workflow.stats.ignoredCount == 0) { + log.info("-${colors.purple}[${workflow.manifest.name}]${colors.green} Pipeline completed successfully${colors.reset}-") + } + else { + log.info("-${colors.purple}[${workflow.manifest.name}]${colors.yellow} Pipeline completed successfully, but with errored process(es) ${colors.reset}-") + } + } + else { + log.info("-${colors.purple}[${workflow.manifest.name}]${colors.red} Pipeline completed with errors${colors.reset}-") + } +} + +// +// Construct and send a notification to a web server as JSON e.g. Microsoft Teams and Slack +// +def imNotification(summary_params, hook_url) { + def summary = [:] + summary_params + .keySet() + .sort() + .each { group -> + summary << summary_params[group] + } + + def misc_fields = [:] + misc_fields['start'] = workflow.start + misc_fields['complete'] = workflow.complete + misc_fields['scriptfile'] = workflow.scriptFile + misc_fields['scriptid'] = workflow.scriptId + if (workflow.repository) { + misc_fields['repository'] = workflow.repository + } + if (workflow.commitId) { + misc_fields['commitid'] = workflow.commitId + } + if (workflow.revision) { + misc_fields['revision'] = workflow.revision + } + misc_fields['nxf_version'] = workflow.nextflow.version + misc_fields['nxf_build'] = workflow.nextflow.build + misc_fields['nxf_timestamp'] = workflow.nextflow.timestamp + + def msg_fields = [:] + msg_fields['version'] = getWorkflowVersion() + msg_fields['runName'] = workflow.runName + msg_fields['success'] = workflow.success + msg_fields['dateComplete'] = workflow.complete + msg_fields['duration'] = workflow.duration + msg_fields['exitStatus'] = workflow.exitStatus + msg_fields['errorMessage'] = (workflow.errorMessage ?: 'None') + msg_fields['errorReport'] = (workflow.errorReport ?: 'None') + msg_fields['commandLine'] = workflow.commandLine.replaceFirst(/ +--hook_url +[^ ]+/, "") + msg_fields['projectDir'] = workflow.projectDir + msg_fields['summary'] = summary << misc_fields + + // Render the JSON template + def engine = new groovy.text.GStringTemplateEngine() + // Different JSON depending on the service provider + // Defaults to "Adaptive Cards" (https://adaptivecards.io), except Slack which has its own format + def json_path = hook_url.contains("hooks.slack.com") ? "slackreport.json" : "adaptivecard.json" + def hf = new File("${workflow.projectDir}/assets/${json_path}") + def json_template = engine.createTemplate(hf).make(msg_fields) + def json_message = json_template.toString() + + // POST + def post = new URL(hook_url).openConnection() + post.setRequestMethod("POST") + post.setDoOutput(true) + post.setRequestProperty("Content-Type", "application/json") + post.getOutputStream().write(json_message.getBytes("UTF-8")) + def postRC = post.getResponseCode() + if (!postRC.equals(200)) { + log.warn(post.getErrorStream().getText()) + } +} diff --git a/subworkflows/nf-core/utils_nfcore_pipeline/meta.yml b/subworkflows/nf-core/utils_nfcore_pipeline/meta.yml new file mode 100644 index 0000000..d08d243 --- /dev/null +++ b/subworkflows/nf-core/utils_nfcore_pipeline/meta.yml @@ -0,0 +1,24 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/subworkflows/yaml-schema.json +name: "UTILS_NFCORE_PIPELINE" +description: Subworkflow with utility functions specific to the nf-core pipeline template +keywords: + - utility + - pipeline + - initialise + - version +components: [] +input: + - nextflow_cli_args: + type: list + description: | + Nextflow CLI positional arguments +output: + - success: + type: boolean + description: | + Dummy output to indicate success +authors: + - "@adamrtalbot" +maintainers: + - "@adamrtalbot" + - "@maxulysse" diff --git a/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test b/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test new file mode 100644 index 0000000..1dc317f --- /dev/null +++ b/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test @@ -0,0 +1,134 @@ + +nextflow_function { + + name "Test Functions" + script "../main.nf" + config "subworkflows/nf-core/utils_nfcore_pipeline/tests/nextflow.config" + tag "subworkflows" + tag "subworkflows_nfcore" + tag "utils_nfcore_pipeline" + tag "subworkflows/utils_nfcore_pipeline" + + test("Test Function checkConfigProvided") { + + function "checkConfigProvided" + + then { + assertAll( + { assert function.success }, + { assert snapshot(function.result).match() } + ) + } + } + + test("Test Function checkProfileProvided") { + + function "checkProfileProvided" + + when { + function { + """ + input[0] = [] + """ + } + } + + then { + assertAll( + { assert function.success }, + { assert snapshot(function.result).match() } + ) + } + } + + test("Test Function workflowCitation") { + + function "workflowCitation" + + then { + assertAll( + { assert function.success }, + { assert snapshot(function.result).match() } + ) + } + } + + test("Test Function nfCoreLogo") { + + function "nfCoreLogo" + + when { + function { + """ + input[0] = false + """ + } + } + + then { + assertAll( + { assert function.success }, + { assert snapshot(function.result).match() } + ) + } + } + + test("Test Function dashedLine") { + + function "dashedLine" + + when { + function { + """ + input[0] = false + """ + } + } + + then { + assertAll( + { assert function.success }, + { assert snapshot(function.result).match() } + ) + } + } + + test("Test Function without logColours") { + + function "logColours" + + when { + function { + """ + input[0] = true + """ + } + } + + then { + assertAll( + { assert function.success }, + { assert snapshot(function.result).match() } + ) + } + } + + test("Test Function with logColours") { + function "logColours" + + when { + function { + """ + input[0] = false + """ + } + } + + then { + assertAll( + { assert function.success }, + { assert snapshot(function.result).match() } + ) + } + } +} diff --git a/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test.snap b/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test.snap new file mode 100644 index 0000000..1037232 --- /dev/null +++ b/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test.snap @@ -0,0 +1,166 @@ +{ + "Test Function checkProfileProvided": { + "content": null, + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:03:03.360873" + }, + "Test Function checkConfigProvided": { + "content": [ + true + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:02:59.729647" + }, + "Test Function nfCoreLogo": { + "content": [ + "\n\n-\u001b[2m----------------------------------------------------\u001b[0m-\n \u001b[0;32m,--.\u001b[0;30m/\u001b[0;32m,-.\u001b[0m\n\u001b[0;34m ___ __ __ __ ___ \u001b[0;32m/,-._.--~'\u001b[0m\n\u001b[0;34m |\\ | |__ __ / ` / \\ |__) |__ \u001b[0;33m} {\u001b[0m\n\u001b[0;34m | \\| | \\__, \\__/ | \\ |___ \u001b[0;32m\\`-._,-`-,\u001b[0m\n \u001b[0;32m`._,._,'\u001b[0m\n\u001b[0;35m nextflow_workflow v9.9.9\u001b[0m\n-\u001b[2m----------------------------------------------------\u001b[0m-\n" + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:03:10.562934" + }, + "Test Function workflowCitation": { + "content": [ + "If you use nextflow_workflow for your analysis please cite:\n\n* The pipeline\n https://doi.org/10.5281/zenodo.5070524\n\n* The nf-core framework\n https://doi.org/10.1038/s41587-020-0439-x\n\n* Software dependencies\n https://github.com/nextflow_workflow/blob/master/CITATIONS.md" + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:03:07.019761" + }, + "Test Function without logColours": { + "content": [ + { + "reset": "", + "bold": "", + "dim": "", + "underlined": "", + "blink": "", + "reverse": "", + "hidden": "", + "black": "", + "red": "", + "green": "", + "yellow": "", + "blue": "", + "purple": "", + "cyan": "", + "white": "", + "bblack": "", + "bred": "", + "bgreen": "", + "byellow": "", + "bblue": "", + "bpurple": "", + "bcyan": "", + "bwhite": "", + "ublack": "", + "ured": "", + "ugreen": "", + "uyellow": "", + "ublue": "", + "upurple": "", + "ucyan": "", + "uwhite": "", + "iblack": "", + "ired": "", + "igreen": "", + "iyellow": "", + "iblue": "", + "ipurple": "", + "icyan": "", + "iwhite": "", + "biblack": "", + "bired": "", + "bigreen": "", + "biyellow": "", + "biblue": "", + "bipurple": "", + "bicyan": "", + "biwhite": "" + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:03:17.969323" + }, + "Test Function dashedLine": { + "content": [ + "-\u001b[2m----------------------------------------------------\u001b[0m-" + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:03:14.366181" + }, + "Test Function with logColours": { + "content": [ + { + "reset": "\u001b[0m", + "bold": "\u001b[1m", + "dim": "\u001b[2m", + "underlined": "\u001b[4m", + "blink": "\u001b[5m", + "reverse": "\u001b[7m", + "hidden": "\u001b[8m", + "black": "\u001b[0;30m", + "red": "\u001b[0;31m", + "green": "\u001b[0;32m", + "yellow": "\u001b[0;33m", + "blue": "\u001b[0;34m", + "purple": "\u001b[0;35m", + "cyan": "\u001b[0;36m", + "white": "\u001b[0;37m", + "bblack": "\u001b[1;30m", + "bred": "\u001b[1;31m", + "bgreen": "\u001b[1;32m", + "byellow": "\u001b[1;33m", + "bblue": "\u001b[1;34m", + "bpurple": "\u001b[1;35m", + "bcyan": "\u001b[1;36m", + "bwhite": "\u001b[1;37m", + "ublack": "\u001b[4;30m", + "ured": "\u001b[4;31m", + "ugreen": "\u001b[4;32m", + "uyellow": "\u001b[4;33m", + "ublue": "\u001b[4;34m", + "upurple": "\u001b[4;35m", + "ucyan": "\u001b[4;36m", + "uwhite": "\u001b[4;37m", + "iblack": "\u001b[0;90m", + "ired": "\u001b[0;91m", + "igreen": "\u001b[0;92m", + "iyellow": "\u001b[0;93m", + "iblue": "\u001b[0;94m", + "ipurple": "\u001b[0;95m", + "icyan": "\u001b[0;96m", + "iwhite": "\u001b[0;97m", + "biblack": "\u001b[1;90m", + "bired": "\u001b[1;91m", + "bigreen": "\u001b[1;92m", + "biyellow": "\u001b[1;93m", + "biblue": "\u001b[1;94m", + "bipurple": "\u001b[1;95m", + "bicyan": "\u001b[1;96m", + "biwhite": "\u001b[1;97m" + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:03:21.714424" + } +} \ No newline at end of file diff --git a/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.workflow.nf.test b/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.workflow.nf.test new file mode 100644 index 0000000..8940d32 --- /dev/null +++ b/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.workflow.nf.test @@ -0,0 +1,29 @@ +nextflow_workflow { + + name "Test Workflow UTILS_NFCORE_PIPELINE" + script "../main.nf" + config "subworkflows/nf-core/utils_nfcore_pipeline/tests/nextflow.config" + workflow "UTILS_NFCORE_PIPELINE" + tag "subworkflows" + tag "subworkflows_nfcore" + tag "utils_nfcore_pipeline" + tag "subworkflows/utils_nfcore_pipeline" + + test("Should run without failures") { + + when { + workflow { + """ + input[0] = [] + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert snapshot(workflow.out).match() } + ) + } + } +} diff --git a/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.workflow.nf.test.snap b/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.workflow.nf.test.snap new file mode 100644 index 0000000..859d103 --- /dev/null +++ b/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.workflow.nf.test.snap @@ -0,0 +1,19 @@ +{ + "Should run without failures": { + "content": [ + { + "0": [ + true + ], + "valid_config": [ + true + ] + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:03:25.726491" + } +} \ No newline at end of file diff --git a/subworkflows/nf-core/utils_nfcore_pipeline/tests/nextflow.config b/subworkflows/nf-core/utils_nfcore_pipeline/tests/nextflow.config new file mode 100644 index 0000000..d0a926b --- /dev/null +++ b/subworkflows/nf-core/utils_nfcore_pipeline/tests/nextflow.config @@ -0,0 +1,9 @@ +manifest { + name = 'nextflow_workflow' + author = """nf-core""" + homePage = 'https://127.0.0.1' + description = """Dummy pipeline""" + nextflowVersion = '!>=23.04.0' + version = '9.9.9' + doi = 'https://doi.org/10.5281/zenodo.5070524' +} diff --git a/subworkflows/nf-core/utils_nfcore_pipeline/tests/tags.yml b/subworkflows/nf-core/utils_nfcore_pipeline/tests/tags.yml new file mode 100644 index 0000000..ac8523c --- /dev/null +++ b/subworkflows/nf-core/utils_nfcore_pipeline/tests/tags.yml @@ -0,0 +1,2 @@ +subworkflows/utils_nfcore_pipeline: + - subworkflows/nf-core/utils_nfcore_pipeline/** diff --git a/subworkflows/nf-core/utils_nfschema_plugin/main.nf b/subworkflows/nf-core/utils_nfschema_plugin/main.nf new file mode 100644 index 0000000..4994303 --- /dev/null +++ b/subworkflows/nf-core/utils_nfschema_plugin/main.nf @@ -0,0 +1,46 @@ +// +// Subworkflow that uses the nf-schema plugin to validate parameters and render the parameter summary +// + +include { paramsSummaryLog } from 'plugin/nf-schema' +include { validateParameters } from 'plugin/nf-schema' + +workflow UTILS_NFSCHEMA_PLUGIN { + + take: + input_workflow // workflow: the workflow object used by nf-schema to get metadata from the workflow + validate_params // boolean: validate the parameters + parameters_schema // string: path to the parameters JSON schema. + // this has to be the same as the schema given to `validation.parametersSchema` + // when this input is empty it will automatically use the configured schema or + // "${projectDir}/nextflow_schema.json" as default. This input should not be empty + // for meta pipelines + + main: + + // + // Print parameter summary to stdout. This will display the parameters + // that differ from the default given in the JSON schema + // + if(parameters_schema) { + log.info paramsSummaryLog(input_workflow, parameters_schema:parameters_schema) + } else { + log.info paramsSummaryLog(input_workflow) + } + + // + // Validate the parameters using nextflow_schema.json or the schema + // given via the validation.parametersSchema configuration option + // + if(validate_params) { + if(parameters_schema) { + validateParameters(parameters_schema:parameters_schema) + } else { + validateParameters() + } + } + + emit: + dummy_emit = true +} + diff --git a/subworkflows/nf-core/utils_nfschema_plugin/meta.yml b/subworkflows/nf-core/utils_nfschema_plugin/meta.yml new file mode 100644 index 0000000..f7d9f02 --- /dev/null +++ b/subworkflows/nf-core/utils_nfschema_plugin/meta.yml @@ -0,0 +1,35 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/subworkflows/yaml-schema.json +name: "utils_nfschema_plugin" +description: Run nf-schema to validate parameters and create a summary of changed parameters +keywords: + - validation + - JSON schema + - plugin + - parameters + - summary +components: [] +input: + - input_workflow: + type: object + description: | + The workflow object of the used pipeline. + This object contains meta data used to create the params summary log + - validate_params: + type: boolean + description: Validate the parameters and error if invalid. + - parameters_schema: + type: string + description: | + Path to the parameters JSON schema. + This has to be the same as the schema given to the `validation.parametersSchema` config + option. When this input is empty it will automatically use the configured schema or + "${projectDir}/nextflow_schema.json" as default. The schema should not be given in this way + for meta pipelines. +output: + - dummy_emit: + type: boolean + description: Dummy emit to make nf-core subworkflows lint happy +authors: + - "@nvnieuwk" +maintainers: + - "@nvnieuwk" diff --git a/subworkflows/nf-core/utils_nfschema_plugin/tests/main.nf.test b/subworkflows/nf-core/utils_nfschema_plugin/tests/main.nf.test new file mode 100644 index 0000000..8fb3016 --- /dev/null +++ b/subworkflows/nf-core/utils_nfschema_plugin/tests/main.nf.test @@ -0,0 +1,117 @@ +nextflow_workflow { + + name "Test Subworkflow UTILS_NFSCHEMA_PLUGIN" + script "../main.nf" + workflow "UTILS_NFSCHEMA_PLUGIN" + + tag "subworkflows" + tag "subworkflows_nfcore" + tag "subworkflows/utils_nfschema_plugin" + tag "plugin/nf-schema" + + config "./nextflow.config" + + test("Should run nothing") { + + when { + + params { + test_data = '' + } + + workflow { + """ + validate_params = false + input[0] = workflow + input[1] = validate_params + input[2] = "" + """ + } + } + + then { + assertAll( + { assert workflow.success } + ) + } + } + + test("Should validate params") { + + when { + + params { + test_data = '' + outdir = null + } + + workflow { + """ + validate_params = true + input[0] = workflow + input[1] = validate_params + input[2] = "" + """ + } + } + + then { + assertAll( + { assert workflow.failed }, + { assert workflow.stdout.any { it.contains('ERROR ~ Validation of pipeline parameters failed!') } } + ) + } + } + + test("Should run nothing - custom schema") { + + when { + + params { + test_data = '' + } + + workflow { + """ + validate_params = false + input[0] = workflow + input[1] = validate_params + input[2] = "${projectDir}/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow_schema.json" + """ + } + } + + then { + assertAll( + { assert workflow.success } + ) + } + } + + test("Should validate params - custom schema") { + + when { + + params { + test_data = '' + outdir = null + } + + workflow { + """ + validate_params = true + input[0] = workflow + input[1] = validate_params + input[2] = "${projectDir}/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow_schema.json" + """ + } + } + + then { + assertAll( + { assert workflow.failed }, + { assert workflow.stdout.any { it.contains('ERROR ~ Validation of pipeline parameters failed!') } } + ) + } + } +} diff --git a/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow.config b/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow.config new file mode 100644 index 0000000..0907ac5 --- /dev/null +++ b/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow.config @@ -0,0 +1,8 @@ +plugins { + id "nf-schema@2.1.0" +} + +validation { + parametersSchema = "${projectDir}/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow_schema.json" + monochromeLogs = true +} \ No newline at end of file diff --git a/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow_schema.json b/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow_schema.json new file mode 100644 index 0000000..331e0d2 --- /dev/null +++ b/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow_schema.json @@ -0,0 +1,96 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://raw.githubusercontent.com/./master/nextflow_schema.json", + "title": ". pipeline parameters", + "description": "", + "type": "object", + "$defs": { + "input_output_options": { + "title": "Input/output options", + "type": "object", + "fa_icon": "fas fa-terminal", + "description": "Define where the pipeline should find input data and save output data.", + "required": ["outdir"], + "properties": { + "validate_params": { + "type": "boolean", + "description": "Validate parameters?", + "default": true, + "hidden": true + }, + "outdir": { + "type": "string", + "format": "directory-path", + "description": "The output directory where the results will be saved. You have to use absolute paths to storage on Cloud infrastructure.", + "fa_icon": "fas fa-folder-open" + }, + "test_data_base": { + "type": "string", + "default": "https://raw.githubusercontent.com/nf-core/test-datasets/modules", + "description": "Base for test data directory", + "hidden": true + }, + "test_data": { + "type": "string", + "description": "Fake test data param", + "hidden": true + } + } + }, + "generic_options": { + "title": "Generic options", + "type": "object", + "fa_icon": "fas fa-file-import", + "description": "Less common options for the pipeline, typically set in a config file.", + "help_text": "These options are common to all nf-core pipelines and allow you to customise some of the core preferences for how the pipeline runs.\n\nTypically these options would be set in a Nextflow config file loaded for all pipeline runs, such as `~/.nextflow/config`.", + "properties": { + "help": { + "type": "boolean", + "description": "Display help text.", + "fa_icon": "fas fa-question-circle", + "hidden": true + }, + "version": { + "type": "boolean", + "description": "Display version and exit.", + "fa_icon": "fas fa-question-circle", + "hidden": true + }, + "logo": { + "type": "boolean", + "default": true, + "description": "Display nf-core logo in console output.", + "fa_icon": "fas fa-image", + "hidden": true + }, + "singularity_pull_docker_container": { + "type": "boolean", + "description": "Pull Singularity container from Docker?", + "hidden": true + }, + "publish_dir_mode": { + "type": "string", + "default": "copy", + "description": "Method used to save pipeline results to output directory.", + "help_text": "The Nextflow `publishDir` option specifies which intermediate files should be saved to the output directory. This option tells the pipeline what method should be used to move these files. See [Nextflow docs](https://www.nextflow.io/docs/latest/process.html#publishdir) for details.", + "fa_icon": "fas fa-copy", + "enum": ["symlink", "rellink", "link", "copy", "copyNoFollow", "move"], + "hidden": true + }, + "monochrome_logs": { + "type": "boolean", + "description": "Use monochrome_logs", + "hidden": true + } + } + } + }, + "allOf": [ + { + "$ref": "#/$defs/input_output_options" + }, + { + "$ref": "#/$defs/generic_options" + } + ] +} diff --git a/tests/check_test_results.nf.test b/tests/check_test_results.nf.test new file mode 100644 index 0000000..27c41a6 --- /dev/null +++ b/tests/check_test_results.nf.test @@ -0,0 +1,54 @@ +// This is a pipeline level-test, not a module test, even though the main script is a module. +// The entire workflow is executed in the setup scope using the same config as the test profile. +// Afterwards, a single module is executed in the when scope to ensure that the pipeline generated correct results. + +nextflow_process { + + name "Check the correctness of the pipeline results using the test profile" + script "../modules/local/check_results/main.nf" + process "CHECK_RESULTS" + + test("Checking pipeline results for: -profile test") { + + setup { + run("NFCORE_RANGELAND") { + script "../main.nf" + workflow {} + } + + } + + when { + params { + // Reference data + woody_change_ref = 'https://github.com/nf-core/test-datasets/raw/rangeland/reference/woody_cover_chg_ref.tif' + woody_yoc_ref = 'https://github.com/nf-core/test-datasets/raw/rangeland/reference/woody_cover_yoc_ref.tif' + + herbaceous_change_ref = 'https://github.com/nf-core/test-datasets/raw/rangeland/reference/herbaceous_cover_chg_ref.tif' + herbaceous_yoc_ref = 'https://github.com/nf-core/test-datasets/raw/rangeland/reference/herbaceous_cover_yoc_ref.tif' + + peak_change_ref = 'https://github.com/nf-core/test-datasets/raw/rangeland/reference/peak_chg_ref.tif' + peak_yoc_ref = 'https://github.com/nf-core/test-datasets/raw/rangeland/reference/peak_yoc_ref.tif' + } + + process { + """ + // inputs for CHECK_RESULTS + input[0] = NFCORE_RANGELAND.out.mosaic.map{ it[1] }.flatten().buffer( size: Integer.MAX_VALUE, remainder: true ) + input[1] = file( params.woody_change_ref ) + input[2] = file( params.woody_yoc_ref ) + input[3] = file( params.herbaceous_change_ref ) + input[4] = file( params.herbaceous_yoc_ref ) + input[5] = file( params.peak_change_ref ) + input[6] = file( params.peak_yoc_ref ) + """ + } + } + + then { + assert process.success + } + + } + +} diff --git a/tests/default.nf.test b/tests/default.nf.test new file mode 100644 index 0000000..31f3c3e --- /dev/null +++ b/tests/default.nf.test @@ -0,0 +1,26 @@ +nextflow_pipeline { + + name "Test pipeline with default test profile settings " + script "../main.nf" + + test("Params: test profile default") { + + when { + params { + outdir = "$outputDir" + } + } + + then { + def stable_name = getAllFilesFromDir(params.outdir, relative: true, includeDir: true, ignore: ['pipeline_info/*.{html,json,txt}']) + assert workflow.success + assert snapshot( + workflow.trace.succeeded().size(), + stable_name, + ).match() + + } + + } + +} diff --git a/tests/default.nf.test.snap b/tests/default.nf.test.snap new file mode 100644 index 0000000..d652eae --- /dev/null +++ b/tests/default.nf.test.snap @@ -0,0 +1,1274 @@ +{ + "Params: test profile default": { + "content": [ + 280, + [ + "higher-level", + "higher-level/X0109_Y0102", + "higher-level/X0109_Y0102/param_files", + "higher-level/X0109_Y0102/param_files/trend_X0109_Y0102.prm", + "multiqc", + "multiqc/multiqc_data", + "multiqc/multiqc_data/multiqc.log", + "multiqc/multiqc_data/multiqc_citations.txt", + "multiqc/multiqc_data/multiqc_data.json", + "multiqc/multiqc_data/multiqc_software_versions.txt", + "multiqc/multiqc_data/multiqc_sources.txt", + "multiqc/multiqc_report.html", + "pipeline_info", + "pipeline_info/nf_core_pipeline_software_mqc_versions.yml", + "preparation", + "preparation/mask", + "preparation/mask/X0103_Y0101", + "preparation/mask/X0103_Y0101/aoi.tif", + "preparation/mask/X0103_Y0102", + "preparation/mask/X0103_Y0102/aoi.tif", + "preparation/mask/X0103_Y0103", + "preparation/mask/X0103_Y0103/aoi.tif", + "preparation/mask/X0103_Y0104", + "preparation/mask/X0103_Y0104/aoi.tif", + "preparation/mask/X0103_Y0105", + "preparation/mask/X0103_Y0105/aoi.tif", + "preparation/mask/X0104_Y0101", + "preparation/mask/X0104_Y0101/aoi.tif", + "preparation/mask/X0104_Y0102", + "preparation/mask/X0104_Y0102/aoi.tif", + "preparation/mask/X0104_Y0103", + "preparation/mask/X0104_Y0103/aoi.tif", + "preparation/mask/X0104_Y0104", + "preparation/mask/X0104_Y0104/aoi.tif", + "preparation/mask/X0104_Y0105", + "preparation/mask/X0104_Y0105/aoi.tif", + "preparation/mask/X0105_Y0101", + "preparation/mask/X0105_Y0101/aoi.tif", + "preparation/mask/X0105_Y0102", + "preparation/mask/X0105_Y0102/aoi.tif", + "preparation/mask/X0105_Y0103", + "preparation/mask/X0105_Y0103/aoi.tif", + "preparation/mask/X0105_Y0104", + "preparation/mask/X0105_Y0104/aoi.tif", + "preparation/mask/X0105_Y0105", + "preparation/mask/X0105_Y0105/aoi.tif", + "preparation/mask/X0106_Y0101", + "preparation/mask/X0106_Y0101/aoi.tif", + "preparation/mask/X0106_Y0102", + "preparation/mask/X0106_Y0102/aoi.tif", + "preparation/mask/X0106_Y0103", + "preparation/mask/X0106_Y0103/aoi.tif", + "preparation/mask/X0106_Y0104", + "preparation/mask/X0106_Y0104/aoi.tif", + "preparation/mask/X0106_Y0105", + "preparation/mask/X0106_Y0105/aoi.tif", + "preparation/mask/X0107_Y0101", + "preparation/mask/X0107_Y0101/aoi.tif", + "preparation/mask/X0107_Y0102", + "preparation/mask/X0107_Y0102/aoi.tif", + "preparation/mask/X0107_Y0103", + "preparation/mask/X0107_Y0103/aoi.tif", + "preparation/mask/X0107_Y0104", + "preparation/mask/X0107_Y0104/aoi.tif", + "preparation/mask/X0107_Y0105", + "preparation/mask/X0107_Y0105/aoi.tif", + "preparation/mask/X0108_Y0101", + "preparation/mask/X0108_Y0101/aoi.tif", + "preparation/mask/X0108_Y0102", + "preparation/mask/X0108_Y0102/aoi.tif", + "preparation/mask/X0108_Y0103", + "preparation/mask/X0108_Y0103/aoi.tif", + "preparation/mask/X0108_Y0104", + "preparation/mask/X0108_Y0104/aoi.tif", + "preparation/mask/X0108_Y0105", + "preparation/mask/X0108_Y0105/aoi.tif", + "preparation/mask/X0109_Y0101", + "preparation/mask/X0109_Y0101/aoi.tif", + "preparation/mask/X0109_Y0102", + "preparation/mask/X0109_Y0102/aoi.tif", + "preparation/mask/X0109_Y0103", + "preparation/mask/X0109_Y0103/aoi.tif", + "preparation/mask/X0109_Y0104", + "preparation/mask/X0109_Y0104/aoi.tif", + "preparation/mask/X0109_Y0105", + "preparation/mask/X0109_Y0105/aoi.tif", + "preparation/mask/X0110_Y0101", + "preparation/mask/X0110_Y0101/aoi.tif", + "preparation/mask/X0110_Y0102", + "preparation/mask/X0110_Y0102/aoi.tif", + "preparation/mask/X0110_Y0103", + "preparation/mask/X0110_Y0103/aoi.tif", + "preparation/mask/X0110_Y0104", + "preparation/mask/X0110_Y0104/aoi.tif", + "preparation/mask/X0110_Y0105", + "preparation/mask/X0110_Y0105/aoi.tif", + "preparation/mask/X0111_Y0101", + "preparation/mask/X0111_Y0101/aoi.tif", + "preparation/mask/X0111_Y0102", + "preparation/mask/X0111_Y0102/aoi.tif", + "preparation/mask/X0111_Y0103", + "preparation/mask/X0111_Y0103/aoi.tif", + "preparation/mask/X0111_Y0104", + "preparation/mask/X0111_Y0104/aoi.tif", + "preparation/mask/X0111_Y0105", + "preparation/mask/X0111_Y0105/aoi.tif", + "preparation/tile_allow.txt", + "preprocess", + "preprocess/LT04_L1TP_181036_19880130_20200917_02_T1", + "preprocess/LT04_L1TP_181036_19880130_20200917_02_T1/logs", + "preprocess/LT04_L1TP_181036_19880130_20200917_02_T1/logs/level2_log", + "preprocess/LT04_L1TP_181036_19880130_20200917_02_T1/logs/level2_log/LT04_L1TP_181036_19880130_20200917_02_T1.log", + "preprocess/LT04_L1TP_181036_19880130_20200917_02_T1/param_files", + "preprocess/LT04_L1TP_181036_19880130_20200917_02_T1/param_files/LT04_L1TP_181036_19880130_20200917_02_T1.prm", + "preprocess/LT04_L1TP_181036_19881129_20200917_02_T1", + "preprocess/LT04_L1TP_181036_19881129_20200917_02_T1/logs", + "preprocess/LT04_L1TP_181036_19881129_20200917_02_T1/logs/level2_log", + "preprocess/LT04_L1TP_181036_19881129_20200917_02_T1/logs/level2_log/LT04_L1TP_181036_19881129_20200917_02_T1.log", + "preprocess/LT04_L1TP_181036_19881129_20200917_02_T1/param_files", + "preprocess/LT04_L1TP_181036_19881129_20200917_02_T1/param_files/LT04_L1TP_181036_19881129_20200917_02_T1.prm", + "preprocess/LT04_L1TP_181036_19881231_20200916_02_T1", + "preprocess/LT04_L1TP_181036_19881231_20200916_02_T1/logs", + "preprocess/LT04_L1TP_181036_19881231_20200916_02_T1/logs/level2_log", + "preprocess/LT04_L1TP_181036_19881231_20200916_02_T1/logs/level2_log/LT04_L1TP_181036_19881231_20200916_02_T1.log", + "preprocess/LT04_L1TP_181036_19881231_20200916_02_T1/param_files", + "preprocess/LT04_L1TP_181036_19881231_20200916_02_T1/param_files/LT04_L1TP_181036_19881231_20200916_02_T1.prm", + "preprocess/LT04_L1TP_181036_19891202_20200916_02_T1", + "preprocess/LT04_L1TP_181036_19891202_20200916_02_T1/logs", + "preprocess/LT04_L1TP_181036_19891202_20200916_02_T1/logs/level2_log", + "preprocess/LT04_L1TP_181036_19891202_20200916_02_T1/logs/level2_log/LT04_L1TP_181036_19891202_20200916_02_T1.log", + "preprocess/LT04_L1TP_181036_19891202_20200916_02_T1/param_files", + "preprocess/LT04_L1TP_181036_19891202_20200916_02_T1/param_files/LT04_L1TP_181036_19891202_20200916_02_T1.prm", + "preprocess/LT04_L1TP_181036_19891218_20200916_02_T1", + "preprocess/LT04_L1TP_181036_19891218_20200916_02_T1/logs", + "preprocess/LT04_L1TP_181036_19891218_20200916_02_T1/logs/level2_log", + "preprocess/LT04_L1TP_181036_19891218_20200916_02_T1/logs/level2_log/LT04_L1TP_181036_19891218_20200916_02_T1.log", + "preprocess/LT04_L1TP_181036_19891218_20200916_02_T1/param_files", + "preprocess/LT04_L1TP_181036_19891218_20200916_02_T1/param_files/LT04_L1TP_181036_19891218_20200916_02_T1.prm", + "preprocess/LT05_L1TP_181036_19870119_20201014_02_T1", + "preprocess/LT05_L1TP_181036_19870119_20201014_02_T1/logs", + "preprocess/LT05_L1TP_181036_19870119_20201014_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19870119_20201014_02_T1/logs/level2_log/LT05_L1TP_181036_19870119_20201014_02_T1.log", + "preprocess/LT05_L1TP_181036_19870119_20201014_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19870119_20201014_02_T1/param_files/LT05_L1TP_181036_19870119_20201014_02_T1.prm", + "preprocess/LT05_L1TP_181036_19870204_20201014_02_T1", + "preprocess/LT05_L1TP_181036_19870204_20201014_02_T1/logs", + "preprocess/LT05_L1TP_181036_19870204_20201014_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19870204_20201014_02_T1/logs/level2_log/LT05_L1TP_181036_19870204_20201014_02_T1.log", + "preprocess/LT05_L1TP_181036_19870204_20201014_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19870204_20201014_02_T1/param_files/LT05_L1TP_181036_19870204_20201014_02_T1.prm", + "preprocess/LT05_L1TP_181036_19870324_20211112_02_T1", + "preprocess/LT05_L1TP_181036_19870324_20211112_02_T1/logs", + "preprocess/LT05_L1TP_181036_19870324_20211112_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19870324_20211112_02_T1/logs/level2_log/LT05_L1TP_181036_19870324_20211112_02_T1.log", + "preprocess/LT05_L1TP_181036_19870324_20211112_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19870324_20211112_02_T1/param_files/LT05_L1TP_181036_19870324_20211112_02_T1.prm", + "preprocess/LT05_L1TP_181036_19870409_20201014_02_T1", + "preprocess/LT05_L1TP_181036_19870409_20201014_02_T1/logs", + "preprocess/LT05_L1TP_181036_19870409_20201014_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19870409_20201014_02_T1/logs/level2_log/LT05_L1TP_181036_19870409_20201014_02_T1.log", + "preprocess/LT05_L1TP_181036_19870409_20201014_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19870409_20201014_02_T1/param_files/LT05_L1TP_181036_19870409_20201014_02_T1.prm", + "preprocess/LT05_L1TP_181036_19870425_20201014_02_T1", + "preprocess/LT05_L1TP_181036_19870425_20201014_02_T1/logs", + "preprocess/LT05_L1TP_181036_19870425_20201014_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19870425_20201014_02_T1/logs/level2_log/LT05_L1TP_181036_19870425_20201014_02_T1.log", + "preprocess/LT05_L1TP_181036_19870425_20201014_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19870425_20201014_02_T1/param_files/LT05_L1TP_181036_19870425_20201014_02_T1.prm", + "preprocess/LT05_L1TP_181036_19870511_20201014_02_T1", + "preprocess/LT05_L1TP_181036_19870511_20201014_02_T1/logs", + "preprocess/LT05_L1TP_181036_19870511_20201014_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19870511_20201014_02_T1/logs/level2_log/LT05_L1TP_181036_19870511_20201014_02_T1.log", + "preprocess/LT05_L1TP_181036_19870511_20201014_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19870511_20201014_02_T1/param_files/LT05_L1TP_181036_19870511_20201014_02_T1.prm", + "preprocess/LT05_L1TP_181036_19870527_20211113_02_T1", + "preprocess/LT05_L1TP_181036_19870527_20211113_02_T1/logs", + "preprocess/LT05_L1TP_181036_19870527_20211113_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19870527_20211113_02_T1/logs/level2_log/LT05_L1TP_181036_19870527_20211113_02_T1.log", + "preprocess/LT05_L1TP_181036_19870527_20211113_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19870527_20211113_02_T1/param_files/LT05_L1TP_181036_19870527_20211113_02_T1.prm", + "preprocess/LT05_L1TP_181036_19870612_20201014_02_T1", + "preprocess/LT05_L1TP_181036_19870612_20201014_02_T1/logs", + "preprocess/LT05_L1TP_181036_19870612_20201014_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19870612_20201014_02_T1/logs/level2_log/LT05_L1TP_181036_19870612_20201014_02_T1.log", + "preprocess/LT05_L1TP_181036_19870612_20201014_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19870612_20201014_02_T1/param_files/LT05_L1TP_181036_19870612_20201014_02_T1.prm", + "preprocess/LT05_L1TP_181036_19870628_20201014_02_T1", + "preprocess/LT05_L1TP_181036_19870628_20201014_02_T1/logs", + "preprocess/LT05_L1TP_181036_19870628_20201014_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19870628_20201014_02_T1/logs/level2_log/LT05_L1TP_181036_19870628_20201014_02_T1.log", + "preprocess/LT05_L1TP_181036_19870628_20201014_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19870628_20201014_02_T1/param_files/LT05_L1TP_181036_19870628_20201014_02_T1.prm", + "preprocess/LT05_L1TP_181036_19870714_20201014_02_T1", + "preprocess/LT05_L1TP_181036_19870714_20201014_02_T1/logs", + "preprocess/LT05_L1TP_181036_19870714_20201014_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19870714_20201014_02_T1/logs/level2_log/LT05_L1TP_181036_19870714_20201014_02_T1.log", + "preprocess/LT05_L1TP_181036_19870714_20201014_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19870714_20201014_02_T1/param_files/LT05_L1TP_181036_19870714_20201014_02_T1.prm", + "preprocess/LT05_L1TP_181036_19870730_20201014_02_T1", + "preprocess/LT05_L1TP_181036_19870730_20201014_02_T1/logs", + "preprocess/LT05_L1TP_181036_19870730_20201014_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19870730_20201014_02_T1/logs/level2_log/LT05_L1TP_181036_19870730_20201014_02_T1.log", + "preprocess/LT05_L1TP_181036_19870730_20201014_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19870730_20201014_02_T1/param_files/LT05_L1TP_181036_19870730_20201014_02_T1.prm", + "preprocess/LT05_L1TP_181036_19870815_20201014_02_T1", + "preprocess/LT05_L1TP_181036_19870815_20201014_02_T1/logs", + "preprocess/LT05_L1TP_181036_19870815_20201014_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19870815_20201014_02_T1/logs/level2_log/LT05_L1TP_181036_19870815_20201014_02_T1.log", + "preprocess/LT05_L1TP_181036_19870815_20201014_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19870815_20201014_02_T1/param_files/LT05_L1TP_181036_19870815_20201014_02_T1.prm", + "preprocess/LT05_L1TP_181036_19870831_20201014_02_T1", + "preprocess/LT05_L1TP_181036_19870831_20201014_02_T1/logs", + "preprocess/LT05_L1TP_181036_19870831_20201014_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19870831_20201014_02_T1/logs/level2_log/LT05_L1TP_181036_19870831_20201014_02_T1.log", + "preprocess/LT05_L1TP_181036_19870831_20201014_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19870831_20201014_02_T1/param_files/LT05_L1TP_181036_19870831_20201014_02_T1.prm", + "preprocess/LT05_L1TP_181036_19870916_20201014_02_T1", + "preprocess/LT05_L1TP_181036_19870916_20201014_02_T1/logs", + "preprocess/LT05_L1TP_181036_19870916_20201014_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19870916_20201014_02_T1/logs/level2_log/LT05_L1TP_181036_19870916_20201014_02_T1.log", + "preprocess/LT05_L1TP_181036_19870916_20201014_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19870916_20201014_02_T1/param_files/LT05_L1TP_181036_19870916_20201014_02_T1.prm", + "preprocess/LT05_L1TP_181036_19871002_20201014_02_T1", + "preprocess/LT05_L1TP_181036_19871002_20201014_02_T1/logs", + "preprocess/LT05_L1TP_181036_19871002_20201014_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19871002_20201014_02_T1/logs/level2_log/LT05_L1TP_181036_19871002_20201014_02_T1.log", + "preprocess/LT05_L1TP_181036_19871002_20201014_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19871002_20201014_02_T1/param_files/LT05_L1TP_181036_19871002_20201014_02_T1.prm", + "preprocess/LT05_L1TP_181036_19871205_20201014_02_T1", + "preprocess/LT05_L1TP_181036_19871205_20201014_02_T1/logs", + "preprocess/LT05_L1TP_181036_19871205_20201014_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19871205_20201014_02_T1/logs/level2_log/LT05_L1TP_181036_19871205_20201014_02_T1.log", + "preprocess/LT05_L1TP_181036_19871205_20201014_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19871205_20201014_02_T1/param_files/LT05_L1TP_181036_19871205_20201014_02_T1.prm", + "preprocess/LT05_L1TP_181036_19880106_20211116_02_T1", + "preprocess/LT05_L1TP_181036_19880106_20211116_02_T1/logs", + "preprocess/LT05_L1TP_181036_19880106_20211116_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19880106_20211116_02_T1/logs/level2_log/LT05_L1TP_181036_19880106_20211116_02_T1.log", + "preprocess/LT05_L1TP_181036_19880106_20211116_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19880106_20211116_02_T1/param_files/LT05_L1TP_181036_19880106_20211116_02_T1.prm", + "preprocess/LT05_L1TP_181036_19880122_20200917_02_T1", + "preprocess/LT05_L1TP_181036_19880122_20200917_02_T1/logs", + "preprocess/LT05_L1TP_181036_19880122_20200917_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19880122_20200917_02_T1/logs/level2_log/LT05_L1TP_181036_19880122_20200917_02_T1.log", + "preprocess/LT05_L1TP_181036_19880122_20200917_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19880122_20200917_02_T1/param_files/LT05_L1TP_181036_19880122_20200917_02_T1.prm", + "preprocess/LT05_L1TP_181036_19880223_20200917_02_T1", + "preprocess/LT05_L1TP_181036_19880223_20200917_02_T1/logs", + "preprocess/LT05_L1TP_181036_19880223_20200917_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19880223_20200917_02_T1/logs/level2_log/LT05_L1TP_181036_19880223_20200917_02_T1.log", + "preprocess/LT05_L1TP_181036_19880223_20200917_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19880223_20200917_02_T1/param_files/LT05_L1TP_181036_19880223_20200917_02_T1.prm", + "preprocess/LT05_L1TP_181036_19880310_20200917_02_T1", + "preprocess/LT05_L1TP_181036_19880310_20200917_02_T1/logs", + "preprocess/LT05_L1TP_181036_19880310_20200917_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19880310_20200917_02_T1/logs/level2_log/LT05_L1TP_181036_19880310_20200917_02_T1.log", + "preprocess/LT05_L1TP_181036_19880310_20200917_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19880310_20200917_02_T1/param_files/LT05_L1TP_181036_19880310_20200917_02_T1.prm", + "preprocess/LT05_L1TP_181036_19880326_20200917_02_T1", + "preprocess/LT05_L1TP_181036_19880326_20200917_02_T1/logs", + "preprocess/LT05_L1TP_181036_19880326_20200917_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19880326_20200917_02_T1/logs/level2_log/LT05_L1TP_181036_19880326_20200917_02_T1.log", + "preprocess/LT05_L1TP_181036_19880326_20200917_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19880326_20200917_02_T1/param_files/LT05_L1TP_181036_19880326_20200917_02_T1.prm", + "preprocess/LT05_L1TP_181036_19880411_20200917_02_T1", + "preprocess/LT05_L1TP_181036_19880411_20200917_02_T1/logs", + "preprocess/LT05_L1TP_181036_19880411_20200917_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19880411_20200917_02_T1/logs/level2_log/LT05_L1TP_181036_19880411_20200917_02_T1.log", + "preprocess/LT05_L1TP_181036_19880411_20200917_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19880411_20200917_02_T1/param_files/LT05_L1TP_181036_19880411_20200917_02_T1.prm", + "preprocess/LT05_L1TP_181036_19880529_20200917_02_T1", + "preprocess/LT05_L1TP_181036_19880529_20200917_02_T1/logs", + "preprocess/LT05_L1TP_181036_19880529_20200917_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19880529_20200917_02_T1/logs/level2_log/LT05_L1TP_181036_19880529_20200917_02_T1.log", + "preprocess/LT05_L1TP_181036_19880529_20200917_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19880529_20200917_02_T1/param_files/LT05_L1TP_181036_19880529_20200917_02_T1.prm", + "preprocess/LT05_L1TP_181036_19880614_20211118_02_T1", + "preprocess/LT05_L1TP_181036_19880614_20211118_02_T1/logs", + "preprocess/LT05_L1TP_181036_19880614_20211118_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19880614_20211118_02_T1/logs/level2_log/LT05_L1TP_181036_19880614_20211118_02_T1.log", + "preprocess/LT05_L1TP_181036_19880614_20211118_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19880614_20211118_02_T1/param_files/LT05_L1TP_181036_19880614_20211118_02_T1.prm", + "preprocess/LT05_L1TP_181036_19880630_20200917_02_T1", + "preprocess/LT05_L1TP_181036_19880630_20200917_02_T1/logs", + "preprocess/LT05_L1TP_181036_19880630_20200917_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19880630_20200917_02_T1/logs/level2_log/LT05_L1TP_181036_19880630_20200917_02_T1.log", + "preprocess/LT05_L1TP_181036_19880630_20200917_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19880630_20200917_02_T1/param_files/LT05_L1TP_181036_19880630_20200917_02_T1.prm", + "preprocess/LT05_L1TP_181036_19880716_20200917_02_T1", + "preprocess/LT05_L1TP_181036_19880716_20200917_02_T1/logs", + "preprocess/LT05_L1TP_181036_19880716_20200917_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19880716_20200917_02_T1/logs/level2_log/LT05_L1TP_181036_19880716_20200917_02_T1.log", + "preprocess/LT05_L1TP_181036_19880716_20200917_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19880716_20200917_02_T1/param_files/LT05_L1TP_181036_19880716_20200917_02_T1.prm", + "preprocess/LT05_L1TP_181036_19880801_20200917_02_T1", + "preprocess/LT05_L1TP_181036_19880801_20200917_02_T1/logs", + "preprocess/LT05_L1TP_181036_19880801_20200917_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19880801_20200917_02_T1/logs/level2_log/LT05_L1TP_181036_19880801_20200917_02_T1.log", + "preprocess/LT05_L1TP_181036_19880801_20200917_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19880801_20200917_02_T1/param_files/LT05_L1TP_181036_19880801_20200917_02_T1.prm", + "preprocess/LT05_L1TP_181036_19880902_20211118_02_T1", + "preprocess/LT05_L1TP_181036_19880902_20211118_02_T1/logs", + "preprocess/LT05_L1TP_181036_19880902_20211118_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19880902_20211118_02_T1/logs/level2_log/LT05_L1TP_181036_19880902_20211118_02_T1.log", + "preprocess/LT05_L1TP_181036_19880902_20211118_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19880902_20211118_02_T1/param_files/LT05_L1TP_181036_19880902_20211118_02_T1.prm", + "preprocess/LT05_L1TP_181036_19880918_20200917_02_T1", + "preprocess/LT05_L1TP_181036_19880918_20200917_02_T1/logs", + "preprocess/LT05_L1TP_181036_19880918_20200917_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19880918_20200917_02_T1/logs/level2_log/LT05_L1TP_181036_19880918_20200917_02_T1.log", + "preprocess/LT05_L1TP_181036_19880918_20200917_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19880918_20200917_02_T1/param_files/LT05_L1TP_181036_19880918_20200917_02_T1.prm", + "preprocess/LT05_L1TP_181036_19881004_20200917_02_T1", + "preprocess/LT05_L1TP_181036_19881004_20200917_02_T1/logs", + "preprocess/LT05_L1TP_181036_19881004_20200917_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19881004_20200917_02_T1/logs/level2_log/LT05_L1TP_181036_19881004_20200917_02_T1.log", + "preprocess/LT05_L1TP_181036_19881004_20200917_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19881004_20200917_02_T1/param_files/LT05_L1TP_181036_19881004_20200917_02_T1.prm", + "preprocess/LT05_L1TP_181036_19881020_20200917_02_T1", + "preprocess/LT05_L1TP_181036_19881020_20200917_02_T1/logs", + "preprocess/LT05_L1TP_181036_19881020_20200917_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19881020_20200917_02_T1/logs/level2_log/LT05_L1TP_181036_19881020_20200917_02_T1.log", + "preprocess/LT05_L1TP_181036_19881020_20200917_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19881020_20200917_02_T1/param_files/LT05_L1TP_181036_19881020_20200917_02_T1.prm", + "preprocess/LT05_L1TP_181036_19881121_20211118_02_T1", + "preprocess/LT05_L1TP_181036_19881121_20211118_02_T1/logs", + "preprocess/LT05_L1TP_181036_19881121_20211118_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19881121_20211118_02_T1/logs/level2_log/LT05_L1TP_181036_19881121_20211118_02_T1.log", + "preprocess/LT05_L1TP_181036_19881121_20211118_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19881121_20211118_02_T1/param_files/LT05_L1TP_181036_19881121_20211118_02_T1.prm", + "preprocess/LT05_L1TP_181036_19881223_20211119_02_T1", + "preprocess/LT05_L1TP_181036_19881223_20211119_02_T1/logs", + "preprocess/LT05_L1TP_181036_19881223_20211119_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19881223_20211119_02_T1/logs/level2_log/LT05_L1TP_181036_19881223_20211119_02_T1.log", + "preprocess/LT05_L1TP_181036_19881223_20211119_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19881223_20211119_02_T1/param_files/LT05_L1TP_181036_19881223_20211119_02_T1.prm", + "preprocess/LT05_L1TP_181036_19890108_20200916_02_T1", + "preprocess/LT05_L1TP_181036_19890108_20200916_02_T1/logs", + "preprocess/LT05_L1TP_181036_19890108_20200916_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19890108_20200916_02_T1/logs/level2_log/LT05_L1TP_181036_19890108_20200916_02_T1.log", + "preprocess/LT05_L1TP_181036_19890108_20200916_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19890108_20200916_02_T1/param_files/LT05_L1TP_181036_19890108_20200916_02_T1.prm", + "preprocess/LT05_L1TP_181036_19890209_20211120_02_T1", + "preprocess/LT05_L1TP_181036_19890209_20211120_02_T1/logs", + "preprocess/LT05_L1TP_181036_19890209_20211120_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19890209_20211120_02_T1/logs/level2_log/LT05_L1TP_181036_19890209_20211120_02_T1.log", + "preprocess/LT05_L1TP_181036_19890209_20211120_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19890209_20211120_02_T1/param_files/LT05_L1TP_181036_19890209_20211120_02_T1.prm", + "preprocess/LT05_L1TP_181036_19890414_20200916_02_T1", + "preprocess/LT05_L1TP_181036_19890414_20200916_02_T1/logs", + "preprocess/LT05_L1TP_181036_19890414_20200916_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19890414_20200916_02_T1/logs/level2_log/LT05_L1TP_181036_19890414_20200916_02_T1.log", + "preprocess/LT05_L1TP_181036_19890414_20200916_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19890414_20200916_02_T1/param_files/LT05_L1TP_181036_19890414_20200916_02_T1.prm", + "preprocess/LT05_L1TP_181036_19890430_20200916_02_T1", + "preprocess/LT05_L1TP_181036_19890430_20200916_02_T1/logs", + "preprocess/LT05_L1TP_181036_19890430_20200916_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19890430_20200916_02_T1/logs/level2_log/LT05_L1TP_181036_19890430_20200916_02_T1.log", + "preprocess/LT05_L1TP_181036_19890430_20200916_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19890430_20200916_02_T1/param_files/LT05_L1TP_181036_19890430_20200916_02_T1.prm", + "preprocess/LT05_L1TP_181036_19890516_20200916_02_T1", + "preprocess/LT05_L1TP_181036_19890516_20200916_02_T1/logs", + "preprocess/LT05_L1TP_181036_19890516_20200916_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19890516_20200916_02_T1/logs/level2_log/LT05_L1TP_181036_19890516_20200916_02_T1.log", + "preprocess/LT05_L1TP_181036_19890516_20200916_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19890516_20200916_02_T1/param_files/LT05_L1TP_181036_19890516_20200916_02_T1.prm", + "preprocess/LT05_L1TP_181036_19890601_20200916_02_T1", + "preprocess/LT05_L1TP_181036_19890601_20200916_02_T1/logs", + "preprocess/LT05_L1TP_181036_19890601_20200916_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19890601_20200916_02_T1/logs/level2_log/LT05_L1TP_181036_19890601_20200916_02_T1.log", + "preprocess/LT05_L1TP_181036_19890601_20200916_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19890601_20200916_02_T1/param_files/LT05_L1TP_181036_19890601_20200916_02_T1.prm", + "preprocess/LT05_L1TP_181036_19890617_20200916_02_T1", + "preprocess/LT05_L1TP_181036_19890617_20200916_02_T1/logs", + "preprocess/LT05_L1TP_181036_19890617_20200916_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19890617_20200916_02_T1/logs/level2_log/LT05_L1TP_181036_19890617_20200916_02_T1.log", + "preprocess/LT05_L1TP_181036_19890617_20200916_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19890617_20200916_02_T1/param_files/LT05_L1TP_181036_19890617_20200916_02_T1.prm", + "preprocess/LT05_L1TP_181036_19890703_20200916_02_T1", + "preprocess/LT05_L1TP_181036_19890703_20200916_02_T1/logs", + "preprocess/LT05_L1TP_181036_19890703_20200916_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19890703_20200916_02_T1/logs/level2_log/LT05_L1TP_181036_19890703_20200916_02_T1.log", + "preprocess/LT05_L1TP_181036_19890703_20200916_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19890703_20200916_02_T1/param_files/LT05_L1TP_181036_19890703_20200916_02_T1.prm", + "preprocess/LT05_L1TP_181036_19890719_20200916_02_T1", + "preprocess/LT05_L1TP_181036_19890719_20200916_02_T1/logs", + "preprocess/LT05_L1TP_181036_19890719_20200916_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19890719_20200916_02_T1/logs/level2_log/LT05_L1TP_181036_19890719_20200916_02_T1.log", + "preprocess/LT05_L1TP_181036_19890719_20200916_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19890719_20200916_02_T1/param_files/LT05_L1TP_181036_19890719_20200916_02_T1.prm", + "preprocess/LT05_L1TP_181036_19890804_20200916_02_T1", + "preprocess/LT05_L1TP_181036_19890804_20200916_02_T1/logs", + "preprocess/LT05_L1TP_181036_19890804_20200916_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19890804_20200916_02_T1/logs/level2_log/LT05_L1TP_181036_19890804_20200916_02_T1.log", + "preprocess/LT05_L1TP_181036_19890804_20200916_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19890804_20200916_02_T1/param_files/LT05_L1TP_181036_19890804_20200916_02_T1.prm", + "preprocess/LT05_L1TP_181036_19890820_20200916_02_T1", + "preprocess/LT05_L1TP_181036_19890820_20200916_02_T1/logs", + "preprocess/LT05_L1TP_181036_19890820_20200916_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19890820_20200916_02_T1/logs/level2_log/LT05_L1TP_181036_19890820_20200916_02_T1.log", + "preprocess/LT05_L1TP_181036_19890820_20200916_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19890820_20200916_02_T1/param_files/LT05_L1TP_181036_19890820_20200916_02_T1.prm", + "preprocess/LT05_L1TP_181036_19890905_20200916_02_T1", + "preprocess/LT05_L1TP_181036_19890905_20200916_02_T1/logs", + "preprocess/LT05_L1TP_181036_19890905_20200916_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19890905_20200916_02_T1/logs/level2_log/LT05_L1TP_181036_19890905_20200916_02_T1.log", + "preprocess/LT05_L1TP_181036_19890905_20200916_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19890905_20200916_02_T1/param_files/LT05_L1TP_181036_19890905_20200916_02_T1.prm", + "preprocess/LT05_L1TP_181036_19890921_20200916_02_T1", + "preprocess/LT05_L1TP_181036_19890921_20200916_02_T1/logs", + "preprocess/LT05_L1TP_181036_19890921_20200916_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19890921_20200916_02_T1/logs/level2_log/LT05_L1TP_181036_19890921_20200916_02_T1.log", + "preprocess/LT05_L1TP_181036_19890921_20200916_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19890921_20200916_02_T1/param_files/LT05_L1TP_181036_19890921_20200916_02_T1.prm", + "preprocess/LT05_L1TP_181036_19891007_20200916_02_T1", + "preprocess/LT05_L1TP_181036_19891007_20200916_02_T1/logs", + "preprocess/LT05_L1TP_181036_19891007_20200916_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19891007_20200916_02_T1/logs/level2_log/LT05_L1TP_181036_19891007_20200916_02_T1.log", + "preprocess/LT05_L1TP_181036_19891007_20200916_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19891007_20200916_02_T1/param_files/LT05_L1TP_181036_19891007_20200916_02_T1.prm", + "preprocess/LT05_L1TP_181036_19891023_20200916_02_T1", + "preprocess/LT05_L1TP_181036_19891023_20200916_02_T1/logs", + "preprocess/LT05_L1TP_181036_19891023_20200916_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19891023_20200916_02_T1/logs/level2_log/LT05_L1TP_181036_19891023_20200916_02_T1.log", + "preprocess/LT05_L1TP_181036_19891023_20200916_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19891023_20200916_02_T1/param_files/LT05_L1TP_181036_19891023_20200916_02_T1.prm", + "preprocess/LT05_L1TP_181036_19891108_20200916_02_T1", + "preprocess/LT05_L1TP_181036_19891108_20200916_02_T1/logs", + "preprocess/LT05_L1TP_181036_19891108_20200916_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19891108_20200916_02_T1/logs/level2_log/LT05_L1TP_181036_19891108_20200916_02_T1.log", + "preprocess/LT05_L1TP_181036_19891108_20200916_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19891108_20200916_02_T1/param_files/LT05_L1TP_181036_19891108_20200916_02_T1.prm", + "preprocess/LT05_L1TP_181036_19891124_20211121_02_T1", + "preprocess/LT05_L1TP_181036_19891124_20211121_02_T1/logs", + "preprocess/LT05_L1TP_181036_19891124_20211121_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19891124_20211121_02_T1/logs/level2_log/LT05_L1TP_181036_19891124_20211121_02_T1.log", + "preprocess/LT05_L1TP_181036_19891124_20211121_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19891124_20211121_02_T1/param_files/LT05_L1TP_181036_19891124_20211121_02_T1.prm", + "preprocess/LT05_L1TP_181036_19891210_20211121_02_T1", + "preprocess/LT05_L1TP_181036_19891210_20211121_02_T1/logs", + "preprocess/LT05_L1TP_181036_19891210_20211121_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19891210_20211121_02_T1/logs/level2_log/LT05_L1TP_181036_19891210_20211121_02_T1.log", + "preprocess/LT05_L1TP_181036_19891210_20211121_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19891210_20211121_02_T1/param_files/LT05_L1TP_181036_19891210_20211121_02_T1.prm", + "trend", + "trend/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_TSI", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_TSI/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_TSI/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_BLU_TSI.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_TSI/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_BLU_TSI.tif.aux.xml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_TSI/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_TSI/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_TSI.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_TSI/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_VBL-CAO", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_VBL-CAO/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_VBL-CAO/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_BLU_VBL-CAO.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_VBL-CAO/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_VBL-CAO/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_VBL-CAO.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_VBL-CAO/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_VBL-POL", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_VBL-POL/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_VBL-POL/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_BLU_VBL-POL.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_VBL-POL/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_VBL-POL/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_VBL-POL.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_VBL-POL/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_VBL-TRO", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_VBL-TRO/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_VBL-TRO/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_BLU_VBL-TRO.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_VBL-TRO/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_VBL-TRO/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_VBL-TRO.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_VBL-TRO/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_VPS-CAO", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_VPS-CAO/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_VPS-CAO/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_BLU_VPS-CAO.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_VPS-CAO/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_VPS-CAO/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_VPS-CAO.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_VPS-CAO/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_VPS-POL", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_VPS-POL/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_VPS-POL/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_BLU_VPS-POL.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_VPS-POL/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_VPS-POL/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_VPS-POL.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_VPS-POL/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_VPS-TRO", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_VPS-TRO/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_VPS-TRO/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_BLU_VPS-TRO.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_VPS-TRO/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_VPS-TRO/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_VPS-TRO.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_VPS-TRO/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_VSA-CAO", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_VSA-CAO/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_VSA-CAO/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_BLU_VSA-CAO.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_VSA-CAO/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_VSA-CAO/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_VSA-CAO.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_VSA-CAO/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_VSA-POL", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_VSA-POL/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_VSA-POL/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_BLU_VSA-POL.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_VSA-POL/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_VSA-POL/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_VSA-POL.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_VSA-POL/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_VSA-TRO", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_VSA-TRO/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_VSA-TRO/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_BLU_VSA-TRO.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_VSA-TRO/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_VSA-TRO/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_VSA-TRO.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_BLU_VSA-TRO/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_TSI", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_TSI/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_TSI/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_GRN_TSI.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_TSI/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_GRN_TSI.tif.aux.xml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_TSI/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_TSI/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_TSI.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_TSI/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_VBL-CAO", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_VBL-CAO/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_VBL-CAO/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_GRN_VBL-CAO.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_VBL-CAO/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_VBL-CAO/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_VBL-CAO.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_VBL-CAO/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_VBL-POL", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_VBL-POL/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_VBL-POL/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_GRN_VBL-POL.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_VBL-POL/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_VBL-POL/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_VBL-POL.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_VBL-POL/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_VBL-TRO", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_VBL-TRO/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_VBL-TRO/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_GRN_VBL-TRO.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_VBL-TRO/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_VBL-TRO/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_VBL-TRO.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_VBL-TRO/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_VPS-CAO", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_VPS-CAO/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_VPS-CAO/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_GRN_VPS-CAO.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_VPS-CAO/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_VPS-CAO/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_VPS-CAO.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_VPS-CAO/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_VPS-POL", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_VPS-POL/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_VPS-POL/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_GRN_VPS-POL.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_VPS-POL/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_VPS-POL/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_VPS-POL.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_VPS-POL/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_VPS-TRO", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_VPS-TRO/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_VPS-TRO/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_GRN_VPS-TRO.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_VPS-TRO/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_VPS-TRO/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_VPS-TRO.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_VPS-TRO/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_VSA-CAO", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_VSA-CAO/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_VSA-CAO/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_GRN_VSA-CAO.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_VSA-CAO/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_VSA-CAO/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_VSA-CAO.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_VSA-CAO/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_VSA-POL", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_VSA-POL/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_VSA-POL/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_GRN_VSA-POL.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_VSA-POL/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_VSA-POL/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_VSA-POL.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_VSA-POL/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_VSA-TRO", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_VSA-TRO/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_VSA-TRO/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_GRN_VSA-TRO.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_VSA-TRO/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_VSA-TRO/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_VSA-TRO.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_GRN_VSA-TRO/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_TSI", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_TSI/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_TSI/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_NDV_TSI.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_TSI/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_NDV_TSI.tif.aux.xml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_TSI/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_TSI/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_TSI.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_TSI/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_VBL-CAO", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_VBL-CAO/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_VBL-CAO/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_NDV_VBL-CAO.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_VBL-CAO/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_VBL-CAO/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_VBL-CAO.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_VBL-CAO/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_VBL-POL", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_VBL-POL/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_VBL-POL/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_NDV_VBL-POL.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_VBL-POL/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_VBL-POL/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_VBL-POL.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_VBL-POL/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_VBL-TRO", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_VBL-TRO/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_VBL-TRO/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_NDV_VBL-TRO.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_VBL-TRO/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_VBL-TRO/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_VBL-TRO.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_VBL-TRO/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_VPS-CAO", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_VPS-CAO/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_VPS-CAO/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_NDV_VPS-CAO.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_VPS-CAO/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_VPS-CAO/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_VPS-CAO.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_VPS-CAO/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_VPS-POL", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_VPS-POL/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_VPS-POL/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_NDV_VPS-POL.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_VPS-POL/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_VPS-POL/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_VPS-POL.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_VPS-POL/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_VPS-TRO", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_VPS-TRO/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_VPS-TRO/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_NDV_VPS-TRO.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_VPS-TRO/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_VPS-TRO/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_VPS-TRO.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_VPS-TRO/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_VSA-CAO", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_VSA-CAO/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_VSA-CAO/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_NDV_VSA-CAO.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_VSA-CAO/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_VSA-CAO/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_VSA-CAO.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_VSA-CAO/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_VSA-POL", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_VSA-POL/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_VSA-POL/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_NDV_VSA-POL.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_VSA-POL/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_VSA-POL/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_VSA-POL.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_VSA-POL/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_VSA-TRO", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_VSA-TRO/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_VSA-TRO/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_NDV_VSA-TRO.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_VSA-TRO/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_VSA-TRO/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_VSA-TRO.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NDV_VSA-TRO/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_TSI", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_TSI/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_TSI/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_NIR_TSI.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_TSI/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_NIR_TSI.tif.aux.xml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_TSI/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_TSI/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_TSI.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_TSI/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_VBL-CAO", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_VBL-CAO/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_VBL-CAO/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_NIR_VBL-CAO.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_VBL-CAO/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_VBL-CAO/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_VBL-CAO.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_VBL-CAO/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_VBL-POL", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_VBL-POL/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_VBL-POL/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_NIR_VBL-POL.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_VBL-POL/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_VBL-POL/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_VBL-POL.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_VBL-POL/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_VBL-TRO", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_VBL-TRO/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_VBL-TRO/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_NIR_VBL-TRO.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_VBL-TRO/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_VBL-TRO/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_VBL-TRO.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_VBL-TRO/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_VPS-CAO", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_VPS-CAO/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_VPS-CAO/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_NIR_VPS-CAO.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_VPS-CAO/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_VPS-CAO/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_VPS-CAO.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_VPS-CAO/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_VPS-POL", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_VPS-POL/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_VPS-POL/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_NIR_VPS-POL.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_VPS-POL/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_VPS-POL/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_VPS-POL.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_VPS-POL/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_VPS-TRO", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_VPS-TRO/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_VPS-TRO/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_NIR_VPS-TRO.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_VPS-TRO/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_VPS-TRO/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_VPS-TRO.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_VPS-TRO/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_VSA-CAO", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_VSA-CAO/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_VSA-CAO/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_NIR_VSA-CAO.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_VSA-CAO/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_VSA-CAO/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_VSA-CAO.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_VSA-CAO/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_VSA-POL", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_VSA-POL/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_VSA-POL/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_NIR_VSA-POL.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_VSA-POL/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_VSA-POL/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_VSA-POL.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_VSA-POL/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_VSA-TRO", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_VSA-TRO/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_VSA-TRO/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_NIR_VSA-TRO.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_VSA-TRO/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_VSA-TRO/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_VSA-TRO.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_NIR_VSA-TRO/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_TSI", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_TSI/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_TSI/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_RED_TSI.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_TSI/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_RED_TSI.tif.aux.xml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_TSI/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_TSI/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_TSI.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_TSI/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_VBL-CAO", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_VBL-CAO/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_VBL-CAO/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_RED_VBL-CAO.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_VBL-CAO/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_VBL-CAO/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_VBL-CAO.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_VBL-CAO/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_VBL-POL", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_VBL-POL/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_VBL-POL/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_RED_VBL-POL.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_VBL-POL/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_VBL-POL/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_VBL-POL.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_VBL-POL/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_VBL-TRO", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_VBL-TRO/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_VBL-TRO/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_RED_VBL-TRO.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_VBL-TRO/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_VBL-TRO/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_VBL-TRO.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_VBL-TRO/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_VPS-CAO", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_VPS-CAO/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_VPS-CAO/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_RED_VPS-CAO.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_VPS-CAO/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_VPS-CAO/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_VPS-CAO.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_VPS-CAO/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_VPS-POL", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_VPS-POL/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_VPS-POL/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_RED_VPS-POL.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_VPS-POL/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_VPS-POL/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_VPS-POL.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_VPS-POL/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_VPS-TRO", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_VPS-TRO/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_VPS-TRO/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_RED_VPS-TRO.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_VPS-TRO/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_VPS-TRO/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_VPS-TRO.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_VPS-TRO/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_VSA-CAO", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_VSA-CAO/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_VSA-CAO/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_RED_VSA-CAO.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_VSA-CAO/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_VSA-CAO/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_VSA-CAO.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_VSA-CAO/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_VSA-POL", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_VSA-POL/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_VSA-POL/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_RED_VSA-POL.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_VSA-POL/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_VSA-POL/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_VSA-POL.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_VSA-POL/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_VSA-TRO", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_VSA-TRO/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_VSA-TRO/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_RED_VSA-TRO.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_VSA-TRO/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_VSA-TRO/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_VSA-TRO.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_RED_VSA-TRO/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_TSI", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_TSI/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_TSI/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_SMA_TSI.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_TSI/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_SMA_TSI.tif.aux.xml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_TSI/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_TSI/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_TSI.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_TSI/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_VBL-CAO", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_VBL-CAO/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_VBL-CAO/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_SMA_VBL-CAO.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_VBL-CAO/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_VBL-CAO/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_VBL-CAO.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_VBL-CAO/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_VBL-POL", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_VBL-POL/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_VBL-POL/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_SMA_VBL-POL.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_VBL-POL/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_VBL-POL/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_VBL-POL.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_VBL-POL/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_VBL-TRO", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_VBL-TRO/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_VBL-TRO/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_SMA_VBL-TRO.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_VBL-TRO/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_VBL-TRO/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_VBL-TRO.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_VBL-TRO/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_VPS-CAO", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_VPS-CAO/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_VPS-CAO/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_SMA_VPS-CAO.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_VPS-CAO/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_VPS-CAO/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_VPS-CAO.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_VPS-CAO/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_VPS-POL", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_VPS-POL/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_VPS-POL/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_SMA_VPS-POL.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_VPS-POL/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_VPS-POL/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_VPS-POL.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_VPS-POL/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_VPS-TRO", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_VPS-TRO/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_VPS-TRO/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_SMA_VPS-TRO.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_VPS-TRO/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_VPS-TRO/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_VPS-TRO.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_VPS-TRO/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_VSA-CAO", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_VSA-CAO/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_VSA-CAO/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_SMA_VSA-CAO.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_VSA-CAO/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_VSA-CAO/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_VSA-CAO.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_VSA-CAO/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_VSA-POL", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_VSA-POL/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_VSA-POL/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_SMA_VSA-POL.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_VSA-POL/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_VSA-POL/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_VSA-POL.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_VSA-POL/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_VSA-TRO", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_VSA-TRO/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_VSA-TRO/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_SMA_VSA-TRO.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_VSA-TRO/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_VSA-TRO/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_VSA-TRO.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SMA_VSA-TRO/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_TSI", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_TSI/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_TSI/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_SW1_TSI.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_TSI/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_SW1_TSI.tif.aux.xml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_TSI/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_TSI/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_TSI.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_TSI/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_VBL-CAO", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_VBL-CAO/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_VBL-CAO/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_SW1_VBL-CAO.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_VBL-CAO/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_VBL-CAO/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_VBL-CAO.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_VBL-CAO/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_VBL-POL", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_VBL-POL/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_VBL-POL/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_SW1_VBL-POL.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_VBL-POL/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_VBL-POL/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_VBL-POL.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_VBL-POL/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_VBL-TRO", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_VBL-TRO/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_VBL-TRO/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_SW1_VBL-TRO.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_VBL-TRO/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_VBL-TRO/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_VBL-TRO.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_VBL-TRO/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_VPS-CAO", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_VPS-CAO/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_VPS-CAO/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_SW1_VPS-CAO.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_VPS-CAO/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_VPS-CAO/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_VPS-CAO.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_VPS-CAO/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_VPS-POL", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_VPS-POL/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_VPS-POL/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_SW1_VPS-POL.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_VPS-POL/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_VPS-POL/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_VPS-POL.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_VPS-POL/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_VPS-TRO", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_VPS-TRO/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_VPS-TRO/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_SW1_VPS-TRO.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_VPS-TRO/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_VPS-TRO/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_VPS-TRO.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_VPS-TRO/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_VSA-CAO", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_VSA-CAO/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_VSA-CAO/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_SW1_VSA-CAO.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_VSA-CAO/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_VSA-CAO/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_VSA-CAO.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_VSA-CAO/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_VSA-POL", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_VSA-POL/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_VSA-POL/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_SW1_VSA-POL.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_VSA-POL/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_VSA-POL/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_VSA-POL.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_VSA-POL/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_VSA-TRO", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_VSA-TRO/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_VSA-TRO/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_SW1_VSA-TRO.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_VSA-TRO/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_VSA-TRO/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_VSA-TRO.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW1_VSA-TRO/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_TSI", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_TSI/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_TSI/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_SW2_TSI.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_TSI/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_SW2_TSI.tif.aux.xml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_TSI/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_TSI/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_TSI.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_TSI/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_VBL-CAO", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_VBL-CAO/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_VBL-CAO/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_SW2_VBL-CAO.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_VBL-CAO/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_VBL-CAO/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_VBL-CAO.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_VBL-CAO/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_VBL-POL", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_VBL-POL/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_VBL-POL/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_SW2_VBL-POL.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_VBL-POL/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_VBL-POL/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_VBL-POL.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_VBL-POL/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_VBL-TRO", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_VBL-TRO/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_VBL-TRO/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_SW2_VBL-TRO.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_VBL-TRO/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_VBL-TRO/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_VBL-TRO.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_VBL-TRO/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_VPS-CAO", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_VPS-CAO/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_VPS-CAO/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_SW2_VPS-CAO.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_VPS-CAO/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_VPS-CAO/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_VPS-CAO.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_VPS-CAO/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_VPS-POL", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_VPS-POL/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_VPS-POL/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_SW2_VPS-POL.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_VPS-POL/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_VPS-POL/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_VPS-POL.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_VPS-POL/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_VPS-TRO", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_VPS-TRO/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_VPS-TRO/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_SW2_VPS-TRO.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_VPS-TRO/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_VPS-TRO/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_VPS-TRO.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_VPS-TRO/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_VSA-CAO", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_VSA-CAO/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_VSA-CAO/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_SW2_VSA-CAO.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_VSA-CAO/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_VSA-CAO/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_VSA-CAO.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_VSA-CAO/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_VSA-POL", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_VSA-POL/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_VSA-POL/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_SW2_VSA-POL.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_VSA-POL/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_VSA-POL/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_VSA-POL.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_VSA-POL/versions.yml", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_VSA-TRO", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_VSA-TRO/X0109_Y0102", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_VSA-TRO/X0109_Y0102/1987-1989_001-365_HL_TSA_LNDLG_SW2_VSA-TRO.tif", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_VSA-TRO/mosaic", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_VSA-TRO/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_VSA-TRO.vrt", + "trend/mosaic/1987-1989_001-365_HL_TSA_LNDLG_SW2_VSA-TRO/versions.yml", + "trend/pyramid", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_BLU_TSI", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_BLU_TSI/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_BLU_TSI/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_BLU_TSI/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_BLU_TSI.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_BLU_VBL-CAO", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_BLU_VBL-CAO/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_BLU_VBL-CAO/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_BLU_VBL-CAO/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_BLU_VBL-CAO.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_BLU_VBL-POL", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_BLU_VBL-POL/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_BLU_VBL-POL/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_BLU_VBL-POL/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_BLU_VBL-POL.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_BLU_VBL-TRO", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_BLU_VBL-TRO/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_BLU_VBL-TRO/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_BLU_VBL-TRO/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_BLU_VBL-TRO.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_BLU_VPS-CAO", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_BLU_VPS-CAO/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_BLU_VPS-CAO/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_BLU_VPS-CAO/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_BLU_VPS-CAO.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_BLU_VPS-POL", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_BLU_VPS-POL/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_BLU_VPS-POL/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_BLU_VPS-POL/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_BLU_VPS-POL.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_BLU_VPS-TRO", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_BLU_VPS-TRO/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_BLU_VPS-TRO/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_BLU_VPS-TRO/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_BLU_VPS-TRO.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_BLU_VSA-CAO", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_BLU_VSA-CAO/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_BLU_VSA-CAO/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_BLU_VSA-CAO/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_BLU_VSA-CAO.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_BLU_VSA-POL", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_BLU_VSA-POL/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_BLU_VSA-POL/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_BLU_VSA-POL/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_BLU_VSA-POL.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_BLU_VSA-TRO", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_BLU_VSA-TRO/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_BLU_VSA-TRO/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_BLU_VSA-TRO/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_BLU_VSA-TRO.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_GRN_TSI", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_GRN_TSI/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_GRN_TSI/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_GRN_TSI/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_GRN_TSI.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_GRN_VBL-CAO", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_GRN_VBL-CAO/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_GRN_VBL-CAO/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_GRN_VBL-CAO/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_GRN_VBL-CAO.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_GRN_VBL-POL", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_GRN_VBL-POL/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_GRN_VBL-POL/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_GRN_VBL-POL/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_GRN_VBL-POL.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_GRN_VBL-TRO", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_GRN_VBL-TRO/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_GRN_VBL-TRO/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_GRN_VBL-TRO/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_GRN_VBL-TRO.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_GRN_VPS-CAO", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_GRN_VPS-CAO/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_GRN_VPS-CAO/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_GRN_VPS-CAO/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_GRN_VPS-CAO.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_GRN_VPS-POL", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_GRN_VPS-POL/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_GRN_VPS-POL/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_GRN_VPS-POL/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_GRN_VPS-POL.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_GRN_VPS-TRO", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_GRN_VPS-TRO/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_GRN_VPS-TRO/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_GRN_VPS-TRO/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_GRN_VPS-TRO.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_GRN_VSA-CAO", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_GRN_VSA-CAO/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_GRN_VSA-CAO/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_GRN_VSA-CAO/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_GRN_VSA-CAO.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_GRN_VSA-POL", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_GRN_VSA-POL/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_GRN_VSA-POL/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_GRN_VSA-POL/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_GRN_VSA-POL.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_GRN_VSA-TRO", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_GRN_VSA-TRO/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_GRN_VSA-TRO/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_GRN_VSA-TRO/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_GRN_VSA-TRO.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NDV_TSI", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NDV_TSI/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NDV_TSI/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NDV_TSI/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_NDV_TSI.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NDV_VBL-CAO", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NDV_VBL-CAO/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NDV_VBL-CAO/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NDV_VBL-CAO/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_NDV_VBL-CAO.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NDV_VBL-POL", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NDV_VBL-POL/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NDV_VBL-POL/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NDV_VBL-POL/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_NDV_VBL-POL.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NDV_VBL-TRO", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NDV_VBL-TRO/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NDV_VBL-TRO/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NDV_VBL-TRO/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_NDV_VBL-TRO.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NDV_VPS-CAO", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NDV_VPS-CAO/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NDV_VPS-CAO/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NDV_VPS-CAO/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_NDV_VPS-CAO.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NDV_VPS-POL", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NDV_VPS-POL/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NDV_VPS-POL/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NDV_VPS-POL/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_NDV_VPS-POL.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NDV_VPS-TRO", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NDV_VPS-TRO/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NDV_VPS-TRO/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NDV_VPS-TRO/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_NDV_VPS-TRO.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NDV_VSA-CAO", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NDV_VSA-CAO/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NDV_VSA-CAO/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NDV_VSA-CAO/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_NDV_VSA-CAO.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NDV_VSA-POL", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NDV_VSA-POL/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NDV_VSA-POL/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NDV_VSA-POL/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_NDV_VSA-POL.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NDV_VSA-TRO", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NDV_VSA-TRO/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NDV_VSA-TRO/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NDV_VSA-TRO/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_NDV_VSA-TRO.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NIR_TSI", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NIR_TSI/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NIR_TSI/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NIR_TSI/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_NIR_TSI.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NIR_VBL-CAO", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NIR_VBL-CAO/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NIR_VBL-CAO/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NIR_VBL-CAO/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_NIR_VBL-CAO.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NIR_VBL-POL", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NIR_VBL-POL/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NIR_VBL-POL/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NIR_VBL-POL/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_NIR_VBL-POL.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NIR_VBL-TRO", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NIR_VBL-TRO/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NIR_VBL-TRO/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NIR_VBL-TRO/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_NIR_VBL-TRO.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NIR_VPS-CAO", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NIR_VPS-CAO/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NIR_VPS-CAO/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NIR_VPS-CAO/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_NIR_VPS-CAO.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NIR_VPS-POL", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NIR_VPS-POL/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NIR_VPS-POL/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NIR_VPS-POL/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_NIR_VPS-POL.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NIR_VPS-TRO", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NIR_VPS-TRO/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NIR_VPS-TRO/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NIR_VPS-TRO/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_NIR_VPS-TRO.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NIR_VSA-CAO", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NIR_VSA-CAO/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NIR_VSA-CAO/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NIR_VSA-CAO/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_NIR_VSA-CAO.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NIR_VSA-POL", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NIR_VSA-POL/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NIR_VSA-POL/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NIR_VSA-POL/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_NIR_VSA-POL.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NIR_VSA-TRO", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NIR_VSA-TRO/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NIR_VSA-TRO/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_NIR_VSA-TRO/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_NIR_VSA-TRO.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_RED_TSI", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_RED_TSI/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_RED_TSI/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_RED_TSI/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_RED_TSI.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_RED_VBL-CAO", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_RED_VBL-CAO/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_RED_VBL-CAO/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_RED_VBL-CAO/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_RED_VBL-CAO.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_RED_VBL-POL", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_RED_VBL-POL/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_RED_VBL-POL/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_RED_VBL-POL/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_RED_VBL-POL.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_RED_VBL-TRO", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_RED_VBL-TRO/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_RED_VBL-TRO/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_RED_VBL-TRO/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_RED_VBL-TRO.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_RED_VPS-CAO", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_RED_VPS-CAO/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_RED_VPS-CAO/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_RED_VPS-CAO/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_RED_VPS-CAO.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_RED_VPS-POL", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_RED_VPS-POL/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_RED_VPS-POL/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_RED_VPS-POL/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_RED_VPS-POL.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_RED_VPS-TRO", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_RED_VPS-TRO/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_RED_VPS-TRO/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_RED_VPS-TRO/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_RED_VPS-TRO.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_RED_VSA-CAO", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_RED_VSA-CAO/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_RED_VSA-CAO/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_RED_VSA-CAO/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_RED_VSA-CAO.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_RED_VSA-POL", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_RED_VSA-POL/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_RED_VSA-POL/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_RED_VSA-POL/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_RED_VSA-POL.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_RED_VSA-TRO", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_RED_VSA-TRO/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_RED_VSA-TRO/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_RED_VSA-TRO/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_RED_VSA-TRO.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SMA_TSI", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SMA_TSI/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SMA_TSI/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SMA_TSI/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_SMA_TSI.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SMA_VBL-CAO", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SMA_VBL-CAO/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SMA_VBL-CAO/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SMA_VBL-CAO/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_SMA_VBL-CAO.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SMA_VBL-POL", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SMA_VBL-POL/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SMA_VBL-POL/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SMA_VBL-POL/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_SMA_VBL-POL.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SMA_VBL-TRO", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SMA_VBL-TRO/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SMA_VBL-TRO/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SMA_VBL-TRO/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_SMA_VBL-TRO.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SMA_VPS-CAO", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SMA_VPS-CAO/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SMA_VPS-CAO/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SMA_VPS-CAO/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_SMA_VPS-CAO.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SMA_VPS-POL", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SMA_VPS-POL/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SMA_VPS-POL/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SMA_VPS-POL/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_SMA_VPS-POL.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SMA_VPS-TRO", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SMA_VPS-TRO/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SMA_VPS-TRO/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SMA_VPS-TRO/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_SMA_VPS-TRO.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SMA_VSA-CAO", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SMA_VSA-CAO/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SMA_VSA-CAO/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SMA_VSA-CAO/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_SMA_VSA-CAO.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SMA_VSA-POL", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SMA_VSA-POL/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SMA_VSA-POL/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SMA_VSA-POL/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_SMA_VSA-POL.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SMA_VSA-TRO", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SMA_VSA-TRO/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SMA_VSA-TRO/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SMA_VSA-TRO/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_SMA_VSA-TRO.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW1_TSI", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW1_TSI/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW1_TSI/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW1_TSI/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_SW1_TSI.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW1_VBL-CAO", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW1_VBL-CAO/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW1_VBL-CAO/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW1_VBL-CAO/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_SW1_VBL-CAO.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW1_VBL-POL", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW1_VBL-POL/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW1_VBL-POL/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW1_VBL-POL/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_SW1_VBL-POL.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW1_VBL-TRO", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW1_VBL-TRO/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW1_VBL-TRO/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW1_VBL-TRO/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_SW1_VBL-TRO.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW1_VPS-CAO", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW1_VPS-CAO/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW1_VPS-CAO/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW1_VPS-CAO/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_SW1_VPS-CAO.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW1_VPS-POL", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW1_VPS-POL/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW1_VPS-POL/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW1_VPS-POL/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_SW1_VPS-POL.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW1_VPS-TRO", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW1_VPS-TRO/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW1_VPS-TRO/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW1_VPS-TRO/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_SW1_VPS-TRO.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW1_VSA-CAO", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW1_VSA-CAO/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW1_VSA-CAO/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW1_VSA-CAO/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_SW1_VSA-CAO.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW1_VSA-POL", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW1_VSA-POL/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW1_VSA-POL/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW1_VSA-POL/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_SW1_VSA-POL.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW1_VSA-TRO", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW1_VSA-TRO/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW1_VSA-TRO/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW1_VSA-TRO/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_SW1_VSA-TRO.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW2_TSI", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW2_TSI/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW2_TSI/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW2_TSI/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_SW2_TSI.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW2_VBL-CAO", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW2_VBL-CAO/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW2_VBL-CAO/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW2_VBL-CAO/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_SW2_VBL-CAO.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW2_VBL-POL", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW2_VBL-POL/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW2_VBL-POL/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW2_VBL-POL/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_SW2_VBL-POL.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW2_VBL-TRO", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW2_VBL-TRO/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW2_VBL-TRO/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW2_VBL-TRO/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_SW2_VBL-TRO.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW2_VPS-CAO", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW2_VPS-CAO/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW2_VPS-CAO/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW2_VPS-CAO/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_SW2_VPS-CAO.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW2_VPS-POL", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW2_VPS-POL/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW2_VPS-POL/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW2_VPS-POL/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_SW2_VPS-POL.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW2_VPS-TRO", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW2_VPS-TRO/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW2_VPS-TRO/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW2_VPS-TRO/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_SW2_VPS-TRO.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW2_VSA-CAO", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW2_VSA-CAO/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW2_VSA-CAO/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW2_VSA-CAO/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_SW2_VSA-CAO.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW2_VSA-POL", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW2_VSA-POL/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW2_VSA-POL/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW2_VSA-POL/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_SW2_VSA-POL.tif.ovr", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW2_VSA-TRO", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW2_VSA-TRO/trend", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW2_VSA-TRO/trend/X0109_Y0102", + "trend/pyramid/1987-1989_001-365_HL_TSA_LNDLG_SW2_VSA-TRO/trend/X0109_Y0102/X0109_Y0102_1987-1989_001-365_HL_TSA_LNDLG_SW2_VSA-TRO.tif.ovr", + "trend/pyramid/versions.yml", + "untar", + "untar/Landsat_data", + "untar/dem", + "untar/wvdb" + ] + ], + "meta": { + "nf-test": "0.9.1", + "nextflow": "24.10.0" + }, + "timestamp": "2024-11-24T15:08:30.683117395" + } +} \ No newline at end of file diff --git a/tests/nextflow.config b/tests/nextflow.config new file mode 100644 index 0000000..c19b1ad --- /dev/null +++ b/tests/nextflow.config @@ -0,0 +1,5 @@ +/* +======================================================================================== + Nextflow config file for running tests +======================================================================================== +*/ diff --git a/tests/tweaked_params.nf.test b/tests/tweaked_params.nf.test new file mode 100644 index 0000000..f2a170c --- /dev/null +++ b/tests/tweaked_params.nf.test @@ -0,0 +1,32 @@ +nextflow_pipeline { + + name "Test pipeline with non-default parameters" + script "../main.nf" + + test("Params: --save_ard true --save_tsa true --return_tss true --sensors_level2 'LND05' --indexes 'RED NIR SWIR2' --end_date '1988-12-31'") { + + when { + params { + outdir = "$outputDir" + save_ard = true + save_tsa = true + return_tss = true + sensors_level2 = "LND05" + indexes = "RED NIR SWIR2" + end_date = "1988-12-31" + } + } + + then { + def stable_name = getAllFilesFromDir(params.outdir, relative: true, includeDir: true, ignore: ['pipeline_info/*.{html,json,txt}']) + assert workflow.success + assert snapshot( + workflow.trace.succeeded().size(), + stable_name, + ).match() + + } + + } + +} diff --git a/tests/tweaked_params.nf.test.snap b/tests/tweaked_params.nf.test.snap new file mode 100644 index 0000000..00dc346 --- /dev/null +++ b/tests/tweaked_params.nf.test.snap @@ -0,0 +1,947 @@ +{ + "Params: --save_ard true --save_tsa true --return_tss true --sensors_level2 'LND05' --indexes 'RED NIR SWIR2' --end_date '1988-12-31'": { + "content": [ + 168, + [ + "higher-level", + "higher-level/X0109_Y0102", + "higher-level/X0109_Y0102/param_files", + "higher-level/X0109_Y0102/param_files/trend_X0109_Y0102.prm", + "higher-level/X0109_Y0102/trend_files", + "higher-level/X0109_Y0102/trend_files/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_NIR_TSI.tif", + "higher-level/X0109_Y0102/trend_files/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_NIR_TSS.tif", + "higher-level/X0109_Y0102/trend_files/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_NIR_VBL-CAO.tif", + "higher-level/X0109_Y0102/trend_files/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_NIR_VBL-POL.tif", + "higher-level/X0109_Y0102/trend_files/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_NIR_VBL-TRO.tif", + "higher-level/X0109_Y0102/trend_files/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_NIR_VPS-CAO.tif", + "higher-level/X0109_Y0102/trend_files/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_NIR_VPS-POL.tif", + "higher-level/X0109_Y0102/trend_files/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_NIR_VPS-TRO.tif", + "higher-level/X0109_Y0102/trend_files/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_NIR_VSA-CAO.tif", + "higher-level/X0109_Y0102/trend_files/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_NIR_VSA-POL.tif", + "higher-level/X0109_Y0102/trend_files/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_NIR_VSA-TRO.tif", + "higher-level/X0109_Y0102/trend_files/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_RED_TSI.tif", + "higher-level/X0109_Y0102/trend_files/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_RED_TSS.tif", + "higher-level/X0109_Y0102/trend_files/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_RED_VBL-CAO.tif", + "higher-level/X0109_Y0102/trend_files/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_RED_VBL-POL.tif", + "higher-level/X0109_Y0102/trend_files/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_RED_VBL-TRO.tif", + "higher-level/X0109_Y0102/trend_files/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_RED_VPS-CAO.tif", + "higher-level/X0109_Y0102/trend_files/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_RED_VPS-POL.tif", + "higher-level/X0109_Y0102/trend_files/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_RED_VPS-TRO.tif", + "higher-level/X0109_Y0102/trend_files/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_RED_VSA-CAO.tif", + "higher-level/X0109_Y0102/trend_files/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_RED_VSA-POL.tif", + "higher-level/X0109_Y0102/trend_files/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_RED_VSA-TRO.tif", + "higher-level/X0109_Y0102/trend_files/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_SMA_TSI.tif", + "higher-level/X0109_Y0102/trend_files/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_SMA_TSS.tif", + "higher-level/X0109_Y0102/trend_files/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_SMA_VBL-CAO.tif", + "higher-level/X0109_Y0102/trend_files/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_SMA_VBL-POL.tif", + "higher-level/X0109_Y0102/trend_files/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_SMA_VBL-TRO.tif", + "higher-level/X0109_Y0102/trend_files/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_SMA_VPS-CAO.tif", + "higher-level/X0109_Y0102/trend_files/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_SMA_VPS-POL.tif", + "higher-level/X0109_Y0102/trend_files/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_SMA_VPS-TRO.tif", + "higher-level/X0109_Y0102/trend_files/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_SMA_VSA-CAO.tif", + "higher-level/X0109_Y0102/trend_files/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_SMA_VSA-POL.tif", + "higher-level/X0109_Y0102/trend_files/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_SMA_VSA-TRO.tif", + "higher-level/X0109_Y0102/trend_files/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_SW2_TSI.tif", + "higher-level/X0109_Y0102/trend_files/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_SW2_TSS.tif", + "higher-level/X0109_Y0102/trend_files/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_SW2_VBL-CAO.tif", + "higher-level/X0109_Y0102/trend_files/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_SW2_VBL-POL.tif", + "higher-level/X0109_Y0102/trend_files/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_SW2_VBL-TRO.tif", + "higher-level/X0109_Y0102/trend_files/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_SW2_VPS-CAO.tif", + "higher-level/X0109_Y0102/trend_files/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_SW2_VPS-POL.tif", + "higher-level/X0109_Y0102/trend_files/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_SW2_VPS-TRO.tif", + "higher-level/X0109_Y0102/trend_files/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_SW2_VSA-CAO.tif", + "higher-level/X0109_Y0102/trend_files/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_SW2_VSA-POL.tif", + "higher-level/X0109_Y0102/trend_files/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_SW2_VSA-TRO.tif", + "multiqc", + "multiqc/multiqc_data", + "multiqc/multiqc_data/multiqc.log", + "multiqc/multiqc_data/multiqc_citations.txt", + "multiqc/multiqc_data/multiqc_data.json", + "multiqc/multiqc_data/multiqc_software_versions.txt", + "multiqc/multiqc_data/multiqc_sources.txt", + "multiqc/multiqc_report.html", + "pipeline_info", + "pipeline_info/nf_core_pipeline_software_mqc_versions.yml", + "preparation", + "preparation/mask", + "preparation/mask/X0103_Y0101", + "preparation/mask/X0103_Y0101/aoi.tif", + "preparation/mask/X0103_Y0102", + "preparation/mask/X0103_Y0102/aoi.tif", + "preparation/mask/X0103_Y0103", + "preparation/mask/X0103_Y0103/aoi.tif", + "preparation/mask/X0103_Y0104", + "preparation/mask/X0103_Y0104/aoi.tif", + "preparation/mask/X0103_Y0105", + "preparation/mask/X0103_Y0105/aoi.tif", + "preparation/mask/X0104_Y0101", + "preparation/mask/X0104_Y0101/aoi.tif", + "preparation/mask/X0104_Y0102", + "preparation/mask/X0104_Y0102/aoi.tif", + "preparation/mask/X0104_Y0103", + "preparation/mask/X0104_Y0103/aoi.tif", + "preparation/mask/X0104_Y0104", + "preparation/mask/X0104_Y0104/aoi.tif", + "preparation/mask/X0104_Y0105", + "preparation/mask/X0104_Y0105/aoi.tif", + "preparation/mask/X0105_Y0101", + "preparation/mask/X0105_Y0101/aoi.tif", + "preparation/mask/X0105_Y0102", + "preparation/mask/X0105_Y0102/aoi.tif", + "preparation/mask/X0105_Y0103", + "preparation/mask/X0105_Y0103/aoi.tif", + "preparation/mask/X0105_Y0104", + "preparation/mask/X0105_Y0104/aoi.tif", + "preparation/mask/X0105_Y0105", + "preparation/mask/X0105_Y0105/aoi.tif", + "preparation/mask/X0106_Y0101", + "preparation/mask/X0106_Y0101/aoi.tif", + "preparation/mask/X0106_Y0102", + "preparation/mask/X0106_Y0102/aoi.tif", + "preparation/mask/X0106_Y0103", + "preparation/mask/X0106_Y0103/aoi.tif", + "preparation/mask/X0106_Y0104", + "preparation/mask/X0106_Y0104/aoi.tif", + "preparation/mask/X0106_Y0105", + "preparation/mask/X0106_Y0105/aoi.tif", + "preparation/mask/X0107_Y0101", + "preparation/mask/X0107_Y0101/aoi.tif", + "preparation/mask/X0107_Y0102", + "preparation/mask/X0107_Y0102/aoi.tif", + "preparation/mask/X0107_Y0103", + "preparation/mask/X0107_Y0103/aoi.tif", + "preparation/mask/X0107_Y0104", + "preparation/mask/X0107_Y0104/aoi.tif", + "preparation/mask/X0107_Y0105", + "preparation/mask/X0107_Y0105/aoi.tif", + "preparation/mask/X0108_Y0101", + "preparation/mask/X0108_Y0101/aoi.tif", + "preparation/mask/X0108_Y0102", + "preparation/mask/X0108_Y0102/aoi.tif", + "preparation/mask/X0108_Y0103", + "preparation/mask/X0108_Y0103/aoi.tif", + "preparation/mask/X0108_Y0104", + "preparation/mask/X0108_Y0104/aoi.tif", + "preparation/mask/X0108_Y0105", + "preparation/mask/X0108_Y0105/aoi.tif", + "preparation/mask/X0109_Y0101", + "preparation/mask/X0109_Y0101/aoi.tif", + "preparation/mask/X0109_Y0102", + "preparation/mask/X0109_Y0102/aoi.tif", + "preparation/mask/X0109_Y0103", + "preparation/mask/X0109_Y0103/aoi.tif", + "preparation/mask/X0109_Y0104", + "preparation/mask/X0109_Y0104/aoi.tif", + "preparation/mask/X0109_Y0105", + "preparation/mask/X0109_Y0105/aoi.tif", + "preparation/mask/X0110_Y0101", + "preparation/mask/X0110_Y0101/aoi.tif", + "preparation/mask/X0110_Y0102", + "preparation/mask/X0110_Y0102/aoi.tif", + "preparation/mask/X0110_Y0103", + "preparation/mask/X0110_Y0103/aoi.tif", + "preparation/mask/X0110_Y0104", + "preparation/mask/X0110_Y0104/aoi.tif", + "preparation/mask/X0110_Y0105", + "preparation/mask/X0110_Y0105/aoi.tif", + "preparation/mask/X0111_Y0101", + "preparation/mask/X0111_Y0101/aoi.tif", + "preparation/mask/X0111_Y0102", + "preparation/mask/X0111_Y0102/aoi.tif", + "preparation/mask/X0111_Y0103", + "preparation/mask/X0111_Y0103/aoi.tif", + "preparation/mask/X0111_Y0104", + "preparation/mask/X0111_Y0104/aoi.tif", + "preparation/mask/X0111_Y0105", + "preparation/mask/X0111_Y0105/aoi.tif", + "preparation/tile_allow.txt", + "preprocess", + "preprocess/LT04_L1TP_181036_19880130_20200917_02_T1", + "preprocess/LT04_L1TP_181036_19880130_20200917_02_T1/level2_ard", + "preprocess/LT04_L1TP_181036_19880130_20200917_02_T1/level2_ard/X0109_Y0102", + "preprocess/LT04_L1TP_181036_19880130_20200917_02_T1/level2_ard/X0109_Y0102/19880130_LEVEL2_LND04_BOA.tif", + "preprocess/LT04_L1TP_181036_19880130_20200917_02_T1/level2_ard/X0109_Y0102/19880130_LEVEL2_LND04_QAI.tif", + "preprocess/LT04_L1TP_181036_19880130_20200917_02_T1/logs", + "preprocess/LT04_L1TP_181036_19880130_20200917_02_T1/logs/level2_log", + "preprocess/LT04_L1TP_181036_19880130_20200917_02_T1/logs/level2_log/LT04_L1TP_181036_19880130_20200917_02_T1.log", + "preprocess/LT04_L1TP_181036_19880130_20200917_02_T1/param_files", + "preprocess/LT04_L1TP_181036_19880130_20200917_02_T1/param_files/LT04_L1TP_181036_19880130_20200917_02_T1.prm", + "preprocess/LT04_L1TP_181036_19881129_20200917_02_T1", + "preprocess/LT04_L1TP_181036_19881129_20200917_02_T1/level2_ard", + "preprocess/LT04_L1TP_181036_19881129_20200917_02_T1/level2_ard/X0109_Y0102", + "preprocess/LT04_L1TP_181036_19881129_20200917_02_T1/level2_ard/X0109_Y0102/19881129_LEVEL2_LND04_BOA.tif", + "preprocess/LT04_L1TP_181036_19881129_20200917_02_T1/level2_ard/X0109_Y0102/19881129_LEVEL2_LND04_QAI.tif", + "preprocess/LT04_L1TP_181036_19881129_20200917_02_T1/logs", + "preprocess/LT04_L1TP_181036_19881129_20200917_02_T1/logs/level2_log", + "preprocess/LT04_L1TP_181036_19881129_20200917_02_T1/logs/level2_log/LT04_L1TP_181036_19881129_20200917_02_T1.log", + "preprocess/LT04_L1TP_181036_19881129_20200917_02_T1/param_files", + "preprocess/LT04_L1TP_181036_19881129_20200917_02_T1/param_files/LT04_L1TP_181036_19881129_20200917_02_T1.prm", + "preprocess/LT04_L1TP_181036_19881231_20200916_02_T1", + "preprocess/LT04_L1TP_181036_19881231_20200916_02_T1/logs", + "preprocess/LT04_L1TP_181036_19881231_20200916_02_T1/logs/level2_log", + "preprocess/LT04_L1TP_181036_19881231_20200916_02_T1/logs/level2_log/LT04_L1TP_181036_19881231_20200916_02_T1.log", + "preprocess/LT04_L1TP_181036_19881231_20200916_02_T1/param_files", + "preprocess/LT04_L1TP_181036_19881231_20200916_02_T1/param_files/LT04_L1TP_181036_19881231_20200916_02_T1.prm", + "preprocess/LT05_L1TP_181036_19870119_20201014_02_T1", + "preprocess/LT05_L1TP_181036_19870119_20201014_02_T1/level2_ard", + "preprocess/LT05_L1TP_181036_19870119_20201014_02_T1/level2_ard/X0109_Y0102", + "preprocess/LT05_L1TP_181036_19870119_20201014_02_T1/level2_ard/X0109_Y0102/19870119_LEVEL2_LND05_BOA.tif", + "preprocess/LT05_L1TP_181036_19870119_20201014_02_T1/level2_ard/X0109_Y0102/19870119_LEVEL2_LND05_QAI.tif", + "preprocess/LT05_L1TP_181036_19870119_20201014_02_T1/logs", + "preprocess/LT05_L1TP_181036_19870119_20201014_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19870119_20201014_02_T1/logs/level2_log/LT05_L1TP_181036_19870119_20201014_02_T1.log", + "preprocess/LT05_L1TP_181036_19870119_20201014_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19870119_20201014_02_T1/param_files/LT05_L1TP_181036_19870119_20201014_02_T1.prm", + "preprocess/LT05_L1TP_181036_19870204_20201014_02_T1", + "preprocess/LT05_L1TP_181036_19870204_20201014_02_T1/logs", + "preprocess/LT05_L1TP_181036_19870204_20201014_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19870204_20201014_02_T1/logs/level2_log/LT05_L1TP_181036_19870204_20201014_02_T1.log", + "preprocess/LT05_L1TP_181036_19870204_20201014_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19870204_20201014_02_T1/param_files/LT05_L1TP_181036_19870204_20201014_02_T1.prm", + "preprocess/LT05_L1TP_181036_19870324_20211112_02_T1", + "preprocess/LT05_L1TP_181036_19870324_20211112_02_T1/level2_ard", + "preprocess/LT05_L1TP_181036_19870324_20211112_02_T1/level2_ard/X0109_Y0102", + "preprocess/LT05_L1TP_181036_19870324_20211112_02_T1/level2_ard/X0109_Y0102/19870324_LEVEL2_LND05_BOA.tif", + "preprocess/LT05_L1TP_181036_19870324_20211112_02_T1/level2_ard/X0109_Y0102/19870324_LEVEL2_LND05_QAI.tif", + "preprocess/LT05_L1TP_181036_19870324_20211112_02_T1/logs", + "preprocess/LT05_L1TP_181036_19870324_20211112_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19870324_20211112_02_T1/logs/level2_log/LT05_L1TP_181036_19870324_20211112_02_T1.log", + "preprocess/LT05_L1TP_181036_19870324_20211112_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19870324_20211112_02_T1/param_files/LT05_L1TP_181036_19870324_20211112_02_T1.prm", + "preprocess/LT05_L1TP_181036_19870409_20201014_02_T1", + "preprocess/LT05_L1TP_181036_19870409_20201014_02_T1/level2_ard", + "preprocess/LT05_L1TP_181036_19870409_20201014_02_T1/level2_ard/X0109_Y0102", + "preprocess/LT05_L1TP_181036_19870409_20201014_02_T1/level2_ard/X0109_Y0102/19870409_LEVEL2_LND05_BOA.tif", + "preprocess/LT05_L1TP_181036_19870409_20201014_02_T1/level2_ard/X0109_Y0102/19870409_LEVEL2_LND05_QAI.tif", + "preprocess/LT05_L1TP_181036_19870409_20201014_02_T1/logs", + "preprocess/LT05_L1TP_181036_19870409_20201014_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19870409_20201014_02_T1/logs/level2_log/LT05_L1TP_181036_19870409_20201014_02_T1.log", + "preprocess/LT05_L1TP_181036_19870409_20201014_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19870409_20201014_02_T1/param_files/LT05_L1TP_181036_19870409_20201014_02_T1.prm", + "preprocess/LT05_L1TP_181036_19870425_20201014_02_T1", + "preprocess/LT05_L1TP_181036_19870425_20201014_02_T1/level2_ard", + "preprocess/LT05_L1TP_181036_19870425_20201014_02_T1/level2_ard/X0109_Y0102", + "preprocess/LT05_L1TP_181036_19870425_20201014_02_T1/level2_ard/X0109_Y0102/19870425_LEVEL2_LND05_BOA.tif", + "preprocess/LT05_L1TP_181036_19870425_20201014_02_T1/level2_ard/X0109_Y0102/19870425_LEVEL2_LND05_QAI.tif", + "preprocess/LT05_L1TP_181036_19870425_20201014_02_T1/logs", + "preprocess/LT05_L1TP_181036_19870425_20201014_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19870425_20201014_02_T1/logs/level2_log/LT05_L1TP_181036_19870425_20201014_02_T1.log", + "preprocess/LT05_L1TP_181036_19870425_20201014_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19870425_20201014_02_T1/param_files/LT05_L1TP_181036_19870425_20201014_02_T1.prm", + "preprocess/LT05_L1TP_181036_19870511_20201014_02_T1", + "preprocess/LT05_L1TP_181036_19870511_20201014_02_T1/level2_ard", + "preprocess/LT05_L1TP_181036_19870511_20201014_02_T1/level2_ard/X0109_Y0102", + "preprocess/LT05_L1TP_181036_19870511_20201014_02_T1/level2_ard/X0109_Y0102/19870511_LEVEL2_LND05_BOA.tif", + "preprocess/LT05_L1TP_181036_19870511_20201014_02_T1/level2_ard/X0109_Y0102/19870511_LEVEL2_LND05_QAI.tif", + "preprocess/LT05_L1TP_181036_19870511_20201014_02_T1/logs", + "preprocess/LT05_L1TP_181036_19870511_20201014_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19870511_20201014_02_T1/logs/level2_log/LT05_L1TP_181036_19870511_20201014_02_T1.log", + "preprocess/LT05_L1TP_181036_19870511_20201014_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19870511_20201014_02_T1/param_files/LT05_L1TP_181036_19870511_20201014_02_T1.prm", + "preprocess/LT05_L1TP_181036_19870527_20211113_02_T1", + "preprocess/LT05_L1TP_181036_19870527_20211113_02_T1/level2_ard", + "preprocess/LT05_L1TP_181036_19870527_20211113_02_T1/level2_ard/X0109_Y0102", + "preprocess/LT05_L1TP_181036_19870527_20211113_02_T1/level2_ard/X0109_Y0102/19870527_LEVEL2_LND05_BOA.tif", + "preprocess/LT05_L1TP_181036_19870527_20211113_02_T1/level2_ard/X0109_Y0102/19870527_LEVEL2_LND05_QAI.tif", + "preprocess/LT05_L1TP_181036_19870527_20211113_02_T1/logs", + "preprocess/LT05_L1TP_181036_19870527_20211113_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19870527_20211113_02_T1/logs/level2_log/LT05_L1TP_181036_19870527_20211113_02_T1.log", + "preprocess/LT05_L1TP_181036_19870527_20211113_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19870527_20211113_02_T1/param_files/LT05_L1TP_181036_19870527_20211113_02_T1.prm", + "preprocess/LT05_L1TP_181036_19870612_20201014_02_T1", + "preprocess/LT05_L1TP_181036_19870612_20201014_02_T1/level2_ard", + "preprocess/LT05_L1TP_181036_19870612_20201014_02_T1/level2_ard/X0109_Y0102", + "preprocess/LT05_L1TP_181036_19870612_20201014_02_T1/level2_ard/X0109_Y0102/19870612_LEVEL2_LND05_BOA.tif", + "preprocess/LT05_L1TP_181036_19870612_20201014_02_T1/level2_ard/X0109_Y0102/19870612_LEVEL2_LND05_QAI.tif", + "preprocess/LT05_L1TP_181036_19870612_20201014_02_T1/logs", + "preprocess/LT05_L1TP_181036_19870612_20201014_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19870612_20201014_02_T1/logs/level2_log/LT05_L1TP_181036_19870612_20201014_02_T1.log", + "preprocess/LT05_L1TP_181036_19870612_20201014_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19870612_20201014_02_T1/param_files/LT05_L1TP_181036_19870612_20201014_02_T1.prm", + "preprocess/LT05_L1TP_181036_19870628_20201014_02_T1", + "preprocess/LT05_L1TP_181036_19870628_20201014_02_T1/level2_ard", + "preprocess/LT05_L1TP_181036_19870628_20201014_02_T1/level2_ard/X0109_Y0102", + "preprocess/LT05_L1TP_181036_19870628_20201014_02_T1/level2_ard/X0109_Y0102/19870628_LEVEL2_LND05_BOA.tif", + "preprocess/LT05_L1TP_181036_19870628_20201014_02_T1/level2_ard/X0109_Y0102/19870628_LEVEL2_LND05_QAI.tif", + "preprocess/LT05_L1TP_181036_19870628_20201014_02_T1/logs", + "preprocess/LT05_L1TP_181036_19870628_20201014_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19870628_20201014_02_T1/logs/level2_log/LT05_L1TP_181036_19870628_20201014_02_T1.log", + "preprocess/LT05_L1TP_181036_19870628_20201014_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19870628_20201014_02_T1/param_files/LT05_L1TP_181036_19870628_20201014_02_T1.prm", + "preprocess/LT05_L1TP_181036_19870714_20201014_02_T1", + "preprocess/LT05_L1TP_181036_19870714_20201014_02_T1/level2_ard", + "preprocess/LT05_L1TP_181036_19870714_20201014_02_T1/level2_ard/X0109_Y0102", + "preprocess/LT05_L1TP_181036_19870714_20201014_02_T1/level2_ard/X0109_Y0102/19870714_LEVEL2_LND05_BOA.tif", + "preprocess/LT05_L1TP_181036_19870714_20201014_02_T1/level2_ard/X0109_Y0102/19870714_LEVEL2_LND05_QAI.tif", + "preprocess/LT05_L1TP_181036_19870714_20201014_02_T1/logs", + "preprocess/LT05_L1TP_181036_19870714_20201014_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19870714_20201014_02_T1/logs/level2_log/LT05_L1TP_181036_19870714_20201014_02_T1.log", + "preprocess/LT05_L1TP_181036_19870714_20201014_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19870714_20201014_02_T1/param_files/LT05_L1TP_181036_19870714_20201014_02_T1.prm", + "preprocess/LT05_L1TP_181036_19870730_20201014_02_T1", + "preprocess/LT05_L1TP_181036_19870730_20201014_02_T1/level2_ard", + "preprocess/LT05_L1TP_181036_19870730_20201014_02_T1/level2_ard/X0109_Y0102", + "preprocess/LT05_L1TP_181036_19870730_20201014_02_T1/level2_ard/X0109_Y0102/19870730_LEVEL2_LND05_BOA.tif", + "preprocess/LT05_L1TP_181036_19870730_20201014_02_T1/level2_ard/X0109_Y0102/19870730_LEVEL2_LND05_QAI.tif", + "preprocess/LT05_L1TP_181036_19870730_20201014_02_T1/logs", + "preprocess/LT05_L1TP_181036_19870730_20201014_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19870730_20201014_02_T1/logs/level2_log/LT05_L1TP_181036_19870730_20201014_02_T1.log", + "preprocess/LT05_L1TP_181036_19870730_20201014_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19870730_20201014_02_T1/param_files/LT05_L1TP_181036_19870730_20201014_02_T1.prm", + "preprocess/LT05_L1TP_181036_19870815_20201014_02_T1", + "preprocess/LT05_L1TP_181036_19870815_20201014_02_T1/level2_ard", + "preprocess/LT05_L1TP_181036_19870815_20201014_02_T1/level2_ard/X0109_Y0102", + "preprocess/LT05_L1TP_181036_19870815_20201014_02_T1/level2_ard/X0109_Y0102/19870815_LEVEL2_LND05_BOA.tif", + "preprocess/LT05_L1TP_181036_19870815_20201014_02_T1/level2_ard/X0109_Y0102/19870815_LEVEL2_LND05_QAI.tif", + "preprocess/LT05_L1TP_181036_19870815_20201014_02_T1/logs", + "preprocess/LT05_L1TP_181036_19870815_20201014_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19870815_20201014_02_T1/logs/level2_log/LT05_L1TP_181036_19870815_20201014_02_T1.log", + "preprocess/LT05_L1TP_181036_19870815_20201014_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19870815_20201014_02_T1/param_files/LT05_L1TP_181036_19870815_20201014_02_T1.prm", + "preprocess/LT05_L1TP_181036_19870831_20201014_02_T1", + "preprocess/LT05_L1TP_181036_19870831_20201014_02_T1/level2_ard", + "preprocess/LT05_L1TP_181036_19870831_20201014_02_T1/level2_ard/X0109_Y0102", + "preprocess/LT05_L1TP_181036_19870831_20201014_02_T1/level2_ard/X0109_Y0102/19870831_LEVEL2_LND05_BOA.tif", + "preprocess/LT05_L1TP_181036_19870831_20201014_02_T1/level2_ard/X0109_Y0102/19870831_LEVEL2_LND05_QAI.tif", + "preprocess/LT05_L1TP_181036_19870831_20201014_02_T1/logs", + "preprocess/LT05_L1TP_181036_19870831_20201014_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19870831_20201014_02_T1/logs/level2_log/LT05_L1TP_181036_19870831_20201014_02_T1.log", + "preprocess/LT05_L1TP_181036_19870831_20201014_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19870831_20201014_02_T1/param_files/LT05_L1TP_181036_19870831_20201014_02_T1.prm", + "preprocess/LT05_L1TP_181036_19870916_20201014_02_T1", + "preprocess/LT05_L1TP_181036_19870916_20201014_02_T1/level2_ard", + "preprocess/LT05_L1TP_181036_19870916_20201014_02_T1/level2_ard/X0109_Y0102", + "preprocess/LT05_L1TP_181036_19870916_20201014_02_T1/level2_ard/X0109_Y0102/19870916_LEVEL2_LND05_BOA.tif", + "preprocess/LT05_L1TP_181036_19870916_20201014_02_T1/level2_ard/X0109_Y0102/19870916_LEVEL2_LND05_QAI.tif", + "preprocess/LT05_L1TP_181036_19870916_20201014_02_T1/logs", + "preprocess/LT05_L1TP_181036_19870916_20201014_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19870916_20201014_02_T1/logs/level2_log/LT05_L1TP_181036_19870916_20201014_02_T1.log", + "preprocess/LT05_L1TP_181036_19870916_20201014_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19870916_20201014_02_T1/param_files/LT05_L1TP_181036_19870916_20201014_02_T1.prm", + "preprocess/LT05_L1TP_181036_19871002_20201014_02_T1", + "preprocess/LT05_L1TP_181036_19871002_20201014_02_T1/level2_ard", + "preprocess/LT05_L1TP_181036_19871002_20201014_02_T1/level2_ard/X0109_Y0102", + "preprocess/LT05_L1TP_181036_19871002_20201014_02_T1/level2_ard/X0109_Y0102/19871002_LEVEL2_LND05_BOA.tif", + "preprocess/LT05_L1TP_181036_19871002_20201014_02_T1/level2_ard/X0109_Y0102/19871002_LEVEL2_LND05_QAI.tif", + "preprocess/LT05_L1TP_181036_19871002_20201014_02_T1/logs", + "preprocess/LT05_L1TP_181036_19871002_20201014_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19871002_20201014_02_T1/logs/level2_log/LT05_L1TP_181036_19871002_20201014_02_T1.log", + "preprocess/LT05_L1TP_181036_19871002_20201014_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19871002_20201014_02_T1/param_files/LT05_L1TP_181036_19871002_20201014_02_T1.prm", + "preprocess/LT05_L1TP_181036_19871205_20201014_02_T1", + "preprocess/LT05_L1TP_181036_19871205_20201014_02_T1/logs", + "preprocess/LT05_L1TP_181036_19871205_20201014_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19871205_20201014_02_T1/logs/level2_log/LT05_L1TP_181036_19871205_20201014_02_T1.log", + "preprocess/LT05_L1TP_181036_19871205_20201014_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19871205_20201014_02_T1/param_files/LT05_L1TP_181036_19871205_20201014_02_T1.prm", + "preprocess/LT05_L1TP_181036_19880106_20211116_02_T1", + "preprocess/LT05_L1TP_181036_19880106_20211116_02_T1/level2_ard", + "preprocess/LT05_L1TP_181036_19880106_20211116_02_T1/level2_ard/X0109_Y0102", + "preprocess/LT05_L1TP_181036_19880106_20211116_02_T1/level2_ard/X0109_Y0102/19880106_LEVEL2_LND05_BOA.tif", + "preprocess/LT05_L1TP_181036_19880106_20211116_02_T1/level2_ard/X0109_Y0102/19880106_LEVEL2_LND05_QAI.tif", + "preprocess/LT05_L1TP_181036_19880106_20211116_02_T1/logs", + "preprocess/LT05_L1TP_181036_19880106_20211116_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19880106_20211116_02_T1/logs/level2_log/LT05_L1TP_181036_19880106_20211116_02_T1.log", + "preprocess/LT05_L1TP_181036_19880106_20211116_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19880106_20211116_02_T1/param_files/LT05_L1TP_181036_19880106_20211116_02_T1.prm", + "preprocess/LT05_L1TP_181036_19880122_20200917_02_T1", + "preprocess/LT05_L1TP_181036_19880122_20200917_02_T1/level2_ard", + "preprocess/LT05_L1TP_181036_19880122_20200917_02_T1/level2_ard/X0109_Y0102", + "preprocess/LT05_L1TP_181036_19880122_20200917_02_T1/level2_ard/X0109_Y0102/19880122_LEVEL2_LND05_BOA.tif", + "preprocess/LT05_L1TP_181036_19880122_20200917_02_T1/level2_ard/X0109_Y0102/19880122_LEVEL2_LND05_QAI.tif", + "preprocess/LT05_L1TP_181036_19880122_20200917_02_T1/logs", + "preprocess/LT05_L1TP_181036_19880122_20200917_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19880122_20200917_02_T1/logs/level2_log/LT05_L1TP_181036_19880122_20200917_02_T1.log", + "preprocess/LT05_L1TP_181036_19880122_20200917_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19880122_20200917_02_T1/param_files/LT05_L1TP_181036_19880122_20200917_02_T1.prm", + "preprocess/LT05_L1TP_181036_19880223_20200917_02_T1", + "preprocess/LT05_L1TP_181036_19880223_20200917_02_T1/logs", + "preprocess/LT05_L1TP_181036_19880223_20200917_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19880223_20200917_02_T1/logs/level2_log/LT05_L1TP_181036_19880223_20200917_02_T1.log", + "preprocess/LT05_L1TP_181036_19880223_20200917_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19880223_20200917_02_T1/param_files/LT05_L1TP_181036_19880223_20200917_02_T1.prm", + "preprocess/LT05_L1TP_181036_19880310_20200917_02_T1", + "preprocess/LT05_L1TP_181036_19880310_20200917_02_T1/logs", + "preprocess/LT05_L1TP_181036_19880310_20200917_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19880310_20200917_02_T1/logs/level2_log/LT05_L1TP_181036_19880310_20200917_02_T1.log", + "preprocess/LT05_L1TP_181036_19880310_20200917_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19880310_20200917_02_T1/param_files/LT05_L1TP_181036_19880310_20200917_02_T1.prm", + "preprocess/LT05_L1TP_181036_19880326_20200917_02_T1", + "preprocess/LT05_L1TP_181036_19880326_20200917_02_T1/level2_ard", + "preprocess/LT05_L1TP_181036_19880326_20200917_02_T1/level2_ard/X0109_Y0102", + "preprocess/LT05_L1TP_181036_19880326_20200917_02_T1/level2_ard/X0109_Y0102/19880326_LEVEL2_LND05_BOA.tif", + "preprocess/LT05_L1TP_181036_19880326_20200917_02_T1/level2_ard/X0109_Y0102/19880326_LEVEL2_LND05_QAI.tif", + "preprocess/LT05_L1TP_181036_19880326_20200917_02_T1/logs", + "preprocess/LT05_L1TP_181036_19880326_20200917_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19880326_20200917_02_T1/logs/level2_log/LT05_L1TP_181036_19880326_20200917_02_T1.log", + "preprocess/LT05_L1TP_181036_19880326_20200917_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19880326_20200917_02_T1/param_files/LT05_L1TP_181036_19880326_20200917_02_T1.prm", + "preprocess/LT05_L1TP_181036_19880411_20200917_02_T1", + "preprocess/LT05_L1TP_181036_19880411_20200917_02_T1/level2_ard", + "preprocess/LT05_L1TP_181036_19880411_20200917_02_T1/level2_ard/X0109_Y0102", + "preprocess/LT05_L1TP_181036_19880411_20200917_02_T1/level2_ard/X0109_Y0102/19880411_LEVEL2_LND05_BOA.tif", + "preprocess/LT05_L1TP_181036_19880411_20200917_02_T1/level2_ard/X0109_Y0102/19880411_LEVEL2_LND05_QAI.tif", + "preprocess/LT05_L1TP_181036_19880411_20200917_02_T1/logs", + "preprocess/LT05_L1TP_181036_19880411_20200917_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19880411_20200917_02_T1/logs/level2_log/LT05_L1TP_181036_19880411_20200917_02_T1.log", + "preprocess/LT05_L1TP_181036_19880411_20200917_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19880411_20200917_02_T1/param_files/LT05_L1TP_181036_19880411_20200917_02_T1.prm", + "preprocess/LT05_L1TP_181036_19880529_20200917_02_T1", + "preprocess/LT05_L1TP_181036_19880529_20200917_02_T1/level2_ard", + "preprocess/LT05_L1TP_181036_19880529_20200917_02_T1/level2_ard/X0109_Y0102", + "preprocess/LT05_L1TP_181036_19880529_20200917_02_T1/level2_ard/X0109_Y0102/19880529_LEVEL2_LND05_BOA.tif", + "preprocess/LT05_L1TP_181036_19880529_20200917_02_T1/level2_ard/X0109_Y0102/19880529_LEVEL2_LND05_QAI.tif", + "preprocess/LT05_L1TP_181036_19880529_20200917_02_T1/logs", + "preprocess/LT05_L1TP_181036_19880529_20200917_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19880529_20200917_02_T1/logs/level2_log/LT05_L1TP_181036_19880529_20200917_02_T1.log", + "preprocess/LT05_L1TP_181036_19880529_20200917_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19880529_20200917_02_T1/param_files/LT05_L1TP_181036_19880529_20200917_02_T1.prm", + "preprocess/LT05_L1TP_181036_19880614_20211118_02_T1", + "preprocess/LT05_L1TP_181036_19880614_20211118_02_T1/level2_ard", + "preprocess/LT05_L1TP_181036_19880614_20211118_02_T1/level2_ard/X0109_Y0102", + "preprocess/LT05_L1TP_181036_19880614_20211118_02_T1/level2_ard/X0109_Y0102/19880614_LEVEL2_LND05_BOA.tif", + "preprocess/LT05_L1TP_181036_19880614_20211118_02_T1/level2_ard/X0109_Y0102/19880614_LEVEL2_LND05_QAI.tif", + "preprocess/LT05_L1TP_181036_19880614_20211118_02_T1/logs", + "preprocess/LT05_L1TP_181036_19880614_20211118_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19880614_20211118_02_T1/logs/level2_log/LT05_L1TP_181036_19880614_20211118_02_T1.log", + "preprocess/LT05_L1TP_181036_19880614_20211118_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19880614_20211118_02_T1/param_files/LT05_L1TP_181036_19880614_20211118_02_T1.prm", + "preprocess/LT05_L1TP_181036_19880630_20200917_02_T1", + "preprocess/LT05_L1TP_181036_19880630_20200917_02_T1/level2_ard", + "preprocess/LT05_L1TP_181036_19880630_20200917_02_T1/level2_ard/X0109_Y0102", + "preprocess/LT05_L1TP_181036_19880630_20200917_02_T1/level2_ard/X0109_Y0102/19880630_LEVEL2_LND05_BOA.tif", + "preprocess/LT05_L1TP_181036_19880630_20200917_02_T1/level2_ard/X0109_Y0102/19880630_LEVEL2_LND05_QAI.tif", + "preprocess/LT05_L1TP_181036_19880630_20200917_02_T1/logs", + "preprocess/LT05_L1TP_181036_19880630_20200917_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19880630_20200917_02_T1/logs/level2_log/LT05_L1TP_181036_19880630_20200917_02_T1.log", + "preprocess/LT05_L1TP_181036_19880630_20200917_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19880630_20200917_02_T1/param_files/LT05_L1TP_181036_19880630_20200917_02_T1.prm", + "preprocess/LT05_L1TP_181036_19880716_20200917_02_T1", + "preprocess/LT05_L1TP_181036_19880716_20200917_02_T1/level2_ard", + "preprocess/LT05_L1TP_181036_19880716_20200917_02_T1/level2_ard/X0109_Y0102", + "preprocess/LT05_L1TP_181036_19880716_20200917_02_T1/level2_ard/X0109_Y0102/19880716_LEVEL2_LND05_BOA.tif", + "preprocess/LT05_L1TP_181036_19880716_20200917_02_T1/level2_ard/X0109_Y0102/19880716_LEVEL2_LND05_QAI.tif", + "preprocess/LT05_L1TP_181036_19880716_20200917_02_T1/logs", + "preprocess/LT05_L1TP_181036_19880716_20200917_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19880716_20200917_02_T1/logs/level2_log/LT05_L1TP_181036_19880716_20200917_02_T1.log", + "preprocess/LT05_L1TP_181036_19880716_20200917_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19880716_20200917_02_T1/param_files/LT05_L1TP_181036_19880716_20200917_02_T1.prm", + "preprocess/LT05_L1TP_181036_19880801_20200917_02_T1", + "preprocess/LT05_L1TP_181036_19880801_20200917_02_T1/level2_ard", + "preprocess/LT05_L1TP_181036_19880801_20200917_02_T1/level2_ard/X0109_Y0102", + "preprocess/LT05_L1TP_181036_19880801_20200917_02_T1/level2_ard/X0109_Y0102/19880801_LEVEL2_LND05_BOA.tif", + "preprocess/LT05_L1TP_181036_19880801_20200917_02_T1/level2_ard/X0109_Y0102/19880801_LEVEL2_LND05_QAI.tif", + "preprocess/LT05_L1TP_181036_19880801_20200917_02_T1/logs", + "preprocess/LT05_L1TP_181036_19880801_20200917_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19880801_20200917_02_T1/logs/level2_log/LT05_L1TP_181036_19880801_20200917_02_T1.log", + "preprocess/LT05_L1TP_181036_19880801_20200917_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19880801_20200917_02_T1/param_files/LT05_L1TP_181036_19880801_20200917_02_T1.prm", + "preprocess/LT05_L1TP_181036_19880902_20211118_02_T1", + "preprocess/LT05_L1TP_181036_19880902_20211118_02_T1/level2_ard", + "preprocess/LT05_L1TP_181036_19880902_20211118_02_T1/level2_ard/X0109_Y0102", + "preprocess/LT05_L1TP_181036_19880902_20211118_02_T1/level2_ard/X0109_Y0102/19880902_LEVEL2_LND05_BOA.tif", + "preprocess/LT05_L1TP_181036_19880902_20211118_02_T1/level2_ard/X0109_Y0102/19880902_LEVEL2_LND05_QAI.tif", + "preprocess/LT05_L1TP_181036_19880902_20211118_02_T1/logs", + "preprocess/LT05_L1TP_181036_19880902_20211118_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19880902_20211118_02_T1/logs/level2_log/LT05_L1TP_181036_19880902_20211118_02_T1.log", + "preprocess/LT05_L1TP_181036_19880902_20211118_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19880902_20211118_02_T1/param_files/LT05_L1TP_181036_19880902_20211118_02_T1.prm", + "preprocess/LT05_L1TP_181036_19880918_20200917_02_T1", + "preprocess/LT05_L1TP_181036_19880918_20200917_02_T1/logs", + "preprocess/LT05_L1TP_181036_19880918_20200917_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19880918_20200917_02_T1/logs/level2_log/LT05_L1TP_181036_19880918_20200917_02_T1.log", + "preprocess/LT05_L1TP_181036_19880918_20200917_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19880918_20200917_02_T1/param_files/LT05_L1TP_181036_19880918_20200917_02_T1.prm", + "preprocess/LT05_L1TP_181036_19881004_20200917_02_T1", + "preprocess/LT05_L1TP_181036_19881004_20200917_02_T1/logs", + "preprocess/LT05_L1TP_181036_19881004_20200917_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19881004_20200917_02_T1/logs/level2_log/LT05_L1TP_181036_19881004_20200917_02_T1.log", + "preprocess/LT05_L1TP_181036_19881004_20200917_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19881004_20200917_02_T1/param_files/LT05_L1TP_181036_19881004_20200917_02_T1.prm", + "preprocess/LT05_L1TP_181036_19881020_20200917_02_T1", + "preprocess/LT05_L1TP_181036_19881020_20200917_02_T1/level2_ard", + "preprocess/LT05_L1TP_181036_19881020_20200917_02_T1/level2_ard/X0109_Y0102", + "preprocess/LT05_L1TP_181036_19881020_20200917_02_T1/level2_ard/X0109_Y0102/19881020_LEVEL2_LND05_BOA.tif", + "preprocess/LT05_L1TP_181036_19881020_20200917_02_T1/level2_ard/X0109_Y0102/19881020_LEVEL2_LND05_QAI.tif", + "preprocess/LT05_L1TP_181036_19881020_20200917_02_T1/logs", + "preprocess/LT05_L1TP_181036_19881020_20200917_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19881020_20200917_02_T1/logs/level2_log/LT05_L1TP_181036_19881020_20200917_02_T1.log", + "preprocess/LT05_L1TP_181036_19881020_20200917_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19881020_20200917_02_T1/param_files/LT05_L1TP_181036_19881020_20200917_02_T1.prm", + "preprocess/LT05_L1TP_181036_19881121_20211118_02_T1", + "preprocess/LT05_L1TP_181036_19881121_20211118_02_T1/level2_ard", + "preprocess/LT05_L1TP_181036_19881121_20211118_02_T1/level2_ard/X0109_Y0102", + "preprocess/LT05_L1TP_181036_19881121_20211118_02_T1/level2_ard/X0109_Y0102/19881121_LEVEL2_LND05_BOA.tif", + "preprocess/LT05_L1TP_181036_19881121_20211118_02_T1/level2_ard/X0109_Y0102/19881121_LEVEL2_LND05_QAI.tif", + "preprocess/LT05_L1TP_181036_19881121_20211118_02_T1/logs", + "preprocess/LT05_L1TP_181036_19881121_20211118_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19881121_20211118_02_T1/logs/level2_log/LT05_L1TP_181036_19881121_20211118_02_T1.log", + "preprocess/LT05_L1TP_181036_19881121_20211118_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19881121_20211118_02_T1/param_files/LT05_L1TP_181036_19881121_20211118_02_T1.prm", + "preprocess/LT05_L1TP_181036_19881223_20211119_02_T1", + "preprocess/LT05_L1TP_181036_19881223_20211119_02_T1/logs", + "preprocess/LT05_L1TP_181036_19881223_20211119_02_T1/logs/level2_log", + "preprocess/LT05_L1TP_181036_19881223_20211119_02_T1/logs/level2_log/LT05_L1TP_181036_19881223_20211119_02_T1.log", + "preprocess/LT05_L1TP_181036_19881223_20211119_02_T1/param_files", + "preprocess/LT05_L1TP_181036_19881223_20211119_02_T1/param_files/LT05_L1TP_181036_19881223_20211119_02_T1.prm", + "trend", + "trend/mosaic", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_TSI", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_TSI/X0109_Y0102", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_TSI/X0109_Y0102/1987-1988_001-365_HL_TSA_LNDLG_NIR_TSI.tif", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_TSI/X0109_Y0102/1987-1988_001-365_HL_TSA_LNDLG_NIR_TSI.tif.aux.xml", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_TSI/mosaic", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_TSI/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_TSI.vrt", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_TSI/versions.yml", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_TSS", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_TSS/X0109_Y0102", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_TSS/X0109_Y0102/1987-1988_001-365_HL_TSA_LNDLG_NIR_TSS.tif", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_TSS/mosaic", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_TSS/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_TSS.vrt", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_TSS/versions.yml", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_VBL-CAO", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_VBL-CAO/X0109_Y0102", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_VBL-CAO/X0109_Y0102/1987-1988_001-365_HL_TSA_LNDLG_NIR_VBL-CAO.tif", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_VBL-CAO/mosaic", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_VBL-CAO/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_VBL-CAO.vrt", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_VBL-CAO/versions.yml", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_VBL-POL", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_VBL-POL/X0109_Y0102", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_VBL-POL/X0109_Y0102/1987-1988_001-365_HL_TSA_LNDLG_NIR_VBL-POL.tif", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_VBL-POL/mosaic", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_VBL-POL/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_VBL-POL.vrt", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_VBL-POL/versions.yml", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_VBL-TRO", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_VBL-TRO/X0109_Y0102", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_VBL-TRO/X0109_Y0102/1987-1988_001-365_HL_TSA_LNDLG_NIR_VBL-TRO.tif", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_VBL-TRO/mosaic", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_VBL-TRO/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_VBL-TRO.vrt", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_VBL-TRO/versions.yml", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_VPS-CAO", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_VPS-CAO/X0109_Y0102", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_VPS-CAO/X0109_Y0102/1987-1988_001-365_HL_TSA_LNDLG_NIR_VPS-CAO.tif", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_VPS-CAO/mosaic", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_VPS-CAO/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_VPS-CAO.vrt", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_VPS-CAO/versions.yml", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_VPS-POL", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_VPS-POL/X0109_Y0102", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_VPS-POL/X0109_Y0102/1987-1988_001-365_HL_TSA_LNDLG_NIR_VPS-POL.tif", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_VPS-POL/mosaic", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_VPS-POL/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_VPS-POL.vrt", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_VPS-POL/versions.yml", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_VPS-TRO", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_VPS-TRO/X0109_Y0102", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_VPS-TRO/X0109_Y0102/1987-1988_001-365_HL_TSA_LNDLG_NIR_VPS-TRO.tif", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_VPS-TRO/mosaic", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_VPS-TRO/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_VPS-TRO.vrt", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_VPS-TRO/versions.yml", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_VSA-CAO", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_VSA-CAO/X0109_Y0102", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_VSA-CAO/X0109_Y0102/1987-1988_001-365_HL_TSA_LNDLG_NIR_VSA-CAO.tif", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_VSA-CAO/mosaic", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_VSA-CAO/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_VSA-CAO.vrt", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_VSA-CAO/versions.yml", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_VSA-POL", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_VSA-POL/X0109_Y0102", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_VSA-POL/X0109_Y0102/1987-1988_001-365_HL_TSA_LNDLG_NIR_VSA-POL.tif", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_VSA-POL/mosaic", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_VSA-POL/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_VSA-POL.vrt", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_VSA-POL/versions.yml", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_VSA-TRO", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_VSA-TRO/X0109_Y0102", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_VSA-TRO/X0109_Y0102/1987-1988_001-365_HL_TSA_LNDLG_NIR_VSA-TRO.tif", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_VSA-TRO/mosaic", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_VSA-TRO/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_VSA-TRO.vrt", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_NIR_VSA-TRO/versions.yml", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_TSI", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_TSI/X0109_Y0102", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_TSI/X0109_Y0102/1987-1988_001-365_HL_TSA_LNDLG_RED_TSI.tif", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_TSI/X0109_Y0102/1987-1988_001-365_HL_TSA_LNDLG_RED_TSI.tif.aux.xml", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_TSI/mosaic", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_TSI/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_TSI.vrt", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_TSI/versions.yml", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_TSS", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_TSS/X0109_Y0102", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_TSS/X0109_Y0102/1987-1988_001-365_HL_TSA_LNDLG_RED_TSS.tif", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_TSS/mosaic", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_TSS/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_TSS.vrt", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_TSS/versions.yml", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_VBL-CAO", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_VBL-CAO/X0109_Y0102", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_VBL-CAO/X0109_Y0102/1987-1988_001-365_HL_TSA_LNDLG_RED_VBL-CAO.tif", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_VBL-CAO/mosaic", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_VBL-CAO/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_VBL-CAO.vrt", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_VBL-CAO/versions.yml", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_VBL-POL", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_VBL-POL/X0109_Y0102", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_VBL-POL/X0109_Y0102/1987-1988_001-365_HL_TSA_LNDLG_RED_VBL-POL.tif", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_VBL-POL/mosaic", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_VBL-POL/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_VBL-POL.vrt", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_VBL-POL/versions.yml", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_VBL-TRO", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_VBL-TRO/X0109_Y0102", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_VBL-TRO/X0109_Y0102/1987-1988_001-365_HL_TSA_LNDLG_RED_VBL-TRO.tif", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_VBL-TRO/mosaic", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_VBL-TRO/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_VBL-TRO.vrt", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_VBL-TRO/versions.yml", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_VPS-CAO", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_VPS-CAO/X0109_Y0102", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_VPS-CAO/X0109_Y0102/1987-1988_001-365_HL_TSA_LNDLG_RED_VPS-CAO.tif", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_VPS-CAO/mosaic", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_VPS-CAO/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_VPS-CAO.vrt", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_VPS-CAO/versions.yml", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_VPS-POL", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_VPS-POL/X0109_Y0102", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_VPS-POL/X0109_Y0102/1987-1988_001-365_HL_TSA_LNDLG_RED_VPS-POL.tif", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_VPS-POL/mosaic", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_VPS-POL/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_VPS-POL.vrt", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_VPS-POL/versions.yml", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_VPS-TRO", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_VPS-TRO/X0109_Y0102", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_VPS-TRO/X0109_Y0102/1987-1988_001-365_HL_TSA_LNDLG_RED_VPS-TRO.tif", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_VPS-TRO/mosaic", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_VPS-TRO/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_VPS-TRO.vrt", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_VPS-TRO/versions.yml", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_VSA-CAO", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_VSA-CAO/X0109_Y0102", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_VSA-CAO/X0109_Y0102/1987-1988_001-365_HL_TSA_LNDLG_RED_VSA-CAO.tif", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_VSA-CAO/mosaic", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_VSA-CAO/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_VSA-CAO.vrt", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_VSA-CAO/versions.yml", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_VSA-POL", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_VSA-POL/X0109_Y0102", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_VSA-POL/X0109_Y0102/1987-1988_001-365_HL_TSA_LNDLG_RED_VSA-POL.tif", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_VSA-POL/mosaic", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_VSA-POL/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_VSA-POL.vrt", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_VSA-POL/versions.yml", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_VSA-TRO", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_VSA-TRO/X0109_Y0102", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_VSA-TRO/X0109_Y0102/1987-1988_001-365_HL_TSA_LNDLG_RED_VSA-TRO.tif", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_VSA-TRO/mosaic", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_VSA-TRO/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_VSA-TRO.vrt", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_RED_VSA-TRO/versions.yml", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_TSI", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_TSI/X0109_Y0102", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_TSI/X0109_Y0102/1987-1988_001-365_HL_TSA_LNDLG_SMA_TSI.tif", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_TSI/X0109_Y0102/1987-1988_001-365_HL_TSA_LNDLG_SMA_TSI.tif.aux.xml", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_TSI/mosaic", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_TSI/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_TSI.vrt", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_TSI/versions.yml", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_TSS", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_TSS/X0109_Y0102", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_TSS/X0109_Y0102/1987-1988_001-365_HL_TSA_LNDLG_SMA_TSS.tif", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_TSS/mosaic", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_TSS/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_TSS.vrt", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_TSS/versions.yml", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_VBL-CAO", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_VBL-CAO/X0109_Y0102", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_VBL-CAO/X0109_Y0102/1987-1988_001-365_HL_TSA_LNDLG_SMA_VBL-CAO.tif", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_VBL-CAO/mosaic", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_VBL-CAO/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_VBL-CAO.vrt", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_VBL-CAO/versions.yml", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_VBL-POL", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_VBL-POL/X0109_Y0102", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_VBL-POL/X0109_Y0102/1987-1988_001-365_HL_TSA_LNDLG_SMA_VBL-POL.tif", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_VBL-POL/mosaic", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_VBL-POL/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_VBL-POL.vrt", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_VBL-POL/versions.yml", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_VBL-TRO", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_VBL-TRO/X0109_Y0102", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_VBL-TRO/X0109_Y0102/1987-1988_001-365_HL_TSA_LNDLG_SMA_VBL-TRO.tif", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_VBL-TRO/mosaic", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_VBL-TRO/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_VBL-TRO.vrt", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_VBL-TRO/versions.yml", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_VPS-CAO", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_VPS-CAO/X0109_Y0102", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_VPS-CAO/X0109_Y0102/1987-1988_001-365_HL_TSA_LNDLG_SMA_VPS-CAO.tif", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_VPS-CAO/mosaic", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_VPS-CAO/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_VPS-CAO.vrt", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_VPS-CAO/versions.yml", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_VPS-POL", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_VPS-POL/X0109_Y0102", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_VPS-POL/X0109_Y0102/1987-1988_001-365_HL_TSA_LNDLG_SMA_VPS-POL.tif", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_VPS-POL/mosaic", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_VPS-POL/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_VPS-POL.vrt", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_VPS-POL/versions.yml", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_VPS-TRO", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_VPS-TRO/X0109_Y0102", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_VPS-TRO/X0109_Y0102/1987-1988_001-365_HL_TSA_LNDLG_SMA_VPS-TRO.tif", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_VPS-TRO/mosaic", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_VPS-TRO/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_VPS-TRO.vrt", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_VPS-TRO/versions.yml", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_VSA-CAO", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_VSA-CAO/X0109_Y0102", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_VSA-CAO/X0109_Y0102/1987-1988_001-365_HL_TSA_LNDLG_SMA_VSA-CAO.tif", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_VSA-CAO/mosaic", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_VSA-CAO/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_VSA-CAO.vrt", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_VSA-CAO/versions.yml", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_VSA-POL", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_VSA-POL/X0109_Y0102", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_VSA-POL/X0109_Y0102/1987-1988_001-365_HL_TSA_LNDLG_SMA_VSA-POL.tif", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_VSA-POL/mosaic", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_VSA-POL/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_VSA-POL.vrt", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_VSA-POL/versions.yml", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_VSA-TRO", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_VSA-TRO/X0109_Y0102", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_VSA-TRO/X0109_Y0102/1987-1988_001-365_HL_TSA_LNDLG_SMA_VSA-TRO.tif", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_VSA-TRO/mosaic", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_VSA-TRO/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_VSA-TRO.vrt", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SMA_VSA-TRO/versions.yml", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_TSI", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_TSI/X0109_Y0102", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_TSI/X0109_Y0102/1987-1988_001-365_HL_TSA_LNDLG_SW2_TSI.tif", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_TSI/X0109_Y0102/1987-1988_001-365_HL_TSA_LNDLG_SW2_TSI.tif.aux.xml", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_TSI/mosaic", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_TSI/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_TSI.vrt", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_TSI/versions.yml", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_TSS", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_TSS/X0109_Y0102", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_TSS/X0109_Y0102/1987-1988_001-365_HL_TSA_LNDLG_SW2_TSS.tif", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_TSS/mosaic", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_TSS/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_TSS.vrt", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_TSS/versions.yml", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_VBL-CAO", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_VBL-CAO/X0109_Y0102", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_VBL-CAO/X0109_Y0102/1987-1988_001-365_HL_TSA_LNDLG_SW2_VBL-CAO.tif", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_VBL-CAO/mosaic", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_VBL-CAO/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_VBL-CAO.vrt", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_VBL-CAO/versions.yml", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_VBL-POL", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_VBL-POL/X0109_Y0102", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_VBL-POL/X0109_Y0102/1987-1988_001-365_HL_TSA_LNDLG_SW2_VBL-POL.tif", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_VBL-POL/mosaic", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_VBL-POL/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_VBL-POL.vrt", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_VBL-POL/versions.yml", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_VBL-TRO", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_VBL-TRO/X0109_Y0102", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_VBL-TRO/X0109_Y0102/1987-1988_001-365_HL_TSA_LNDLG_SW2_VBL-TRO.tif", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_VBL-TRO/mosaic", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_VBL-TRO/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_VBL-TRO.vrt", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_VBL-TRO/versions.yml", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_VPS-CAO", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_VPS-CAO/X0109_Y0102", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_VPS-CAO/X0109_Y0102/1987-1988_001-365_HL_TSA_LNDLG_SW2_VPS-CAO.tif", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_VPS-CAO/mosaic", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_VPS-CAO/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_VPS-CAO.vrt", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_VPS-CAO/versions.yml", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_VPS-POL", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_VPS-POL/X0109_Y0102", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_VPS-POL/X0109_Y0102/1987-1988_001-365_HL_TSA_LNDLG_SW2_VPS-POL.tif", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_VPS-POL/mosaic", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_VPS-POL/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_VPS-POL.vrt", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_VPS-POL/versions.yml", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_VPS-TRO", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_VPS-TRO/X0109_Y0102", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_VPS-TRO/X0109_Y0102/1987-1988_001-365_HL_TSA_LNDLG_SW2_VPS-TRO.tif", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_VPS-TRO/mosaic", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_VPS-TRO/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_VPS-TRO.vrt", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_VPS-TRO/versions.yml", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_VSA-CAO", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_VSA-CAO/X0109_Y0102", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_VSA-CAO/X0109_Y0102/1987-1988_001-365_HL_TSA_LNDLG_SW2_VSA-CAO.tif", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_VSA-CAO/mosaic", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_VSA-CAO/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_VSA-CAO.vrt", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_VSA-CAO/versions.yml", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_VSA-POL", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_VSA-POL/X0109_Y0102", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_VSA-POL/X0109_Y0102/1987-1988_001-365_HL_TSA_LNDLG_SW2_VSA-POL.tif", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_VSA-POL/mosaic", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_VSA-POL/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_VSA-POL.vrt", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_VSA-POL/versions.yml", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_VSA-TRO", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_VSA-TRO/X0109_Y0102", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_VSA-TRO/X0109_Y0102/1987-1988_001-365_HL_TSA_LNDLG_SW2_VSA-TRO.tif", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_VSA-TRO/mosaic", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_VSA-TRO/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_VSA-TRO.vrt", + "trend/mosaic/1987-1988_001-365_HL_TSA_LNDLG_SW2_VSA-TRO/versions.yml", + "trend/pyramid", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_NIR_TSI", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_NIR_TSI/trend", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_NIR_TSI/trend/X0109_Y0102", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_NIR_TSI/trend/X0109_Y0102/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_NIR_TSI.tif.ovr", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_NIR_TSS", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_NIR_TSS/trend", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_NIR_TSS/trend/X0109_Y0102", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_NIR_TSS/trend/X0109_Y0102/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_NIR_TSS.tif.ovr", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_NIR_VBL-CAO", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_NIR_VBL-CAO/trend", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_NIR_VBL-CAO/trend/X0109_Y0102", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_NIR_VBL-CAO/trend/X0109_Y0102/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_NIR_VBL-CAO.tif.ovr", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_NIR_VBL-POL", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_NIR_VBL-POL/trend", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_NIR_VBL-POL/trend/X0109_Y0102", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_NIR_VBL-POL/trend/X0109_Y0102/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_NIR_VBL-POL.tif.ovr", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_NIR_VBL-TRO", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_NIR_VBL-TRO/trend", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_NIR_VBL-TRO/trend/X0109_Y0102", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_NIR_VBL-TRO/trend/X0109_Y0102/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_NIR_VBL-TRO.tif.ovr", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_NIR_VPS-CAO", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_NIR_VPS-CAO/trend", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_NIR_VPS-CAO/trend/X0109_Y0102", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_NIR_VPS-CAO/trend/X0109_Y0102/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_NIR_VPS-CAO.tif.ovr", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_NIR_VPS-POL", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_NIR_VPS-POL/trend", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_NIR_VPS-POL/trend/X0109_Y0102", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_NIR_VPS-POL/trend/X0109_Y0102/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_NIR_VPS-POL.tif.ovr", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_NIR_VPS-TRO", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_NIR_VPS-TRO/trend", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_NIR_VPS-TRO/trend/X0109_Y0102", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_NIR_VPS-TRO/trend/X0109_Y0102/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_NIR_VPS-TRO.tif.ovr", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_NIR_VSA-CAO", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_NIR_VSA-CAO/trend", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_NIR_VSA-CAO/trend/X0109_Y0102", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_NIR_VSA-CAO/trend/X0109_Y0102/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_NIR_VSA-CAO.tif.ovr", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_NIR_VSA-POL", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_NIR_VSA-POL/trend", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_NIR_VSA-POL/trend/X0109_Y0102", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_NIR_VSA-POL/trend/X0109_Y0102/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_NIR_VSA-POL.tif.ovr", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_NIR_VSA-TRO", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_NIR_VSA-TRO/trend", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_NIR_VSA-TRO/trend/X0109_Y0102", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_NIR_VSA-TRO/trend/X0109_Y0102/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_NIR_VSA-TRO.tif.ovr", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_RED_TSI", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_RED_TSI/trend", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_RED_TSI/trend/X0109_Y0102", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_RED_TSI/trend/X0109_Y0102/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_RED_TSI.tif.ovr", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_RED_TSS", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_RED_TSS/trend", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_RED_TSS/trend/X0109_Y0102", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_RED_TSS/trend/X0109_Y0102/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_RED_TSS.tif.ovr", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_RED_VBL-CAO", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_RED_VBL-CAO/trend", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_RED_VBL-CAO/trend/X0109_Y0102", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_RED_VBL-CAO/trend/X0109_Y0102/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_RED_VBL-CAO.tif.ovr", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_RED_VBL-POL", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_RED_VBL-POL/trend", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_RED_VBL-POL/trend/X0109_Y0102", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_RED_VBL-POL/trend/X0109_Y0102/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_RED_VBL-POL.tif.ovr", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_RED_VBL-TRO", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_RED_VBL-TRO/trend", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_RED_VBL-TRO/trend/X0109_Y0102", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_RED_VBL-TRO/trend/X0109_Y0102/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_RED_VBL-TRO.tif.ovr", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_RED_VPS-CAO", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_RED_VPS-CAO/trend", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_RED_VPS-CAO/trend/X0109_Y0102", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_RED_VPS-CAO/trend/X0109_Y0102/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_RED_VPS-CAO.tif.ovr", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_RED_VPS-POL", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_RED_VPS-POL/trend", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_RED_VPS-POL/trend/X0109_Y0102", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_RED_VPS-POL/trend/X0109_Y0102/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_RED_VPS-POL.tif.ovr", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_RED_VPS-TRO", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_RED_VPS-TRO/trend", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_RED_VPS-TRO/trend/X0109_Y0102", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_RED_VPS-TRO/trend/X0109_Y0102/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_RED_VPS-TRO.tif.ovr", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_RED_VSA-CAO", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_RED_VSA-CAO/trend", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_RED_VSA-CAO/trend/X0109_Y0102", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_RED_VSA-CAO/trend/X0109_Y0102/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_RED_VSA-CAO.tif.ovr", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_RED_VSA-POL", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_RED_VSA-POL/trend", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_RED_VSA-POL/trend/X0109_Y0102", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_RED_VSA-POL/trend/X0109_Y0102/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_RED_VSA-POL.tif.ovr", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_RED_VSA-TRO", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_RED_VSA-TRO/trend", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_RED_VSA-TRO/trend/X0109_Y0102", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_RED_VSA-TRO/trend/X0109_Y0102/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_RED_VSA-TRO.tif.ovr", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SMA_TSI", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SMA_TSI/trend", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SMA_TSI/trend/X0109_Y0102", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SMA_TSI/trend/X0109_Y0102/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_SMA_TSI.tif.ovr", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SMA_TSS", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SMA_TSS/trend", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SMA_TSS/trend/X0109_Y0102", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SMA_TSS/trend/X0109_Y0102/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_SMA_TSS.tif.ovr", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SMA_VBL-CAO", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SMA_VBL-CAO/trend", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SMA_VBL-CAO/trend/X0109_Y0102", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SMA_VBL-CAO/trend/X0109_Y0102/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_SMA_VBL-CAO.tif.ovr", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SMA_VBL-POL", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SMA_VBL-POL/trend", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SMA_VBL-POL/trend/X0109_Y0102", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SMA_VBL-POL/trend/X0109_Y0102/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_SMA_VBL-POL.tif.ovr", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SMA_VBL-TRO", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SMA_VBL-TRO/trend", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SMA_VBL-TRO/trend/X0109_Y0102", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SMA_VBL-TRO/trend/X0109_Y0102/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_SMA_VBL-TRO.tif.ovr", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SMA_VPS-CAO", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SMA_VPS-CAO/trend", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SMA_VPS-CAO/trend/X0109_Y0102", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SMA_VPS-CAO/trend/X0109_Y0102/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_SMA_VPS-CAO.tif.ovr", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SMA_VPS-POL", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SMA_VPS-POL/trend", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SMA_VPS-POL/trend/X0109_Y0102", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SMA_VPS-POL/trend/X0109_Y0102/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_SMA_VPS-POL.tif.ovr", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SMA_VPS-TRO", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SMA_VPS-TRO/trend", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SMA_VPS-TRO/trend/X0109_Y0102", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SMA_VPS-TRO/trend/X0109_Y0102/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_SMA_VPS-TRO.tif.ovr", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SMA_VSA-CAO", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SMA_VSA-CAO/trend", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SMA_VSA-CAO/trend/X0109_Y0102", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SMA_VSA-CAO/trend/X0109_Y0102/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_SMA_VSA-CAO.tif.ovr", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SMA_VSA-POL", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SMA_VSA-POL/trend", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SMA_VSA-POL/trend/X0109_Y0102", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SMA_VSA-POL/trend/X0109_Y0102/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_SMA_VSA-POL.tif.ovr", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SMA_VSA-TRO", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SMA_VSA-TRO/trend", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SMA_VSA-TRO/trend/X0109_Y0102", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SMA_VSA-TRO/trend/X0109_Y0102/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_SMA_VSA-TRO.tif.ovr", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SW2_TSI", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SW2_TSI/trend", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SW2_TSI/trend/X0109_Y0102", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SW2_TSI/trend/X0109_Y0102/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_SW2_TSI.tif.ovr", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SW2_TSS", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SW2_TSS/trend", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SW2_TSS/trend/X0109_Y0102", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SW2_TSS/trend/X0109_Y0102/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_SW2_TSS.tif.ovr", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SW2_VBL-CAO", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SW2_VBL-CAO/trend", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SW2_VBL-CAO/trend/X0109_Y0102", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SW2_VBL-CAO/trend/X0109_Y0102/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_SW2_VBL-CAO.tif.ovr", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SW2_VBL-POL", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SW2_VBL-POL/trend", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SW2_VBL-POL/trend/X0109_Y0102", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SW2_VBL-POL/trend/X0109_Y0102/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_SW2_VBL-POL.tif.ovr", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SW2_VBL-TRO", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SW2_VBL-TRO/trend", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SW2_VBL-TRO/trend/X0109_Y0102", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SW2_VBL-TRO/trend/X0109_Y0102/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_SW2_VBL-TRO.tif.ovr", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SW2_VPS-CAO", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SW2_VPS-CAO/trend", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SW2_VPS-CAO/trend/X0109_Y0102", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SW2_VPS-CAO/trend/X0109_Y0102/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_SW2_VPS-CAO.tif.ovr", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SW2_VPS-POL", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SW2_VPS-POL/trend", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SW2_VPS-POL/trend/X0109_Y0102", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SW2_VPS-POL/trend/X0109_Y0102/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_SW2_VPS-POL.tif.ovr", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SW2_VPS-TRO", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SW2_VPS-TRO/trend", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SW2_VPS-TRO/trend/X0109_Y0102", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SW2_VPS-TRO/trend/X0109_Y0102/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_SW2_VPS-TRO.tif.ovr", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SW2_VSA-CAO", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SW2_VSA-CAO/trend", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SW2_VSA-CAO/trend/X0109_Y0102", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SW2_VSA-CAO/trend/X0109_Y0102/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_SW2_VSA-CAO.tif.ovr", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SW2_VSA-POL", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SW2_VSA-POL/trend", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SW2_VSA-POL/trend/X0109_Y0102", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SW2_VSA-POL/trend/X0109_Y0102/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_SW2_VSA-POL.tif.ovr", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SW2_VSA-TRO", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SW2_VSA-TRO/trend", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SW2_VSA-TRO/trend/X0109_Y0102", + "trend/pyramid/1987-1988_001-365_HL_TSA_LNDLG_SW2_VSA-TRO/trend/X0109_Y0102/X0109_Y0102_1987-1988_001-365_HL_TSA_LNDLG_SW2_VSA-TRO.tif.ovr", + "trend/pyramid/versions.yml", + "untar", + "untar/Landsat_data", + "untar/dem", + "untar/wvdb" + ] + ], + "meta": { + "nf-test": "0.9.1", + "nextflow": "24.10.0" + }, + "timestamp": "2024-11-24T15:43:30.671676472" + } +} \ No newline at end of file diff --git a/tests_large/check_test_full_results.nf.test b/tests_large/check_test_full_results.nf.test new file mode 100644 index 0000000..4a2affe --- /dev/null +++ b/tests_large/check_test_full_results.nf.test @@ -0,0 +1,57 @@ +// WARNING: This test requires approximately 400GB of free space. +// It is excluded from default test runs due to its high resource demands. +// Run with caution. + +// This is a pipeline level-test, not a module test, even though the main script is a module. +// The entire workflow is executed in the setup scope using the same config as the test_full profile. +// Next, still within the setup scope the reference data is pulled and extracted. +// Afterwards, a single module is executed in the when scope to ensure that the pipeline generated correct results. + +nextflow_process { + + name "Check the correctness of the pipeline results using the test_full profile" + script "../modules/local/check_results_full/main.nf" + process "CHECK_RESULTS_FULL" + config "../conf/test_full.config" + + test("Checking pipeline results for: -profile test_full | requires 400GB free space") { + + setup { + run("NFCORE_RANGELAND") { + script "../main.nf" + workflow {} + } + + run("UNTAR", alias: "UNTAR_REF" ) { + script "../modules/nf-core/untar/main.nf" + process { + """ + input[0] = [[:], params.reference] + """ + } + } + + } + + when { + params { + // reference data + reference = "s3://ngi-igenomes/test-data/rangeland/reference.tar" + } + + process { + """ + // inputs for CHECK_RESULTS_FULL + input[0] = NFCORE_RANGELAND.out.mosaic.map{ it[1] }.flatten().buffer( size: Integer.MAX_VALUE, remainder: true ) + input[1] = UNTAR_REF.out.untar.map(it->it[1]) + """ + } + } + + then { + assert process.success + } + + } + +} diff --git a/workflows/rangeland.nf b/workflows/rangeland.nf index aa9e763..905b9f4 100644 --- a/workflows/rangeland.nf +++ b/workflows/rangeland.nf @@ -1,30 +1,14 @@ /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - PRINT PARAMS SUMMARY + IMPORT MODULES / SUBWORKFLOWS / FUNCTIONS ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ -include { paramsSummaryLog; paramsSummaryMap } from 'plugin/nf-validation' - -def logo = NfcoreTemplate.logo(workflow, params.monochrome_logs) -def citation = '\n' + WorkflowMain.citation(workflow) + '\n' -def summary_params = paramsSummaryMap(workflow) - -// Print parameter summary log to screen -log.info logo + paramsSummaryLog(workflow) + citation - -WorkflowRangeland.initialise(params, log) - -/* -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - CONFIG FILES -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -*/ - -ch_multiqc_config = Channel.fromPath("$projectDir/assets/multiqc_config.yml", checkIfExists: true) -ch_multiqc_custom_config = params.multiqc_config ? Channel.fromPath( params.multiqc_config, checkIfExists: true ) : Channel.empty() -ch_multiqc_logo = params.multiqc_logo ? Channel.fromPath( params.multiqc_logo, checkIfExists: true ) : Channel.empty() -ch_multiqc_custom_methods_description = params.multiqc_methods_description ? file(params.multiqc_methods_description, checkIfExists: true) : file("$projectDir/assets/methods_description_template.yml", checkIfExists: true) +include { MULTIQC } from '../modules/nf-core/multiqc/main' +include { paramsSummaryMap } from 'plugin/nf-schema' +include { paramsSummaryMultiqc } from '../subworkflows/nf-core/utils_nfcore_pipeline' +include { softwareVersionsToYAML } from '../subworkflows/nf-core/utils_nfcore_pipeline' +include { methodsDescriptionText } from '../subworkflows/local/utils_nfcore_rangeland_pipeline' /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -35,17 +19,9 @@ ch_multiqc_custom_methods_description = params.multiqc_methods_description ? fil // // SUBWORKFLOW: Consisting of a mix of local and nf-core/modules // -include { INPUT_CHECK } from '../subworkflows/local/input_check' include { PREPROCESSING } from '../subworkflows/local/preprocessing' include { HIGHER_LEVEL } from '../subworkflows/local/higher_level' -// -// MODULES -// - -include { CHECK_RESULTS } from '../modules/local/check_results' - - /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ IMPORT NF-CORE MODULES/SUBWORKFLOWS @@ -55,19 +31,16 @@ include { CHECK_RESULTS } from '../modules/local/check_results' // // MODULE: Installed directly from nf-core/modules // -include { MULTIQC } from '../modules/nf-core/multiqc/main' -include { CUSTOM_DUMPSOFTWAREVERSIONS } from '../modules/nf-core/custom/dumpsoftwareversions/main' -include { UNTAR as UNTAR_INPUT; UNTAR as UNTAR_DEM; UNTAR as UNTAR_WVDB } from '../modules/nf-core/untar/main' +include { UNTAR as UNTAR_INPUT; UNTAR as UNTAR_DEM; UNTAR as UNTAR_WVDB; UNTAR as UNTAR_REF } from '../modules/nf-core/untar/main' + /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - RUN MAIN WORKFLOW + HELPER FUNCTIONS ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ -// Info required for completion email and summary -def multiqc_report = [] -// check wether provided input is within provided time range +// check whether provided input is within provided time range def inRegion = input -> { Integer date = input.simpleName.split("_")[3] as Integer Integer start = params.start_date.replace('-','') as Integer @@ -76,135 +49,182 @@ def inRegion = input -> { return date >= start && date <= end } +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + RUN MAIN WORKFLOW +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +*/ + + workflow RANGELAND { - ch_versions = Channel.empty() + main: + ch_versions = Channel.empty() + ch_multiqc_files = Channel.empty() // // Stage and validate input files // - data = null - dem = null - wvdb = null - cube_file = file( "$params.data_cube" ) - aoi_file = file( "$params.aoi" ) - endmember_file = file( "$params.endmember" ) + data = Channel.empty() + dem = Channel.empty() + wvdb = Channel.empty() + cube_file = file( params.data_cube ) + aoi_file = file( params.aoi ) + endmember_file = file( params.endmember ) // // MODULE: untar // tar_versions = Channel.empty() - if (params.input_tar) { - UNTAR_INPUT([[:], params.input]) - base_path = UNTAR_INPUT.out.untar.map(it -> it[1]) - data = base_path.map(it -> file("$it/*/*", type: 'dir')).flatten() - data = data.flatten().filter{ inRegion(it) } - - tar_versions = tar_versions.mix(UNTAR_INPUT.out.versions) - } else { - data = Channel.fromPath( "${params.input}/*/*", type: 'dir') .flatten() - data = data.flatten().filter{ inRegion(it) } + // Determine type of params.input and extract when neccessary + ch_input = Channel.of(file(params.input)) + ch_input.branch { it + archives : it.name.endsWith('tar') || it.name.endsWith('tar.gz') + return tuple([:], it) + dirs: true + return it + } + .set{ ch_input_types } + + UNTAR_INPUT(ch_input_types.archives) + ch_untared_inputs = UNTAR_INPUT.out.untar.map(it -> it[1]) + tar_versions = tar_versions.mix(UNTAR_INPUT.out.versions) + + data = data + .mix(ch_untared_inputs, ch_input_types.dirs) + .map(it -> file("$it/*/*", type: 'dir')).flatten() + .filter{ inRegion(it) } + + // Determine type of params.dem and extract when neccessary + ch_dem = Channel.of(file(params.dem)) + ch_dem.branch { it + archives : it.name.endsWith('tar') || it.name.endsWith('tar.gz') + return tuple([:], it) + dirs: true + return file(it) } + .set{ ch_dem_types } + + UNTAR_DEM(ch_dem_types.archives) + ch_untared_dem = UNTAR_DEM.out.untar.map(it -> it[1]) + tar_versions = tar_versions.mix(UNTAR_DEM.out.versions) - if (params.dem_tar) { - UNTAR_DEM([[:], params.dem]) - dem = UNTAR_DEM.out.untar.map(it -> file(it[1])) + dem = dem.mix(ch_untared_dem, ch_dem_types.dirs).first() - tar_versions = tar_versions.mix(UNTAR_DEM.out.versions) - } else { - dem = file("$params.dem") + // Determine type of params.wvdb and extract when neccessary + ch_wvdb = Channel.of(file(params.wvdb)) + ch_wvdb.branch { it + archives : it.name.endsWith('tar') || it.name.endsWith('tar.gz') + return tuple([:], it) + dirs: true + return file(it) } + .set{ ch_wvdb_types } - if (params.wvdb_tar) { - UNTAR_WVDB([[:], params.wvdb]) - wvdb = UNTAR_WVDB.out.untar.map(it -> file(it[1])) + UNTAR_WVDB(ch_wvdb_types.archives) + ch_untared_wvdb = UNTAR_WVDB.out.untar.map(it -> it[1]) + tar_versions = tar_versions.mix(UNTAR_WVDB.out.versions) - tar_versions = tar_versions.mix(UNTAR_WVDB.out.versions) - } else { - wvdb = file("$params.wvdb") - } + wvdb = wvdb.mix(ch_untared_wvdb, ch_wvdb_types.dirs).first() - ch_versions = ch_versions.mix(tar_versions.first().ifEmpty(null)) + ch_versions = ch_versions.mix(tar_versions.first()) // // SUBWORKFLOW: Preprocess satellite imagery // - PREPROCESSING(data, dem, wvdb, cube_file, aoi_file) + PREPROCESSING ( + data, + dem, + wvdb, + cube_file, + aoi_file, + params.group_size, + params.resolution + ) ch_versions = ch_versions.mix(PREPROCESSING.out.versions) - preprocessed_data = PREPROCESSING.out.tiles_and_masks.filter { params.only_tile ? it[0] == params.only_tile : true } - // // SUBWORKFLOW: Generate trend files and visualization // - HIGHER_LEVEL( preprocessed_data, cube_file, endmember_file ) + HIGHER_LEVEL( + PREPROCESSING.out.tiles_and_masks, + cube_file, + endmember_file, + params.mosaic_visualization, + params.pyramid_visualization, + params.resolution, + params.sensors_level2, + params.start_date, + params.end_date, + params.indexes, + params.return_tss + ) ch_versions = ch_versions.mix(HIGHER_LEVEL.out.versions) - grouped_trend_data = HIGHER_LEVEL.out.trend_files.map{ it[1] }.flatten().buffer( size: Integer.MAX_VALUE, remainder: true ) + grouped_trend_data = HIGHER_LEVEL.out.mosaic.map{ it[1] }.flatten().buffer( size: Integer.MAX_VALUE, remainder: true ) // - // MODULE: Check results + // Collate and save software versions // - if ( params.config_profile_name == 'Test profile' ) { - woody_change_ref = file("$params.woody_change_ref") - woody_yoc_ref = file("$params.woody_yoc_ref") - herbaceous_change_ref = file("$params.herbaceous_change_ref") - herbaceous_yoc_ref = file("$params.herbaceous_yoc_ref") - peak_change_ref = file("$params.peak_change_ref") - peak_yoc_ref = file("$params.peak_yoc_ref") - - CHECK_RESULTS( grouped_trend_data, woody_change_ref, woody_yoc_ref, herbaceous_change_ref, herbaceous_yoc_ref, peak_change_ref, peak_yoc_ref) - ch_versions = ch_versions.mix(CHECK_RESULTS.out.versions) - } - + softwareVersionsToYAML(ch_versions) + .collectFile( + storeDir: "${params.outdir}/pipeline_info", + name: 'nf_core_' + 'pipeline_software_' + 'mqc_' + 'versions.yml', + sort: true, + newLine: true + ).set { ch_collated_versions } - // - // MODULE: Pipeline reporting - // - CUSTOM_DUMPSOFTWAREVERSIONS ( - ch_versions.unique().collectFile(name: 'collated_versions.yml') - ) // // MODULE: MultiQC // - workflow_summary = WorkflowRangeland.paramsSummaryMultiqc(workflow, summary_params) - ch_workflow_summary = Channel.value(workflow_summary) - - methods_description = WorkflowRangeland.methodsDescriptionText(workflow, ch_multiqc_custom_methods_description, params) - ch_methods_description = Channel.value(methods_description) - - ch_multiqc_files = Channel.empty() - ch_multiqc_files = ch_multiqc_files.mix(ch_workflow_summary.collectFile(name: 'workflow_summary_mqc.yaml')) - ch_multiqc_files = ch_multiqc_files.mix(ch_methods_description.collectFile(name: 'methods_description_mqc.yaml')) - ch_multiqc_files = ch_multiqc_files.mix(CUSTOM_DUMPSOFTWAREVERSIONS.out.mqc_yml.collect()) + ch_multiqc_config = Channel.fromPath( + "$projectDir/assets/multiqc_config.yml", checkIfExists: true) + ch_multiqc_custom_config = params.multiqc_config ? + Channel.fromPath(params.multiqc_config, checkIfExists: true) : + Channel.empty() + ch_multiqc_logo = params.multiqc_logo ? + Channel.fromPath(params.multiqc_logo, checkIfExists: true) : + Channel.empty() + + summary_params = paramsSummaryMap( + workflow, parameters_schema: "nextflow_schema.json") + ch_workflow_summary = Channel.value(paramsSummaryMultiqc(summary_params)) + ch_multiqc_files = ch_multiqc_files.mix( + ch_workflow_summary.collectFile(name: 'workflow_summary_mqc.yaml')) + ch_multiqc_custom_methods_description = params.multiqc_methods_description ? + file(params.multiqc_methods_description, checkIfExists: true) : + file("$projectDir/assets/methods_description_template.yml", checkIfExists: true) + ch_methods_description = Channel.value( + methodsDescriptionText(ch_multiqc_custom_methods_description)) + + ch_multiqc_files = ch_multiqc_files.mix(ch_collated_versions) + ch_multiqc_files = ch_multiqc_files.mix( + ch_methods_description.collectFile( + name: 'methods_description_mqc.yaml', + sort: true + ) + ) MULTIQC ( ch_multiqc_files.collect(), ch_multiqc_config.toList(), ch_multiqc_custom_config.toList(), - ch_multiqc_logo.toList() + ch_multiqc_logo.toList(), + [], + [] ) - multiqc_report = MULTIQC.out.report.toList() -} -/* -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - COMPLETION EMAIL AND SUMMARY -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -*/ + emit: + level2_ard = PREPROCESSING.out.tiles_and_masks + mosaic = HIGHER_LEVEL.out.mosaic + pyramid = HIGHER_LEVEL.out.pyramid + trends = HIGHER_LEVEL.out.trends + multiqc_report = MULTIQC.out.report.toList() // channel: /path/to/multiqc_report.html + versions = ch_versions // channel: [ path(versions.yml) ] -workflow.onComplete { - if (params.email || params.email_on_fail) { - NfcoreTemplate.email(workflow, params, summary_params, projectDir, log, multiqc_report) - } - NfcoreTemplate.dump_parameters(workflow, params) - NfcoreTemplate.summary(workflow, params, log) - if (params.hook_url) { - NfcoreTemplate.IM_notification(workflow, params, summary_params, projectDir, log) - } } /*