cache example data #249
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Combined Workflow | ||
| on: [push] | ||
| defaults: | ||
| run: | ||
| shell: bash -el {0} # use default shell | ||
| jobs: | ||
| build-and-dryrun: | ||
| runs-on: ubuntu-latest | ||
| strategy: | ||
| max-parallel: 2 | ||
| steps: | ||
| - name: Checkout | ||
| uses: actions/[email protected] | ||
| # - name: Get current month | ||
| # id: date | ||
| # run: echo "date=$(date +%Y-%m)" >> "${GITHUB_OUTPUT}" | ||
| - uses: mamba-org/setup-micromamba@v1 | ||
| with: | ||
| environment-file: atlasenv.yml | ||
| environment-name: atlasenv | ||
| # persist on the same month. | ||
| cache-environment-key: atlasenv- #${{ steps.date.outputs.date }} | ||
| - name: Install atlas | ||
| run: | | ||
| python -m pip install . --no-deps -vv | ||
| - name: Test atlas | ||
| run: | | ||
| atlas --help | ||
| atlas --version | ||
| - name: test import modules atlas | ||
| run: | | ||
| python -c "from atlas import utils" | ||
| - name: Dryrun | ||
| run: test/dryrun.sh | ||
| - name: Test init many samples | ||
| run: ./test/test_init_many_samples.sh | ||
| run-qc-and-assembly: | ||
| runs-on: ubuntu-latest | ||
| needs: | ||
| - build-and-dryrun | ||
| - download-example-data | ||
|
Check failure on line 54 in .github/workflows/python-package-conda.yml
|
||
| env: | ||
| N_THREADS: 2 | ||
| MEM: 3 | ||
| steps: | ||
| - name: Checkout | ||
| uses: actions/[email protected] | ||
| - uses: mamba-org/setup-micromamba@v1 | ||
| with: | ||
| environment-file: atlasenv.yml | ||
| environment-name: atlasenv | ||
| cache-environment-key: atlasenv- | ||
| - name: Install atlas | ||
| run: | | ||
| python -m pip install . --no-deps -vv | ||
| atlas --help | ||
| - name: cache example data | ||
| id: get-data | ||
| uses: actions/cache@v2 | ||
| with: | ||
| path: test_reads.tar.gz | ||
| key: example-data | ||
| - name: Download test data | ||
| if: steps.get-data.outputs.cache-hit != 'true' | ||
| run: wget --quiet https://zenodo.org/record/3992790/files/test_reads.tar.gz | ||
| - name: cache example data | ||
| if: steps.get-data.outputs.cache-hit != 'true' | ||
| uses: actions/cache@v2 | ||
| with: | ||
| path: test_reads.tar.gz | ||
| key: example-data | ||
| - name: extract data | ||
| run: | | ||
| tar -xzf test_reads.tar.gz | ||
| ls -l test_reads | ||
| - name: get conda envs | ||
| id: get-envs | ||
| uses: actions/cache@v2 | ||
| with: | ||
| path: databases/conda_envs | ||
| key: assembly-conda-envs | ||
| # - name: upack conda envs | ||
| # if: steps.get-envs.outputs.cache-hit != 'true' | ||
| # run: tar -xzf assembly_conda_envs.tar.gz | ||
| - name: Init | ||
| run: | | ||
| atlas init "test_reads" --db-dir "$DATABASE_DIR" --threads "$N_THREADS" --working-dir wd | ||
| - name: Install dependencies for qc and assembly | ||
| if: steps.get-envs.outputs.cache-hit != 'true' | ||
| run: atlas run assembly --conda-create-envs-only | ||
| # - name: compress conda envs | ||
| # run: | | ||
| # tar -czf assembly_conda_envs.tar.gz $DATABASE_DIR/conda_envs | ||
| - name: Cache conda envs | ||
| if: steps.get-envs.outputs.cache-hit != 'true' | ||
| uses: actions/cache@v2 | ||
| with: | ||
| path: databases/conda_envs | ||
| key: assembly-conda-envs | ||
| - name: Run QC | ||
| run: | | ||
| atlas run qc --resources mem=$MEM java_mem=$MEM --jobs=$N_THREADS --restart-times=2 --working-dir wd | ||
| - name: Test Assembly | ||
| run: | | ||
| atlas run assembly --resources mem=$MEM java_mem=$MEM --jobs=$N_THREADS --restart-times=2 --working-dir wd | ||
| - name: Store Logs | ||
| uses: actions/upload-artifact@v2 | ||
| with: | ||
| name: logs | ||
| path: wd/logs | ||
| - name: Store Sample Logs | ||
| uses: actions/upload-artifact@v2 | ||
| with: | ||
| name: sample_logs | ||
| path: wd/Sample1/logs | ||
| - name: Cache working dir | ||
| uses: actions/cache@v2 | ||
| with: | ||
| path: wd | ||
| key: assembly-working-dir | ||
| run-genecatalog: | ||
| runs-on: ubuntu-latest | ||
| needs: run-qc-and-assembly | ||
| env: | ||
| N_THREADS: 2 | ||
| MEM: 3 | ||
| steps: | ||
| - name: Checkout | ||
| uses: actions/[email protected] | ||
| - uses: mamba-org/setup-micromamba@v1 | ||
| with: | ||
| environment-file: atlasenv.yml | ||
| environment-name: atlasenv | ||
| cache-environment-key: atlasenv- | ||
| - name: install atlas | ||
| run: | | ||
| python -m pip install . --no-deps -vv | ||
| atlas --help | ||
| - name: get conda envs | ||
| id: get-genecatalog-envs | ||
| uses: actions/cache@v2 | ||
| with: | ||
| path: databases/conda_envs | ||
| key: binning-conda-envs | ||
| restore-keys: | | ||
| assembly-conda-envs | ||
| - name: Restore working dir | ||
| uses: actions/cache@v2 | ||
| with: | ||
| path: wd | ||
| key: assembly-working-dir | ||
| - name: dryrun assembly shold need nothing to be done | ||
| run: | | ||
| ls -l wd | ||
| atlas run assembly -n | ||
| - name: Test Genecatalog | ||
| run: | | ||
| atlas run genecatalog --resources mem=$MEM java_mem=$MEM --jobs=$N_THREADS --restart-times=2 --working-dir wd | ||
| - name: Store Logs | ||
| uses: actions/upload-artifact@v2 | ||
| with: | ||
| name: logs | ||
| path: wd/logs | ||
| - name: Store Sample Logs | ||
| uses: actions/upload-artifact@v2 | ||
| with: | ||
| name: sample_logs | ||
| path: wd/Sample1/logs | ||
| # full-test: | ||
| # runs-on: ubuntu-latest | ||
| # needs: build-and-dryrun | ||
| # strategy: | ||
| # max-parallel: 1 | ||
| # env: | ||
| # N_THREADS: 2 | ||
| # MEM: 3 | ||
| # steps: | ||
| # - name: Checkout | ||
| # uses: actions/[email protected] | ||
| # - uses: mamba-org/setup-micromamba@v1 | ||
| # with: | ||
| # environment-file: atlasenv.yml | ||
| # environment-name: atlasenv | ||
| # cache-environment-key: atlasenv- | ||
| # - name: Install atlas | ||
| # run: | | ||
| # python -m pip install . --no-deps -vv | ||
| # atlas --help | ||
| # # - name: Clone Example Data Repository | ||
| # # run: git clone https://github.com/metagenome-atlas/example_data.git | ||
| # - name: Run Test | ||
| # run: | | ||
| # test/test_ci.sh --resources mem=$MEM java_mem=$MEM --jobs=$N_THREADS --restart-times=2 | ||
| # - name: Store Logs | ||
| # uses: actions/upload-artifact@v2 | ||
| # with: | ||
| # name: logs | ||
| # path: test_ci/logs | ||
| # - name: Store Sample Logs | ||
| # uses: actions/upload-artifact@v2 | ||
| # with: | ||
| # name: sample_logs | ||
| # path: test_ci/Sample1/logs | ||
