Feature: perfect restart and n-cores CI #592
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: build | |
| on: | |
| push: | |
| branches: [ main ] | |
| pull_request: | |
| branches: [ main ] | |
| # Allows you to run this workflow manually from the Actions tab | |
| workflow_dispatch: | |
| inputs: | |
| pr: | |
| description: "PR to test" | |
| required: true | |
| jobs: | |
| Model_Testing: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| configuration: [nwm_ana, nwm_long_range, gridded, reach, reach_lakes] | |
| runs-on: ubuntu-latest | |
| env: | |
| MPI_HOME: /usr/share/miniconda | |
| NETCDF: /usr/share/miniconda | |
| NETCDF_INCLUDES: /usr/share/miniconda/include | |
| NETCDF_LIBRARIES: /usr/share/miniconda/lib | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.10' | |
| - name: Checkout candidate (pull request / push) | |
| if: ${{ github.event_name == 'pull_request' || github.event_name == 'push' }} | |
| uses: actions/checkout@v4 | |
| with: | |
| path: candidate | |
| - name: Checkout candidate (manual) | |
| if: ${{ github.event_name == 'workflow_dispatch' }} | |
| env: | |
| GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} | |
| run: gh repo clone ${{ github.repository }} candidate && cd candidate && gh pr checkout -R ${{ github.repository }} ${{ github.event.inputs.pr }} | |
| - name: Checkout reference (pull request) | |
| if: ${{ github.event_name == 'pull_request' }} | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.pull_request.base.ref }} | |
| path: reference | |
| - name: Checkout reference (push) | |
| if: ${{ github.event_name == 'push' }} | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.before }} | |
| path: reference | |
| - name: Checkout reference (manual) | |
| if: ${{ github.event_name == 'workflow_dispatch' }} | |
| env: | |
| GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} | |
| run: gh repo clone ${{ github.repository }} reference && cd reference && git checkout origin/$(gh pr view ${{ github.event.inputs.pr }} --json baseRefName --jq '.baseRefName') | |
| - name: Install dependencies with apt-get | |
| run: | | |
| sudo apt-get update \ | |
| && sudo apt-get install -yq --no-install-recommends \ | |
| wget \ | |
| curl \ | |
| bzip2 \ | |
| ca-certificates \ | |
| libhdf5-dev \ | |
| gfortran \ | |
| g++ \ | |
| m4 \ | |
| make \ | |
| libswitch-perl \ | |
| git \ | |
| bc \ | |
| openmpi-bin openmpi-common libopenmpi-dev \ | |
| libxml2-dev \ | |
| libnetcdf-dev \ | |
| libnetcdff-dev | |
| - name: Install dependencies with pip | |
| run: | | |
| python3 -m pip install matplotlib numpy xarray dask netCDF4 pygithub | |
| - name: Compile reference | |
| run: | | |
| cd $GITHUB_WORKSPACE/reference | |
| cmake -B build | |
| make -C build -j | |
| - name: Compile candidate | |
| run: | | |
| cd $GITHUB_WORKSPACE/candidate | |
| cmake -B build | |
| make -C build -j | |
| - name: Run reference model | |
| run: | | |
| cd $GITHUB_WORKSPACE/reference/build/Run | |
| make run-croton-${{ matrix.configuration }} | |
| - name: Run candidate model | |
| run: | | |
| cd $GITHUB_WORKSPACE/candidate/build/Run | |
| make run-croton-${{ matrix.configuration }} | |
| - name: generic - Compare HYDRO_RST.* output with xrcmp | |
| if: ${{ always() }} | |
| run: | | |
| cd $GITHUB_WORKSPACE/candidate/build/Run | |
| for file in output_${{ matrix.configuration }}/HYDRO_RST.*; do\ | |
| python ${GITHUB_WORKSPACE}/candidate/tests/utils/xrcmp.py \ | |
| --candidate $file \ | |
| --reference $GITHUB_WORKSPACE/reference/build/Run/$file \ | |
| --log_file $file_diff.txt \ | |
| --n_cores 1; \ | |
| done | |
| - name: generic - Compare RESTART.* output with xrcmp | |
| if: ${{ always() }} | |
| run: | | |
| cd $GITHUB_WORKSPACE/candidate/build/Run | |
| for file in output_${{ matrix.configuration }}/RESTART.*; do\ | |
| python ${GITHUB_WORKSPACE}/candidate/tests/utils/xrcmp.py \ | |
| --candidate $file \ | |
| --reference $GITHUB_WORKSPACE/reference/build/Run/$file \ | |
| --log_file $file_diff.txt \ | |
| --n_cores 1; \ | |
| done | |
| - name: generic - Compare last *.CHANOBS_DOMAIN1 output with xrcmp | |
| if: ${{ always() }} | |
| run: | | |
| cd $GITHUB_WORKSPACE/candidate/build/Run | |
| file=$(ls -t output_${{ matrix.configuration }}/*.CHANOBS_DOMAIN1 | head -n 1) | |
| python ${GITHUB_WORKSPACE}/candidate/tests/utils/xrcmp.py \ | |
| --candidate $file \ | |
| --reference $GITHUB_WORKSPACE/reference/build/Run/$file \ | |
| --log_file $file_diff.txt \ | |
| --n_cores 1; \ | |
| - name: generic - Compare last *.CHRTOUT_DOMAIN1 output with xrcmp | |
| if: ${{ always() }} | |
| run: | | |
| cd $GITHUB_WORKSPACE/candidate/build/Run | |
| file=$(ls -t output_${{ matrix.configuration }}/*.CHRTOUT_DOMAIN1 | head -n 1) | |
| python ${GITHUB_WORKSPACE}/candidate/tests/utils/xrcmp.py \ | |
| --candidate $file \ | |
| --reference $GITHUB_WORKSPACE/reference/build/Run/$file \ | |
| --log_file $file_diff.txt \ | |
| --n_cores 1; \ | |
| - name: generic - Compare last *.LSMOUT_DOMAIN1 output with xrcmp | |
| if: ${{ always() }} | |
| run: | | |
| cd $GITHUB_WORKSPACE/candidate/build/Run | |
| file=$(ls -t output_${{ matrix.configuration }}/*.LSMOUT_DOMAIN1 | head -n 1) | |
| python ${GITHUB_WORKSPACE}/candidate/tests/utils/xrcmp.py \ | |
| --candidate $file \ | |
| --reference $GITHUB_WORKSPACE/reference/build/Run/$file \ | |
| --log_file $file_diff.txt \ | |
| --n_cores 1; \ | |
| - name: generic - Compare last *.RTOUT_DOMAIN1 output with xrcmp | |
| if: ${{ always() }} | |
| run: | | |
| cd $GITHUB_WORKSPACE/candidate/build/Run | |
| file=$(ls -t output_${{ matrix.configuration }}/*.RTOUT_DOMAIN1 | head -n 1) | |
| python ${GITHUB_WORKSPACE}/candidate/tests/utils/xrcmp.py \ | |
| --candidate $file \ | |
| --reference $GITHUB_WORKSPACE/reference/build/Run/$file \ | |
| --log_file $file_diff.txt \ | |
| --n_cores 1; \ | |
| - name: generic - Compare output with compare_output | |
| if: ${{ always() }} | |
| run: | | |
| cd $GITHUB_WORKSPACE/candidate/build/Run | |
| mkdir output_diff | |
| python -c \ | |
| "import sys; \ | |
| sys.path.append('${GITHUB_WORKSPACE}/candidate/tests/utils'); \ | |
| import compare_output; \ | |
| from pathlib import Path; \ | |
| compare_output.plot_diffs('${GITHUB_WORKSPACE}/candidate/build/Run/output_diff', \ | |
| '${GITHUB_WORKSPACE}/candidate/build/Run/output_${{ matrix.configuration }}/', \ | |
| '${GITHUB_WORKSPACE}/reference/build/Run/output_${{ matrix.configuration }}/', \ | |
| '${{ matrix.configuration }}')" | |
| - name: generic - Copy test results from container | |
| if: ${{ always() }} | |
| run: | | |
| mkdir -p $GITHUB_WORKSPACE/test_report | |
| cp -r $GITHUB_WORKSPACE/candidate/build/Run/output_diff/diff_plots/* $GITHUB_WORKSPACE/test_report/ | |
| - name: generic - Attach diff plots to PR | |
| if: ${{ failure() }} | |
| shell: bash | |
| run: | | |
| cd $GITHUB_WORKSPACE/candidate/tests/local/utils | |
| bash attach_all_plots.bash $(jq --raw-output .pull_request.number "$GITHUB_EVENT_PATH") ${{ matrix.configuration }} generic | |
| - name: generic - Archive test results to GitHub | |
| if: ${{ failure() }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: test-reports | |
| path: | | |
| ${{ github.workspace }}/test_report/* | |
| # n-cores test | |
| - name: Run parallel candidate model | |
| run: | | |
| rm -r $GITHUB_WORKSPACE/test_report/ | |
| cd $GITHUB_WORKSPACE/candidate/build/Run | |
| make clean | |
| make run-croton-${{ matrix.configuration }}-parallel | |
| - name: n-cores - Compare HYDRO_RST.* output with xrcmp | |
| if: ${{ always() }} | |
| run: | | |
| cd $GITHUB_WORKSPACE/candidate/build/Run | |
| for file in output_${{ matrix.configuration }}/HYDRO_RST.*; do\ | |
| python ${GITHUB_WORKSPACE}/candidate/tests/utils/xrcmp.py \ | |
| --candidate $file \ | |
| --reference $GITHUB_WORKSPACE/reference/build/Run/$file \ | |
| --log_file $file_diff.txt \ | |
| --n_cores 1; \ | |
| done | |
| - name: n-cores - Compare RESTART.* output with xrcmp | |
| if: ${{ always() }} | |
| run: | | |
| cd $GITHUB_WORKSPACE/candidate/build/Run | |
| for file in output_${{ matrix.configuration }}/RESTART.*; do\ | |
| python ${GITHUB_WORKSPACE}/candidate/tests/utils/xrcmp.py \ | |
| --candidate $file \ | |
| --reference $GITHUB_WORKSPACE/reference/build/Run/$file \ | |
| --log_file $file_diff.txt \ | |
| --n_cores 1; \ | |
| done | |
| - name: n-cores - Compare last *.CHANOBS_DOMAIN1 output with xrcmp | |
| if: ${{ always() }} | |
| run: | | |
| cd $GITHUB_WORKSPACE/candidate/build/Run | |
| file=$(ls -t output_${{ matrix.configuration }}/*.CHANOBS_DOMAIN1 | head -n 1) | |
| python ${GITHUB_WORKSPACE}/candidate/tests/utils/xrcmp.py \ | |
| --candidate $file \ | |
| --reference $GITHUB_WORKSPACE/reference/build/Run/$file \ | |
| --log_file $file_diff.txt \ | |
| --n_cores 1; \ | |
| - name: n-cores - Compare last *.CHRTOUT_DOMAIN1 output with xrcmp | |
| if: ${{ always() }} | |
| run: | | |
| cd $GITHUB_WORKSPACE/candidate/build/Run | |
| file=$(ls -t output_${{ matrix.configuration }}/*.CHRTOUT_DOMAIN1 | head -n 1) | |
| python ${GITHUB_WORKSPACE}/candidate/tests/utils/xrcmp.py \ | |
| --candidate $file \ | |
| --reference $GITHUB_WORKSPACE/reference/build/Run/$file \ | |
| --log_file $file_diff.txt \ | |
| --n_cores 1; \ | |
| - name: n-cores - Compare last *.LSMOUT_DOMAIN1 output with xrcmp | |
| if: ${{ always() }} | |
| run: | | |
| cd $GITHUB_WORKSPACE/candidate/build/Run | |
| file=$(ls -t output_${{ matrix.configuration }}/*.LSMOUT_DOMAIN1 | head -n 1) | |
| python ${GITHUB_WORKSPACE}/candidate/tests/utils/xrcmp.py \ | |
| --candidate $file \ | |
| --reference $GITHUB_WORKSPACE/reference/build/Run/$file \ | |
| --log_file $file_diff.txt \ | |
| --n_cores 1; \ | |
| - name: n-cores - Compare last *.RTOUT_DOMAIN1 output with xrcmp | |
| if: ${{ always() }} | |
| run: | | |
| cd $GITHUB_WORKSPACE/candidate/build/Run | |
| file=$(ls -t output_${{ matrix.configuration }}/*.RTOUT_DOMAIN1 | head -n 1) | |
| python ${GITHUB_WORKSPACE}/candidate/tests/utils/xrcmp.py \ | |
| --candidate $file \ | |
| --reference $GITHUB_WORKSPACE/reference/build/Run/$file \ | |
| --log_file $file_diff.txt \ | |
| --n_cores 1; \ | |
| - name: n-cores - Compare output with compare_output | |
| if: ${{ always() }} | |
| run: | | |
| cd $GITHUB_WORKSPACE/candidate/build/Run | |
| rm -rf output_diff | |
| mkdir output_diff | |
| python -c \ | |
| "import sys; \ | |
| sys.path.append('${GITHUB_WORKSPACE}/candidate/tests/utils'); \ | |
| import compare_output; \ | |
| from pathlib import Path; \ | |
| compare_output.plot_diffs('${GITHUB_WORKSPACE}/candidate/build/Run/output_diff', \ | |
| '${GITHUB_WORKSPACE}/candidate/build/Run/output_${{ matrix.configuration }}/', \ | |
| '${GITHUB_WORKSPACE}/reference/build/Run/output_${{ matrix.configuration }}/', \ | |
| '${{ matrix.configuration }}')" | |
| - name: n-cores - Copy test results from container | |
| if: ${{ always() }} | |
| run: | | |
| mkdir -p $GITHUB_WORKSPACE/test_report | |
| cp -r $GITHUB_WORKSPACE/candidate/build/Run/output_diff/diff_plots/* $GITHUB_WORKSPACE/test_report/ | |
| - name: n-cores - Attach diff plots to PR | |
| if: ${{ failure() }} | |
| shell: bash | |
| run: | | |
| cd $GITHUB_WORKSPACE/candidate/tests/local/utils | |
| bash attach_all_plots.bash $(jq --raw-output .pull_request.number "$GITHUB_EVENT_PATH") ${{ matrix.configuration }} n-cores | |
| - name: n-cores - Archive test results to GitHub | |
| if: ${{ failure() }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: test-reports | |
| path: | | |
| ${{ github.workspace }}/test_report/* | |
| # Testing perfect restart, not cleaning candidate model output | |
| - name: Setup and run candidate model perfect restart startup | |
| run: | | |
| rm -r $GITHUB_WORKSPACE/test_report/ | |
| cd $GITHUB_WORKSPACE/candidate/build/Run | |
| sed -i 's|RESTART_FILENAME_REQUESTED = "RESTART/RESTART.2011082600_DOMAIN1"|RESTART_FILENAME_REQUESTED = "./RESTART.2011090100_DOMAIN1"|' namelist.hrldas | |
| sed -i 's/KDAY = 7/KDAY = 1/' namelist.hrldas | |
| rm output_${{ matrix.configuration }}/RESTART.2011090200_DOMAIN1 | |
| rm output_${{ matrix.configuration }}/HYDRO_RST.2011-09-02_00:00_DOMAIN1 | |
| make run-croton-${{ matrix.configuration }}-parallel | |
| - name: restart - Compare HYDRO_RST.* output with xrcmp | |
| if: ${{ always() }} | |
| run: | | |
| cd $GITHUB_WORKSPACE/candidate/build/Run | |
| for file in output_${{ matrix.configuration }}/HYDRO_RST.2011-09-02_00:00_DOMAIN1; do\ | |
| python ${GITHUB_WORKSPACE}/candidate/tests/utils/xrcmp.py \ | |
| --candidate $file \ | |
| --reference $GITHUB_WORKSPACE/reference/build/Run/$file \ | |
| --log_file $file_diff.txt \ | |
| --n_cores 1; \ | |
| done | |
| - name: restart - Compare RESTART.* output with xrcmp | |
| if: ${{ always() }} | |
| run: | | |
| cd $GITHUB_WORKSPACE/candidate/build/Run | |
| for file in output_${{ matrix.configuration }}/RESTART.2011090200_DOMAIN1; do\ | |
| python ${GITHUB_WORKSPACE}/candidate/tests/utils/xrcmp.py \ | |
| --candidate $file \ | |
| --reference $GITHUB_WORKSPACE/reference/build/Run/$file \ | |
| --log_file $file_diff.txt \ | |
| --n_cores 1; \ | |
| done | |
| - name: restart - Compare last *.CHANOBS_DOMAIN1 output with xrcmp | |
| if: ${{ always() }} | |
| run: | | |
| cd $GITHUB_WORKSPACE/candidate/build/Run | |
| file=$(ls -t output_${{ matrix.configuration }}/*.CHANOBS_DOMAIN1 | head -n 1) | |
| python ${GITHUB_WORKSPACE}/candidate/tests/utils/xrcmp.py \ | |
| --candidate $file \ | |
| --reference $GITHUB_WORKSPACE/reference/build/Run/$file \ | |
| --log_file $file_diff.txt \ | |
| --n_cores 1; \ | |
| - name: restart - Compare last *.CHRTOUT_DOMAIN1 output with xrcmp | |
| if: ${{ always() }} | |
| run: | | |
| cd $GITHUB_WORKSPACE/candidate/build/Run | |
| file=$(ls -t output_${{ matrix.configuration }}/*.CHRTOUT_DOMAIN1 | head -n 1) | |
| python ${GITHUB_WORKSPACE}/candidate/tests/utils/xrcmp.py \ | |
| --candidate $file \ | |
| --reference $GITHUB_WORKSPACE/reference/build/Run/$file \ | |
| --log_file $file_diff.txt \ | |
| --n_cores 1; \ | |
| - name: restart - Compare last *.LSMOUT_DOMAIN1 output with xrcmp | |
| if: ${{ always() }} | |
| run: | | |
| cd $GITHUB_WORKSPACE/candidate/build/Run | |
| file=$(ls -t output_${{ matrix.configuration }}/*.LSMOUT_DOMAIN1 | head -n 1) | |
| python ${GITHUB_WORKSPACE}/candidate/tests/utils/xrcmp.py \ | |
| --candidate $file \ | |
| --reference $GITHUB_WORKSPACE/reference/build/Run/$file \ | |
| --log_file $file_diff.txt \ | |
| --n_cores 1; \ | |
| - name: restart - Compare last *.RTOUT_DOMAIN1 output with xrcmp | |
| if: ${{ always() }} | |
| run: | | |
| cd $GITHUB_WORKSPACE/candidate/build/Run | |
| file=$(ls -t output_${{ matrix.configuration }}/*.RTOUT_DOMAIN1 | head -n 1) | |
| python ${GITHUB_WORKSPACE}/candidate/tests/utils/xrcmp.py \ | |
| --candidate $file \ | |
| --reference $GITHUB_WORKSPACE/reference/build/Run/$file \ | |
| --log_file $file_diff.txt \ | |
| --n_cores 1; \ | |
| - name: restart - Compare output with compare_output | |
| if: ${{ always() }} | |
| run: | | |
| cd $GITHUB_WORKSPACE/candidate/build/Run | |
| rm -rf output_diff | |
| mkdir output_diff | |
| python -c \ | |
| "import sys; \ | |
| sys.path.append('${GITHUB_WORKSPACE}/candidate/tests/utils'); \ | |
| import compare_output; \ | |
| from pathlib import Path; \ | |
| compare_output.plot_diffs('${GITHUB_WORKSPACE}/candidate/build/Run/output_diff', \ | |
| '${GITHUB_WORKSPACE}/candidate/build/Run/output_${{ matrix.configuration }}/', \ | |
| '${GITHUB_WORKSPACE}/reference/build/Run/output_${{ matrix.configuration }}/', \ | |
| '${{ matrix.configuration }}')" | |
| - name: restart - Copy test results from container | |
| if: ${{ always() }} | |
| run: | | |
| mkdir -p $GITHUB_WORKSPACE/test_report | |
| cp -r $GITHUB_WORKSPACE/candidate/build/Run/output_diff/diff_plots/* $GITHUB_WORKSPACE/test_report/ | |
| - name: restart - Attach diff plots to PR | |
| if: ${{ failure() }} | |
| shell: bash | |
| run: | | |
| cd $GITHUB_WORKSPACE/candidate/tests/local/utils | |
| bash attach_all_plots.bash $(jq --raw-output .pull_request.number "$GITHUB_EVENT_PATH") ${{ matrix.configuration }} perfect-restart | |
| - name: restart - Archive test results to GitHub | |
| if: ${{ failure() }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: test-reports | |
| path: | | |
| ${{ github.workspace }}/test_report/* |