Improve the performance of reading and accessing the data of PP and UM fields files #320
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # A GitHub Action to run the cf-python test suite after events on main. | ||
| name: Run test suite | ||
| # Triggers the workflow on push or PR events for the main branch (only) | ||
| on: | ||
| # For now, at least, do not enable on push to save on limited usage resource | ||
| #push: | ||
| # branches: | ||
| # - main | ||
| pull_request: | ||
| # 'reopened' enables manual retrigger via close & re-open. Disable for all | ||
| # edits to manage limited resource (PRs often change before merge-ready). | ||
| types: [opened, reopened, ready_for_review] | ||
| branches: | ||
| - main | ||
| # Note a workflow can have 1+ jobs that can run sequentially or in parallel. | ||
| jobs: | ||
| # TODO: setup parallel runs (-job-2 etc.) of sub-tests for speed-up | ||
| test-suite-job-0: | ||
| # Set-up the build matrix. We run on different distros and Python versions. | ||
| strategy: | ||
| matrix: | ||
| # Skip older ubuntu-16.04 & macos-10.15 to save usage resource | ||
| os: [ubuntu-latest, macos-latest] | ||
| # Note: keep versions quoted as strings else 3.10 taken as 3.1, etc. | ||
| python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] | ||
| # Run on new and old(er) versions of the distros we support (Linux, Mac OS) | ||
| runs-on: ${{ matrix.os }} | ||
| # The sequence of tasks that will be executed as part of this job: | ||
| steps: | ||
| - name: Checkout cf-python | ||
| uses: actions/checkout@v3 | ||
| with: | ||
| path: main | ||
| # Provide a notification message | ||
| - name: Notify about setup | ||
| run: echo Now setting up the environment for the cf-python test suite... | ||
| - name: Checkout the current cfdm main to use as the dependency | ||
| uses: actions/checkout@v3 | ||
| with: | ||
| repository: NCAS-CMS/cfdm | ||
| path: cfdm | ||
| # Prepare to run the test-suite on different versions of Python 3: | ||
| - name: Set up Python ${{ matrix.python-version }} | ||
| uses: actions/setup-python@v4 | ||
| with: | ||
| python-version: ${{ matrix.python-version }} | ||
| # Setup conda, which is the simplest way to access all dependencies, | ||
| # especially as some are C-based so otherwise difficult to setup. | ||
| - name: Setup Miniconda | ||
| uses: conda-incubator/setup-miniconda@v2 | ||
| with: | ||
| auto-update-conda: true | ||
| miniconda-version: "latest" | ||
| activate-environment: cf-latest | ||
| python-version: ${{ matrix.python-version }} | ||
| channels: ncas, conda-forge | ||
| # Ensure shell is configured with conda and pip correctly activated. | ||
| - name: Check conda and pip config | ||
| shell: bash -l {0} | ||
| run: | | ||
| echo "*Conda report:*" | ||
| conda info | ||
| conda list | ||
| conda config --show-sources | ||
| conda config --show | ||
| echo "*Pip report:*" | ||
| pip --version | ||
| pip list | ||
| # Install cf-python dependencies, excluding cfunits and cfdm, pre-testing | ||
| # We do so with conda (and pip) which was setup in a previous step. | ||
| - name: Install dependencies excluding the NCAS CF Data Tools libraries | ||
| shell: bash -l {0} | ||
| run: | | ||
| conda install -c ncas -c conda-forge cf-plot udunits2=2.2.25 | ||
| conda install -c conda-forge mpich esmpy | ||
| conda install scipy matplotlib dask | ||
| pip install pycodestyle | ||
| # Install cfunits and cfdm (from development main branch) separately, | ||
| # since it is most robust to test a no-dependency installation of cf, | ||
| # then finally install the cf-python development version. | ||
| - name: Install cfunits then development versions of cfdm and cf-python | ||
| shell: bash -l {0} | ||
| run: | | ||
| pip install cfunits | ||
| cd ${{ github.workspace }}/cfdm | ||
| pip install -e . | ||
| cd ${{ github.workspace }}/main | ||
| pip install --no-deps -e . | ||
| # Make UMRead library | ||
| - name: Make UMRead | ||
| shell: bash -l {0} | ||
| run: | | ||
| cd ${{ github.workspace }}/main/cf/umread_lib/c-lib | ||
| make | ||
| # Install the coverage library | ||
| # We do so with conda which was setup in a previous step. | ||
| - name: Install coverage | ||
| shell: bash -l {0} | ||
| run: | | ||
| conda install coverage | ||
| # Provide another notification message | ||
| - name: Notify about starting testing | ||
| run: echo Setup complete. Now starting to run the cf-python test suite... | ||
| # Create netCDF files needed for testing. A separate step is required | ||
| # for this so the files can be registered and recognised first; locally | ||
| # they are created and used on-the-fly by 'run_tests_and_coverage'. | ||
| - name: Create netCDF test files, e.g. test_file.nc | ||
| shell: bash -l {0} | ||
| run: | | ||
| cd ${{ github.workspace }}/main/cf/test | ||
| python create_test_files.py | ||
| python setup_create_field.py | ||
| ls -la | ||
| # Finally run the test suite and generate a coverage report! | ||
| - name: Run test suite and generate a coverage report | ||
| shell: bash -l {0} | ||
| run: | | ||
| cd ${{ github.workspace }}/main/cf/test | ||
| ./run_tests_and_coverage --nohtml | ||
| # For one job only, generate a coverage report: | ||
| - name: Upload coverage report to Codecov | ||
| # Get coverage from only one job (choose with Ubuntu Python 3.8 as | ||
| # representative). Note that we could use a separate workflow | ||
| # to setup Codecov reports, but given the amount of steps required to | ||
| # install including dependencies via conda, that a separate workflow | ||
| # would have to run anyway, it is simplest to add it in this flow. | ||
| # Also, it means code coverage is only generated if the test suite is | ||
| # passing at least for that job, avoiding useless coverage reports. | ||
| uses: codecov/codecov-action@v3 | ||
| if: | | ||
|
Check failure on line 149 in .github/workflows/run-test-suite.yml
|
||
| matrix.os == "ubuntu-latest" && matrix.python-version == "3.9" | ||
| with: | ||
| file: | | ||
| ${{ github.workspace }}/main/cf/test/cf_coverage_reports/coverage.xml | ||
| fail_ci_if_error: true | ||
| flags: unittests | ||
| name: codecov-umbrella | ||
| # End with a message indicating the suite has completed its run | ||
| - name: Notify about a completed run | ||
| run: | | ||
| echo The cf-python test suite has completed and you may now | ||
| echo inspect the results. | ||
