Disaster Recovery Updates #11458
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Integration | |
| on: | |
| pull_request: | |
| branches: | |
| - main | |
| jobs: | |
| lint-api: | |
| name: Python - Lint | |
| runs-on: ubuntu-24.04 | |
| strategy: | |
| matrix: | |
| python-version: [3.12.3] | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v4 | |
| - name: Install ubuntu pre-requisites (api) | |
| # The python gdal and R component relies on libgdal-dev being installed. | |
| # cffdrs requires libudunits2-dev | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get -y install libudunits2-dev r-base-dev libgdal-dev libproj-dev libgeos-dev libsqlite3-dev libtirpc-dev | |
| - name: Setup Python ${{ matrix.python-version }} (api) | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Cache poetry installer | |
| uses: actions/cache@v4 | |
| id: cache-poetry-installer | |
| env: | |
| cache-name: cache-poetry-installer | |
| with: | |
| path: "~/poetry_installer" | |
| key: "poetry-installer-1.8.3" | |
| - name: Download poetry installer | |
| if: steps.cache-poetry-installer.outputs.cache-hit != 'true' | |
| run: | | |
| echo | |
| mkdir ~/poetry_installer | |
| curl -sSL https://install.python-poetry.org > ~/poetry_installer/install-poetry.py | |
| - name: Install poetry (api) | |
| run: | | |
| cd ~/poetry_installer | |
| python install-poetry.py --version 1.8.3 | |
| poetry config virtualenvs.create true | |
| poetry config virtualenvs.in-project false | |
| - name: Cache poetry | |
| uses: actions/cache@v4 | |
| env: | |
| cache-name: cache-poetry | |
| with: | |
| path: ~/.cache/pypoetry | |
| key: ${{ runner.os }}-poetry-1.8.3-cache-${{ hashFiles('**/poetry.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-poetry-1.8.3-cache- | |
| - name: Cache .venv | |
| id: cache-venv | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/work/wps/wps/api/.venv | |
| key: ${{ runner.os }}-venv-poetry-1.8.3-${{ hashFiles('**/poetry.lock') }} | |
| - name: Install python dependencies using poetry (api) | |
| if: steps.cache-venv.outputs.cache-hit != 'true' | |
| working-directory: ./api | |
| run: | | |
| poetry run python -m pip install --upgrade pip | |
| poetry install | |
| poetry run python -m pip install -U setuptools wheel | |
| poetry run python -m pip install --no-build-isolation --no-cache-dir --force-reinstall gdal==$(gdal-config --version) | |
| - name: Lint (api) | |
| # We used to be able to do linting before installing gdal, but it's not possible anymore. | |
| # We can however place it ahead of the R installs. | |
| working-directory: ./api | |
| run: | | |
| poetry run ruff app/*.py app/**/*.py | |
| test-api: | |
| name: Python - Test with coverage | |
| runs-on: ubuntu-24.04 | |
| strategy: | |
| matrix: | |
| # Match versions for python to Ubuntu 24.04 LTS at the time of writing. Keep R at same version. | |
| python-version: [3.12.3] | |
| R: ["4.1.2"] | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v4 | |
| with: | |
| # For sonar-scanner to work properly we can't use a shallow fetch. | |
| fetch-depth: 0 | |
| - name: Install ubuntu pre-requisites (api) | |
| # The python gdal and R component relies on libgdal-dev being installed. | |
| # cffdrs requires libudunits2-dev | |
| # The api uses wkhtmltopdf to generate pdf's. | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get -y install wkhtmltopdf libudunits2-dev r-base-dev build-essential libgdal-dev libproj-dev libgeos-dev libsqlite3-dev libtirpc-dev | |
| - name: Setup Python ${{ matrix.python-version }} (api) | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Cache poetry installer | |
| uses: actions/cache@v4 | |
| id: cache-poetry-installer | |
| env: | |
| cache-name: cache-poetry-installer | |
| with: | |
| path: "~/poetry_installer" | |
| key: "poetry-installer-1.8.3" | |
| - name: Download poetry installer | |
| if: steps.cache-poetry-installer.outputs.cache-hit != 'true' | |
| run: | | |
| echo | |
| mkdir ~/poetry_installer | |
| curl -sSL https://install.python-poetry.org > ~/poetry_installer/install-poetry.py | |
| - name: Install poetry (api) | |
| run: | | |
| cd ~/poetry_installer | |
| python install-poetry.py --version 1.8.3 | |
| poetry config virtualenvs.create true | |
| poetry config virtualenvs.in-project false | |
| # poetry cache folder: /home/runner/.cache/pypoetry | |
| - name: Cache poetry | |
| uses: actions/cache@v4 | |
| env: | |
| cache-name: cache-poetry | |
| with: | |
| path: ~/.cache/pypoetry | |
| key: ${{ runner.os }}-poetry-1.8.3-cache-${{ hashFiles('**/poetry.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-poetry-1.8.3-cache- | |
| - name: Cache .venv | |
| id: cache-venv | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/work/wps/wps/api/.venv | |
| key: ${{ runner.os }}-venv-poetry-1.8.3-${{ hashFiles('**/poetry.lock') }} | |
| - name: Install python dependencies using poetry (api) | |
| if: steps.cache-venv.outputs.cache-hit != 'true' | |
| working-directory: ./api | |
| run: | | |
| poetry run python -m pip install --upgrade pip | |
| poetry install | |
| poetry run python -m pip install -U setuptools wheel | |
| poetry run python -m pip install --no-build-isolation --no-cache-dir --force-reinstall gdal==$(gdal-config --version) | |
| - uses: r-lib/actions/setup-r@v2 | |
| # r-lib/actions/setup-r@v2 is supposed to install version ${{ matrix.R }}, BUT, it doesn't. | |
| # When asking for 4.1.2 - it's installing a new version! Or at least reporting a newer | |
| # version. | |
| with: | |
| r-version: ${{ matrix.R }} | |
| - name: Cache /home/runner/work/_temp/Library | |
| # When installing cffdrs, a bunch of stuff is downloaded and compiled and placed in | |
| # /home/runner/work/_temp/Library ; By caching this folder, subsequent calls to | |
| # install cffdrs run much faster. | |
| id: cache-r-cffdrs | |
| uses: actions/cache@v4 | |
| with: | |
| path: /home/runner/work/_temp/Library | |
| key: ${{ runner.os }}-r-${{ matrix.R }}-cffdrs | |
| - name: Install R dependencies (api) | |
| run: R -e "install.packages('cffdrs')" | |
| - name: Unit Test with coverage (api) | |
| working-directory: ./api | |
| run: | | |
| export LD_LIBRARY_PATH=$(poetry run python -m rpy2.situation LD_LIBRARY_PATH):${LD_LIBRARY_PATH} | |
| export CLASSPATH=./libs/REDapp_Lib.jar:./libs/WTime.jar:./libs/hss-java.jar:$CLASSPATH | |
| export ORIGINS=testorigin | |
| export SFMS_SECRET=secret | |
| poetry run coverage run --source=app -m pytest app/tests -x -o log_cli=true --disable-warnings -vvv | |
| - name: Create coverage report (api) | |
| working-directory: ./api | |
| shell: bash | |
| run: | | |
| poetry run coverage report | |
| poetry run coverage xml -o coverage-reports/coverage-report.xml | |
| - name: Archive coverage report (api) | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: api-coverage-report | |
| path: ./api/coverage-reports/coverage-report.xml | |
| lint-and-test-web: | |
| name: Web - Lint, Test with coverage | |
| runs-on: ubuntu-24.04 | |
| strategy: | |
| matrix: | |
| node-version: [20.x] | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v4 | |
| with: | |
| # For sonar-scanner to work properly we can't use a shallow fetch. | |
| fetch-depth: 0 | |
| - name: Use Node.js ${{ matrix.node-version }} | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: ${{ matrix.node-version }} | |
| - uses: actions/cache@v4 | |
| with: | |
| path: | | |
| **/node_modules | |
| ~/.cache/Cypress | |
| key: ${{ runner.os }}-modules-${{ hashFiles('**/yarn.lock') }} | |
| - name: enable corepack | |
| run: corepack enable | |
| - name: use new yarn | |
| run: yarn set version berry | |
| - name: Install node dependencies (web) | |
| working-directory: ./web | |
| if: steps.yarn-cache.outputs.cache-hit != 'true' | |
| run: yarn install | |
| - name: Lint (web) | |
| working-directory: ./web | |
| run: yarn run lint | |
| - name: Cypress tests with coverage (web) | |
| working-directory: ./web | |
| run: yarn run cypress:ci | |
| - name: Unit tests (web) | |
| working-directory: ./web | |
| run: yarn run coverage:ci | |
| - name: Merge and finalize test coverage (web) | |
| working-directory: ./web | |
| run: yarn run finalizeCoverage | |
| - name: Archive coverage report | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: web-coverage-report | |
| path: ./web/finalCoverage | |
| upload-code-coverage: | |
| name: Coverage with CodeCov | |
| runs-on: ubuntu-24.04 | |
| needs: [lint-and-test-web, test-api] | |
| steps: | |
| # we need to checkout, so that we have codecov.yml | |
| - name: Checkout repo | |
| uses: actions/checkout@v4 | |
| - name: Download all workflow run artifacts | |
| uses: actions/download-artifact@v4 | |
| - name: Upload test coverage to Codecov | |
| uses: codecov/codecov-action@v4 | |
| with: | |
| token: ${{secrets.CODECOV_TOKEN}} | |
| # we want to ensure code coverage is good, so we fail on error. (default is false) | |
| fail_ci_if_error: true | |
| lint-and-test-prune: | |
| name: Backup Prune - Lint and Test | |
| # We have to use an older ubuntu version, because of the older version of python that the | |
| # backup script uses. | |
| runs-on: ubuntu-24.04 | |
| strategy: | |
| matrix: | |
| python-version: [3.12.3] | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v4 | |
| with: | |
| # For sonar-scanner to work properly we can't use a shallow fetch. | |
| fetch-depth: 0 | |
| - name: Setup Python ${{ matrix.python-version }} (api) | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Cache poetry installer | |
| uses: actions/cache@v4 | |
| id: cache-poetry-installer | |
| env: | |
| cache-name: cache-poetry-installer | |
| with: | |
| path: "~/poetry_installer" | |
| key: "poetry-installer-1.8.3" | |
| - name: Download poetry installer | |
| if: steps.cache-poetry-installer.outputs.cache-hit != 'true' | |
| run: | | |
| echo | |
| mkdir ~/poetry_installer | |
| curl -sSL https://install.python-poetry.org > ~/poetry_installer/install-poetry.py | |
| - name: Install poetry (api) | |
| run: | | |
| cd ~/poetry_installer | |
| python install-poetry.py --version 1.8.3 | |
| poetry config virtualenvs.create true | |
| poetry config virtualenvs.in-project false | |
| # poetry cache folder: /home/runner/.cache/pypoetry | |
| - name: Cache poetry | |
| uses: actions/cache@v4 | |
| env: | |
| cache-name: cache-poetry | |
| with: | |
| path: ~/.cache/pypoetry | |
| key: ${{ runner.os }}-poetry-prune-cache-${{ hashFiles('**/poetry.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-poetry-prune-cache- | |
| - name: Install python dependencies using poetry (s3-backup) | |
| working-directory: ./openshift/s3-backup/docker | |
| run: | | |
| poetry install | |
| - name: Lint (s3-backup) | |
| working-directory: ./openshift/s3-backup/docker | |
| run: poetry run ruff check *.py | |
| - name: Unit tests (s3-backup) | |
| working-directory: ./openshift/s3-backup/docker | |
| run: | | |
| poetry run pytest -v |