ci-build.yml

name: CI Build
on:
  workflow_dispatch:
  pull_request:
  # By default, the pull_request event type is not triggered when a PR is merged into main or develop
  push:
    branches:
      - main
      - develop

jobs:
  lint-and-unit-test:
    name: Lint & Unit Tests
    runs-on: ubuntu-20.04
    steps:
      - name: Checkout repo
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
      - name: Setup Node.js
        uses: actions/setup-node@b39b52d1213e96004bfcb1c61a8a6fa8ab84f3e8 # v4
        with:
          node-version: 20
          cache: 'yarn'

      - name: Install dependencies
        # Ubuntu 16+ does not install libgconf-2-4 by default, so we need to install it ourselves (for Cypress)
        run: |
          sudo apt-get install libgconf-2-4
          yarn --immutable

      # Linting and unit testing
      - name: Run linting
        run: yarn lint
      - name: Run unit tests
        run: yarn test:unit

      # Test coverage upload
      - name: Upload unit test coverage for the Common package
        if: success()
        uses: codecov/codecov-action@e0b68c6749509c5f83f984dd99a76a1c1a231044 # v4
        with:
          directory: ./packages/datagateway-common/
          flags: common
          fail_ci_if_error: true
          token: ${{ secrets.CODECOV_TOKEN }}
      - name: Upload unit test coverage for the DataView package
        if: success()
        uses: codecov/codecov-action@e0b68c6749509c5f83f984dd99a76a1c1a231044 # v4
        with:
          directory: ./packages/datagateway-dataview/
          flags: dataview
          fail_ci_if_error: true
          token: ${{ secrets.CODECOV_TOKEN }}
      - name: Upload unit test coverage for the Search package
        if: success()
        uses: codecov/codecov-action@e0b68c6749509c5f83f984dd99a76a1c1a231044 # v4
        with:
          directory: ./packages/datagateway-search/
          flags: search
          fail_ci_if_error: true
          token: ${{ secrets.CODECOV_TOKEN }}
      - name: Upload unit test coverage for the Download package
        if: success()
        uses: codecov/codecov-action@e0b68c6749509c5f83f984dd99a76a1c1a231044 # v4
        with:
          directory: ./packages/datagateway-download/
          flags: download
          fail_ci_if_error: true
          token: ${{ secrets.CODECOV_TOKEN }}

  dataview-e2e-tests:
    name: DataGateway DataView End to End Tests
    runs-on: ubuntu-20.04
    steps:
      - name: Checkout repo
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
      - name: Add apt repo
        run: sudo add-apt-repository universe
      - name: Setup Java
        uses: actions/setup-java@387ac29b308b003ca37ba93a6cab5eb57c8f5f93 # v4
        with:
          distribution: 'zulu'
          java-version: 8
          java-package: jdk
      - name: Setup Python
        uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5
        with:
          python-version: 3.6
          architecture: x64

      # ICAT Ansible clone and install dependencies
      - name: Checkout icat-ansible
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
        with:
          repository: icatproject-contrib/icat-ansible
          ref: master
          path: icat-ansible
      - name: Install Ansible
        run: pip install -r icat-ansible/requirements.txt

      # Prep for running the playbook
      - name: Create hosts file
        run: echo -e "[icat_test_hosts]\nlocalhost ansible_connection=local" > icat-ansible/hosts
      - name: Prepare vault pass
        run: echo -e "icattravispw" > icat-ansible/vault_pass.txt
      - name: Move vault to directory it'll get detected by Ansible
        run: mv icat-ansible/vault.yml icat-ansible/group_vars/all
      - name: Replace default payara user with Actions user
        run: |
          sed -i -e "s/^payara_user: \"glassfish\"/payara_user: \"runner\"/" icat-ansible/group_vars/all/vars.yml
      - name: Amending roles
        run: |
          sed -i 's/role: authn_uows_isis/role: authn_anon/' icat-ansible/icat_test_hosts.yml

      # Force hostname to localhost - bug fix for previous ICAT Ansible issues on Actions
      - name: Change hostname to localhost
        run: sudo hostname -b localhost

      # Remove existing MySQL installation so it doesn't interfere with GitHub Actions
      - name: Remove existing mysql
        run: |
          sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
          sudo apt-get remove --purge "mysql*"
          sudo rm -rf /var/lib/mysql* /etc/mysql

      # Create local instance of ICAT
      - name: Run ICAT Ansible Playbook
        run: |
          ansible-playbook icat-ansible/icat_test_hosts.yml -i icat-ansible/hosts --vault-password-file icat-ansible/vault_pass.txt -vv

      # Fixes on ICAT components needed for e2e tests
      - name: Add anon user to rootUserNames
        run: |
          awk -F" =" '/rootUserNames/{$2="= simple/root anon/anon";print;next}1' /home/runner/install/icat.server/run.properties > /home/runner/install/icat.server/run.properties.tmp
      - name: Apply rootUserNames change
        run: |
          mv -f /home/runner/install/icat.server/run.properties.tmp /home/runner/install/icat.server/run.properties
      - name: Reinstall ICAT Server
        run: |
          cd /home/runner/install/icat.server/ && ./setup -vv install

      - name: Checkout datagateway-api
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
        with:
          repository: ral-facilities/datagateway-api
          path: datagateway-api
          ref: v9.0.3

      # DataGateway API file setup
      - name: Create search_api_mapping.json
        run: cp datagateway-api/datagateway_api/search_api_mapping.json.example datagateway-api/datagateway_api/search_api_mapping.json
      - name: Create log file
        run: touch datagateway-api/datagateway_api/logs.log

      - name: Create config.yaml
        run: cp datagateway-api/datagateway_api/config.yaml.example datagateway-api/datagateway_api/config.yaml

      # DataGateway API dependencies
      - name: Install Poetry
        run: pip install poetry
      - name: Install dependencies
        run: cd datagateway-api/; poetry install
      - name: Add dummy data to icatdb
        run: |
          cd datagateway-api/; poetry run python -m util.icat_db_generator

      - name: Start API
        run: cd datagateway-api/; nohup poetry run python -m datagateway_api.src.main > api-output.txt &

      # E2E tests
      - name: Setup Node.js
        uses: actions/setup-node@b39b52d1213e96004bfcb1c61a8a6fa8ab84f3e8 # v4
        with:
          node-version: 20
          cache: 'yarn'
     
      - name: Install dependencies
        # Ubuntu 16+ does not install libgconf-2-4 by default, so we need to install it ourselves (for Cypress)
        run: |
          sudo apt-get install libgconf-2-4
          yarn --immutable

      - name: Run datagateway-dataview e2e tests
        run: yarn workspace datagateway-dataview run e2e
      - name: Upload Cypress screenshots
        if: failure()
        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4
        with:
          name: DataView-Screenshots
          path: packages/datagateway-dataview/cypress/screenshots

  download-e2e-tests:
    name: DataGateway Download End to End Tests
    runs-on: ubuntu-20.04
    steps:
      - name: Checkout repo
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
      - name: Add apt repo
        run: sudo add-apt-repository universe
      - name: Setup Java
        uses: actions/setup-java@387ac29b308b003ca37ba93a6cab5eb57c8f5f93 # v4
        with:
          distribution: 'zulu'
          java-version: 8
          java-package: jdk
      - name: Setup Python
        uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5
        with:
          python-version: 3.6
          architecture: x64

      # ICAT Ansible clone and install dependencies
      - name: Checkout icat-ansible
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
        with:
          repository: icatproject-contrib/icat-ansible
          ref: master
          path: icat-ansible
      - name: Install Ansible
        run: pip install -r icat-ansible/requirements.txt

      # Prep for running the playbook
      - name: Create hosts file
        run: echo -e "[icat_test_hosts]\nlocalhost ansible_connection=local" > icat-ansible/hosts
      - name: Prepare vault pass
        run: echo -e "icattravispw" > icat-ansible/vault_pass.txt
      - name: Move vault to directory it'll get detected by Ansible
        run: mv icat-ansible/vault.yml icat-ansible/group_vars/all
      - name: Replace default payara user with Actions user
        run: |
          sed -i -e "s/^payara_user: \"glassfish\"/payara_user: \"runner\"/" icat-ansible/group_vars/all/vars.yml
      - name: Amending roles
        run: |
          sed -i 's/role: authn_uows_isis/role: authn_anon/' icat-ansible/icat_test_hosts.yml

      # Force hostname to localhost - bug fix for previous ICAT Ansible issues on Actions
      - name: Change hostname to localhost
        run: sudo hostname -b localhost

      # Remove existing MySQL installation so it doesn't interfere with GitHub Actions
      - name: Remove existing mysql
        run: |
          sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
          sudo apt-get remove --purge "mysql*"
          sudo rm -rf /var/lib/mysql* /etc/mysql

      # Create local instance of ICAT
      - name: Run ICAT Ansible Playbook
        run: |
          ansible-playbook icat-ansible/icat_test_hosts.yml -i icat-ansible/hosts --vault-password-file icat-ansible/vault_pass.txt -vv

      # Fixes on ICAT components needed for e2e tests
      - name: Removing authenticator prefix for simple auth
        run: |
          sed -i 's/mechanism = simple/!mechanism = simple/' /home/runner/install/authn.simple/run.properties
      - name: Adding Chris481 user
        run: |
          sed -i '/user\.list/ s/$/ Chris481/' /home/runner/install/authn.simple/run.properties
      - name: Adding Chris481 user password
        run: |
          echo "user.Chris481.password = pw" >> /home/runner/install/authn.simple/run.properties
      - name: Reinstall authn.simple
        run: |
          cd /home/runner/install/authn.simple/ && ./setup -vv install
      - name: Add anon, root (simple without prefix) and Chris481 users to rootUserNames
        run: |
          awk -F" =" '/rootUserNames/{$2="= root Chris481 anon/anon";print;next}1' /home/runner/install/icat.server/run.properties > /home/runner/install/icat.server/run.properties.tmp
      - name: Apply rootUserNames change
        run: |
          mv -f /home/runner/install/icat.server/run.properties.tmp /home/runner/install/icat.server/run.properties
      - name: Reinstall ICAT Server
        run: |
          cd /home/runner/install/icat.server/ && ./setup -vv install
      - name: Disable Two Level Storage
        run: |
          sed -i '/plugin.archive.class/,/tidyBlockSize/ s/^/#/' /home/runner/install/ids.server/run.properties
      - name: Reinstall IDS Server
        run: |
          cd /home/runner/install/ids.server/ && python2 ./setup -vv install
      - name: Add root (simple without prefix) to datagateway-download-api adminUserNames
        run: |
          awk -F" =" '/adminUserNames/{$2="= root";print;next}1' /home/runner/install/datagateway-download-api/run.properties > /home/runner/install/datagateway-download-api/run.properties.tmp
      - name: Apply adminUserNames change
        run: |
          mv -f /home/runner/install/datagateway-download-api/run.properties.tmp /home/runner/install/datagateway-download-api/run.properties
      - name: Reinstall datagateway-download-api
        run: |
          cd /home/runner/install/datagateway-download-api/ && python2 ./setup -vv install

      # Disable Globus for Download e2e tests
      - name: Login to ICAT
        run: |
          curl -k --request POST 'https://localhost:8181/icat/session' --header 'Content-Type: application/x-www-form-urlencoded' --data-urlencode 'json={"plugin":"simple", "credentials": [{"username":"root"}, {"password":"pw"}]}' > login_output
      - name: Disable Globus for testing
        run: |
          curl -k --request PUT 'https://localhost:8181/topcat/admin/downloadType/globus/status' --header 'Content-Type: application/x-www-form-urlencoded' --data-urlencode 'sessionId='`jq -r '.sessionId' login_output` --data-urlencode 'facilityName=LILS' --data-urlencode 'disabled=True' --data-urlencode 'message="GLOBUS has been disabled for testing"'
      - name: Remove session ID data
        run: rm -f login_output

      - name: Checkout datagateway-api
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
        with:
          repository: ral-facilities/datagateway-api
          path: datagateway-api
          ref: v9.0.3

      # DataGateway API file setup
      - name: Create search_api_mapping.json
        run: cp datagateway-api/datagateway_api/search_api_mapping.json.example datagateway-api/datagateway_api/search_api_mapping.json
      - name: Create log file
        run: touch datagateway-api/datagateway_api/logs.log

      - name: Create config.yaml
        run: cp datagateway-api/datagateway_api/config.yaml.example datagateway-api/datagateway_api/config.yaml

      # DataGateway API dependencies
      - name: Install Poetry
        run: pip install poetry
      - name: Install dependencies
        run: cd datagateway-api/; poetry install
      - name: Add dummy data to icatdb
        run: |
          cd datagateway-api/; poetry run python -m util.icat_db_generator

      - name: Start API
        run: cd datagateway-api/; nohup poetry run python -m datagateway_api.src.main > api-output.txt &

      # DOI minter setup
      - name: Adding 'User-defined' DataPublicationType (needed for DOI minting api)
        run: cd datagateway-api/; poetry run python ../.github/add_doi_datapublicationtype.py

      - name: 'Add password to env file'
        run: echo DATACITE_PASSWORD=${{ secrets.DATACITE_PASSWORD }} >> ./.github/config.env

      - name: Run minting api
        run: docker run --env-file ./.github/config.env -p 8000:8000 --add-host host.docker.internal:host-gateway -d harbor.stfc.ac.uk/icat/doi-mint-api:v0.3.0      

      # E2E tests
      - name: Setup Node.js
        uses: actions/setup-node@b39b52d1213e96004bfcb1c61a8a6fa8ab84f3e8 # v4
        with:
          node-version: 20
          cache: 'yarn'
      
      - name: Install dependencies
        # Ubuntu 16+ does not install libgconf-2-4 by default, so we need to install it ourselves (for Cypress)
        run: |
          sudo apt-get install libgconf-2-4
          yarn --immutable

      - name: Run datagateway-download e2e tests
        run: yarn workspace datagateway-download run e2e
      - name: Upload Cypress screenshots
        if: failure()
        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4
        with:
          name: Download-Screenshots
          path: packages/datagateway-download/cypress/screenshots

  search-e2e-tests:
    name: DataGateway Search End to End Tests
    runs-on: ubuntu-20.04
    steps:
      - name: Checkout repo
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
      - name: Add apt repo
        run: sudo add-apt-repository universe
      - name: Setup Java
        uses: actions/setup-java@387ac29b308b003ca37ba93a6cab5eb57c8f5f93 # v4
        with:
          distribution: 'zulu'
          java-version: 8
          java-package: jdk
      - name: Setup Python
        uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5
        with:
          python-version: 3.6
          architecture: x64

      # ICAT Ansible clone and install dependencies
      - name: Checkout icat-ansible
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
        with:
          repository: icatproject-contrib/icat-ansible
          ref: master
          path: icat-ansible
      - name: Install Ansible
        run: pip install -r icat-ansible/requirements.txt

      # Prep for running the playbook
      - name: Create hosts file
        run: echo -e "[icat_test_hosts]\nlocalhost ansible_connection=local" > icat-ansible/hosts
      - name: Prepare vault pass
        run: echo -e "icattravispw" > icat-ansible/vault_pass.txt
      - name: Move vault to directory it'll get detected by Ansible
        run: mv icat-ansible/vault.yml icat-ansible/group_vars/all
      - name: Replace default payara user with Actions user
        run: |
          sed -i -e "s/^payara_user: \"glassfish\"/payara_user: \"runner\"/" icat-ansible/group_vars/all/vars.yml
      - name: Change icat.server version 
        run: |
          echo "icat_server_version: '6.1.0-SNAPSHOT'" >> icat-ansible/group_vars/all/vars.yml
      - name: Change icat.lucene version 
        run: |
          echo "icat_lucene_version: '3.0.0-SNAPSHOT'" >> icat-ansible/group_vars/all/vars.yml
      - name: Amending roles
        run: |
          sed -i 's/role: authn_uows_isis/role: authn_anon/' icat-ansible/icat_test_hosts.yml

      # Force hostname to localhost - bug fix for previous ICAT Ansible issues on Actions
      - name: Change hostname to localhost
        run: sudo hostname -b localhost

      # Remove existing MySQL installation so it doesn't interfere with GitHub Actions
      - name: Remove existing mysql
        run: |
          sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
          sudo apt-get remove --purge "mysql*"
          sudo rm -rf /var/lib/mysql* /etc/mysql

      # Create local instance of ICAT
      - name: Run ICAT Ansible Playbook
        run: |
          ansible-playbook icat-ansible/icat_test_hosts.yml -i icat-ansible/hosts --vault-password-file icat-ansible/vault_pass.txt -vv
      - name: Add anon user to rootUserNames
        run: |
          awk -F" =" '/rootUserNames/{$2="= simple/root anon/anon";print;next}1' /home/runner/install/icat.server/run.properties > /home/runner/install/icat.server/run.properties.tmp
      - name: Apply rootUserNames change
        run: |
          mv -f /home/runner/install/icat.server/run.properties.tmp /home/runner/install/icat.server/run.properties
      - name: Reinstall ICAT Server
        run: |
          cd /home/runner/install/icat.server/ && ./setup -vv install

      - name: Checkout datagateway-api
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
        with:
          repository: ral-facilities/datagateway-api
          path: datagateway-api
          ref: v9.0.3

      # DataGateway API file setup
      - name: Create search_api_mapping.json
        run: cp datagateway-api/datagateway_api/search_api_mapping.json.example datagateway-api/datagateway_api/search_api_mapping.json
      - name: Create log file
        run: touch datagateway-api/datagateway_api/logs.log

      - name: Create config.yaml
        run: cp datagateway-api/datagateway_api/config.yaml.example datagateway-api/datagateway_api/config.yaml

      # DataGateway API dependencies
      - name: Install Poetry
        run: pip install poetry
      - name: Install dependencies
        run: cd datagateway-api/; poetry install
      - name: Add dummy data to icatdb
        run: |
          cd datagateway-api/; poetry run python -m util.icat_db_generator
      - name: Add ICAT rules & public steps
        run: cd datagateway-api/; poetry run python ../.github/add_icat_rules.py

      # Recreate Lucene indexes
      - name: Use icatadmin to reindex all indexes
        run: ~/bin/icatadmin https://localhost:8181 simple username root password pw -- populate --delete
      - name: Let reindexes complete
        run: sleep 30

      - name: Start API
        run: cd datagateway-api/; nohup poetry run python -m datagateway_api.src.main > api-output.txt &

      # E2E tests
      - name: Setup Node.js
        uses: actions/setup-node@b39b52d1213e96004bfcb1c61a8a6fa8ab84f3e8 # v4
        with:
          node-version: 20
          cache: 'yarn'
     
      - name: Install dependencies
        # Ubuntu 16+ does not install libgconf-2-4 by default, so we need to install it ourselves (for Cypress)
        run: |
          sudo apt-get install libgconf-2-4
          yarn --immutable

      - name: Run datagateway-search e2e tests
        run: yarn workspace datagateway-search run e2e
      - name: Upload Cypress screenshots
        if: failure()
        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4
        with:
          name: Search-Screenshots
          path: packages/datagateway-search/cypress/screenshots

  docker:
    # This job triggers only if all the other jobs succeed. It builds the Docker image and if successful,
    # it pushes it to Harbor.
    needs: [lint-and-unit-test, dataview-e2e-tests, download-e2e-tests, search-e2e-tests]
    name: Docker
    runs-on: ubuntu-20.04
    steps:
      - name: Checkout repo
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4

      - name: Login to Harbor
        uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # v2.2.0
        with:
          registry: ${{ secrets.HARBOR_URL }}
          username: ${{ secrets.HARBOR_USERNAME }}
          password: ${{ secrets.HARBOR_TOKEN }}

      - name: Extract metadata (tags, labels) for Docker
        id: meta
        uses: docker/metadata-action@818d4b7b91585d195f67373fd9cb0332e31a7175 # v4.6.0
        with:
          images: ${{ secrets.HARBOR_URL }}/plugins

      - name: Build and push Docker image to Harbor
        uses: docker/build-push-action@0a97817b6ade9f46837855d676c4cca3a2471fc9 # v4.2.1
        with:
          context: .
          push: true
          tags: ${{ steps.meta.outputs.tags }}
          labels: ${{ steps.meta.outputs.labels }}