Skip to content

Update actions/upload-artifact digest to b4b15b8 #4197

Update actions/upload-artifact digest to b4b15b8

Update actions/upload-artifact digest to b4b15b8 #4197

Workflow file for this run

name: CI Build
on:
workflow_dispatch:
pull_request:
# By default, the pull_request event type is not triggered when a PR is merged into main or develop
push:
branches:
- main
- develop
jobs:
lint-and-unit-test:
name: Lint & Unit Tests
runs-on: ubuntu-20.04
steps:
- name: Checkout repo
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
- name: Setup Node.js
uses: actions/setup-node@b39b52d1213e96004bfcb1c61a8a6fa8ab84f3e8 # v4
with:
node-version: 20
cache: 'yarn'
- name: Install dependencies
# Ubuntu 16+ does not install libgconf-2-4 by default, so we need to install it ourselves (for Cypress)
run: |
sudo apt-get install libgconf-2-4
yarn --immutable
# Linting and unit testing
- name: Run linting
run: yarn lint
- name: Run unit tests
run: yarn test:unit
# Test coverage upload
- name: Upload unit test coverage for the Common package
if: success()
uses: codecov/codecov-action@e0b68c6749509c5f83f984dd99a76a1c1a231044 # v4
with:
directory: ./packages/datagateway-common/
flags: common
fail_ci_if_error: true
token: ${{ secrets.CODECOV_TOKEN }}
- name: Upload unit test coverage for the DataView package
if: success()
uses: codecov/codecov-action@e0b68c6749509c5f83f984dd99a76a1c1a231044 # v4
with:
directory: ./packages/datagateway-dataview/
flags: dataview
fail_ci_if_error: true
token: ${{ secrets.CODECOV_TOKEN }}
- name: Upload unit test coverage for the Search package
if: success()
uses: codecov/codecov-action@e0b68c6749509c5f83f984dd99a76a1c1a231044 # v4
with:
directory: ./packages/datagateway-search/
flags: search
fail_ci_if_error: true
token: ${{ secrets.CODECOV_TOKEN }}
- name: Upload unit test coverage for the Download package
if: success()
uses: codecov/codecov-action@e0b68c6749509c5f83f984dd99a76a1c1a231044 # v4
with:
directory: ./packages/datagateway-download/
flags: download
fail_ci_if_error: true
token: ${{ secrets.CODECOV_TOKEN }}
dataview-e2e-tests:
name: DataGateway DataView End to End Tests
runs-on: ubuntu-20.04
steps:
- name: Checkout repo
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
- name: Add apt repo
run: sudo add-apt-repository universe
- name: Setup Java
uses: actions/setup-java@387ac29b308b003ca37ba93a6cab5eb57c8f5f93 # v4
with:
distribution: 'zulu'
java-version: 8
java-package: jdk
- name: Setup Python
uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5
with:
python-version: 3.6
architecture: x64
# ICAT Ansible clone and install dependencies
- name: Checkout icat-ansible
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
with:
repository: icatproject-contrib/icat-ansible
ref: master
path: icat-ansible
- name: Install Ansible
run: pip install -r icat-ansible/requirements.txt
# Prep for running the playbook
- name: Create hosts file
run: echo -e "[icat_test_hosts]\nlocalhost ansible_connection=local" > icat-ansible/hosts
- name: Prepare vault pass
run: echo -e "icattravispw" > icat-ansible/vault_pass.txt
- name: Move vault to directory it'll get detected by Ansible
run: mv icat-ansible/vault.yml icat-ansible/group_vars/all
- name: Replace default payara user with Actions user
run: |
sed -i -e "s/^payara_user: \"glassfish\"/payara_user: \"runner\"/" icat-ansible/group_vars/all/vars.yml
- name: Amending roles
run: |
sed -i 's/role: authn_uows_isis/role: authn_anon/' icat-ansible/icat_test_hosts.yml
# Force hostname to localhost - bug fix for previous ICAT Ansible issues on Actions
- name: Change hostname to localhost
run: sudo hostname -b localhost
# Remove existing MySQL installation so it doesn't interfere with GitHub Actions
- name: Remove existing mysql
run: |
sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
sudo apt-get remove --purge "mysql*"
sudo rm -rf /var/lib/mysql* /etc/mysql
# Create local instance of ICAT
- name: Run ICAT Ansible Playbook
run: |
ansible-playbook icat-ansible/icat_test_hosts.yml -i icat-ansible/hosts --vault-password-file icat-ansible/vault_pass.txt -vv
# Fixes on ICAT components needed for e2e tests
- name: Add anon user to rootUserNames
run: |
awk -F" =" '/rootUserNames/{$2="= simple/root anon/anon";print;next}1' /home/runner/install/icat.server/run.properties > /home/runner/install/icat.server/run.properties.tmp
- name: Apply rootUserNames change
run: |
mv -f /home/runner/install/icat.server/run.properties.tmp /home/runner/install/icat.server/run.properties
- name: Reinstall ICAT Server
run: |
cd /home/runner/install/icat.server/ && ./setup -vv install
- name: Checkout datagateway-api
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
with:
repository: ral-facilities/datagateway-api
path: datagateway-api
ref: v9.0.3
# DataGateway API file setup
- name: Create search_api_mapping.json
run: cp datagateway-api/datagateway_api/search_api_mapping.json.example datagateway-api/datagateway_api/search_api_mapping.json
- name: Create log file
run: touch datagateway-api/datagateway_api/logs.log
- name: Create config.yaml
run: cp datagateway-api/datagateway_api/config.yaml.example datagateway-api/datagateway_api/config.yaml
# DataGateway API dependencies
- name: Install Poetry
run: pip install poetry
- name: Install dependencies
run: cd datagateway-api/; poetry install
- name: Add dummy data to icatdb
run: |
cd datagateway-api/; poetry run python -m util.icat_db_generator
- name: Start API
run: cd datagateway-api/; nohup poetry run python -m datagateway_api.src.main > api-output.txt &
# E2E tests
- name: Setup Node.js
uses: actions/setup-node@b39b52d1213e96004bfcb1c61a8a6fa8ab84f3e8 # v4
with:
node-version: 20
cache: 'yarn'
- name: Install dependencies
# Ubuntu 16+ does not install libgconf-2-4 by default, so we need to install it ourselves (for Cypress)
run: |
sudo apt-get install libgconf-2-4
yarn --immutable
- name: Run datagateway-dataview e2e tests
run: yarn workspace datagateway-dataview run e2e
- name: Upload Cypress screenshots
if: failure()
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4
with:
name: DataView-Screenshots
path: packages/datagateway-dataview/cypress/screenshots
download-e2e-tests:
name: DataGateway Download End to End Tests
runs-on: ubuntu-20.04
steps:
- name: Checkout repo
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
- name: Add apt repo
run: sudo add-apt-repository universe
- name: Setup Java
uses: actions/setup-java@387ac29b308b003ca37ba93a6cab5eb57c8f5f93 # v4
with:
distribution: 'zulu'
java-version: 8
java-package: jdk
- name: Setup Python
uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5
with:
python-version: 3.6
architecture: x64
# ICAT Ansible clone and install dependencies
- name: Checkout icat-ansible
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
with:
repository: icatproject-contrib/icat-ansible
ref: master
path: icat-ansible
- name: Install Ansible
run: pip install -r icat-ansible/requirements.txt
# Prep for running the playbook
- name: Create hosts file
run: echo -e "[icat_test_hosts]\nlocalhost ansible_connection=local" > icat-ansible/hosts
- name: Prepare vault pass
run: echo -e "icattravispw" > icat-ansible/vault_pass.txt
- name: Move vault to directory it'll get detected by Ansible
run: mv icat-ansible/vault.yml icat-ansible/group_vars/all
- name: Replace default payara user with Actions user
run: |
sed -i -e "s/^payara_user: \"glassfish\"/payara_user: \"runner\"/" icat-ansible/group_vars/all/vars.yml
- name: Amending roles
run: |
sed -i 's/role: authn_uows_isis/role: authn_anon/' icat-ansible/icat_test_hosts.yml
# Force hostname to localhost - bug fix for previous ICAT Ansible issues on Actions
- name: Change hostname to localhost
run: sudo hostname -b localhost
# Remove existing MySQL installation so it doesn't interfere with GitHub Actions
- name: Remove existing mysql
run: |
sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
sudo apt-get remove --purge "mysql*"
sudo rm -rf /var/lib/mysql* /etc/mysql
# Create local instance of ICAT
- name: Run ICAT Ansible Playbook
run: |
ansible-playbook icat-ansible/icat_test_hosts.yml -i icat-ansible/hosts --vault-password-file icat-ansible/vault_pass.txt -vv
# Fixes on ICAT components needed for e2e tests
- name: Removing authenticator prefix for simple auth
run: |
sed -i 's/mechanism = simple/!mechanism = simple/' /home/runner/install/authn.simple/run.properties
- name: Adding Chris481 user
run: |
sed -i '/user\.list/ s/$/ Chris481/' /home/runner/install/authn.simple/run.properties
- name: Adding Chris481 user password
run: |
echo "user.Chris481.password = pw" >> /home/runner/install/authn.simple/run.properties
- name: Reinstall authn.simple
run: |
cd /home/runner/install/authn.simple/ && ./setup -vv install
- name: Add anon, root (simple without prefix) and Chris481 users to rootUserNames
run: |
awk -F" =" '/rootUserNames/{$2="= root Chris481 anon/anon";print;next}1' /home/runner/install/icat.server/run.properties > /home/runner/install/icat.server/run.properties.tmp
- name: Apply rootUserNames change
run: |
mv -f /home/runner/install/icat.server/run.properties.tmp /home/runner/install/icat.server/run.properties
- name: Reinstall ICAT Server
run: |
cd /home/runner/install/icat.server/ && ./setup -vv install
- name: Disable Two Level Storage
run: |
sed -i '/plugin.archive.class/,/tidyBlockSize/ s/^/#/' /home/runner/install/ids.server/run.properties
- name: Reinstall IDS Server
run: |
cd /home/runner/install/ids.server/ && python2 ./setup -vv install
- name: Add root (simple without prefix) to datagateway-download-api adminUserNames
run: |
awk -F" =" '/adminUserNames/{$2="= root";print;next}1' /home/runner/install/datagateway-download-api/run.properties > /home/runner/install/datagateway-download-api/run.properties.tmp
- name: Apply adminUserNames change
run: |
mv -f /home/runner/install/datagateway-download-api/run.properties.tmp /home/runner/install/datagateway-download-api/run.properties
- name: Reinstall datagateway-download-api
run: |
cd /home/runner/install/datagateway-download-api/ && python2 ./setup -vv install
# Disable Globus for Download e2e tests
- name: Login to ICAT
run: |
curl -k --request POST 'https://localhost:8181/icat/session' --header 'Content-Type: application/x-www-form-urlencoded' --data-urlencode 'json={"plugin":"simple", "credentials": [{"username":"root"}, {"password":"pw"}]}' > login_output
- name: Disable Globus for testing
run: |
curl -k --request PUT 'https://localhost:8181/topcat/admin/downloadType/globus/status' --header 'Content-Type: application/x-www-form-urlencoded' --data-urlencode 'sessionId='`jq -r '.sessionId' login_output` --data-urlencode 'facilityName=LILS' --data-urlencode 'disabled=True' --data-urlencode 'message="GLOBUS has been disabled for testing"'
- name: Remove session ID data
run: rm -f login_output
- name: Checkout datagateway-api
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
with:
repository: ral-facilities/datagateway-api
path: datagateway-api
ref: v9.0.3
# DataGateway API file setup
- name: Create search_api_mapping.json
run: cp datagateway-api/datagateway_api/search_api_mapping.json.example datagateway-api/datagateway_api/search_api_mapping.json
- name: Create log file
run: touch datagateway-api/datagateway_api/logs.log
- name: Create config.yaml
run: cp datagateway-api/datagateway_api/config.yaml.example datagateway-api/datagateway_api/config.yaml
# DataGateway API dependencies
- name: Install Poetry
run: pip install poetry
- name: Install dependencies
run: cd datagateway-api/; poetry install
- name: Add dummy data to icatdb
run: |
cd datagateway-api/; poetry run python -m util.icat_db_generator
- name: Start API
run: cd datagateway-api/; nohup poetry run python -m datagateway_api.src.main > api-output.txt &
# DOI minter setup
- name: Adding 'User-defined' DataPublicationType (needed for DOI minting api)
run: cd datagateway-api/; poetry run python ../.github/add_doi_datapublicationtype.py
- name: 'Add password to env file'
run: echo DATACITE_PASSWORD=${{ secrets.DATACITE_PASSWORD }} >> ./.github/config.env
- name: Run minting api
run: docker run --env-file ./.github/config.env -p 8000:8000 --add-host host.docker.internal:host-gateway -d harbor.stfc.ac.uk/icat/doi-mint-api:v0.3.0
# E2E tests
- name: Setup Node.js
uses: actions/setup-node@b39b52d1213e96004bfcb1c61a8a6fa8ab84f3e8 # v4
with:
node-version: 20
cache: 'yarn'
- name: Install dependencies
# Ubuntu 16+ does not install libgconf-2-4 by default, so we need to install it ourselves (for Cypress)
run: |
sudo apt-get install libgconf-2-4
yarn --immutable
- name: Run datagateway-download e2e tests
run: yarn workspace datagateway-download run e2e
- name: Upload Cypress screenshots
if: failure()
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4
with:
name: Download-Screenshots
path: packages/datagateway-download/cypress/screenshots
search-e2e-tests:
name: DataGateway Search End to End Tests
runs-on: ubuntu-20.04
steps:
- name: Checkout repo
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
- name: Add apt repo
run: sudo add-apt-repository universe
- name: Setup Java
uses: actions/setup-java@387ac29b308b003ca37ba93a6cab5eb57c8f5f93 # v4
with:
distribution: 'zulu'
java-version: 8
java-package: jdk
- name: Setup Python
uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5
with:
python-version: 3.6
architecture: x64
# ICAT Ansible clone and install dependencies
- name: Checkout icat-ansible
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
with:
repository: icatproject-contrib/icat-ansible
ref: master
path: icat-ansible
- name: Install Ansible
run: pip install -r icat-ansible/requirements.txt
# Prep for running the playbook
- name: Create hosts file
run: echo -e "[icat_test_hosts]\nlocalhost ansible_connection=local" > icat-ansible/hosts
- name: Prepare vault pass
run: echo -e "icattravispw" > icat-ansible/vault_pass.txt
- name: Move vault to directory it'll get detected by Ansible
run: mv icat-ansible/vault.yml icat-ansible/group_vars/all
- name: Replace default payara user with Actions user
run: |
sed -i -e "s/^payara_user: \"glassfish\"/payara_user: \"runner\"/" icat-ansible/group_vars/all/vars.yml
- name: Change icat.server version
run: |
echo "icat_server_version: '6.1.0-SNAPSHOT'" >> icat-ansible/group_vars/all/vars.yml
- name: Change icat.lucene version
run: |
echo "icat_lucene_version: '3.0.0-SNAPSHOT'" >> icat-ansible/group_vars/all/vars.yml
- name: Amending roles
run: |
sed -i 's/role: authn_uows_isis/role: authn_anon/' icat-ansible/icat_test_hosts.yml
# Force hostname to localhost - bug fix for previous ICAT Ansible issues on Actions
- name: Change hostname to localhost
run: sudo hostname -b localhost
# Remove existing MySQL installation so it doesn't interfere with GitHub Actions
- name: Remove existing mysql
run: |
sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
sudo apt-get remove --purge "mysql*"
sudo rm -rf /var/lib/mysql* /etc/mysql
# Create local instance of ICAT
- name: Run ICAT Ansible Playbook
run: |
ansible-playbook icat-ansible/icat_test_hosts.yml -i icat-ansible/hosts --vault-password-file icat-ansible/vault_pass.txt -vv
- name: Add anon user to rootUserNames
run: |
awk -F" =" '/rootUserNames/{$2="= simple/root anon/anon";print;next}1' /home/runner/install/icat.server/run.properties > /home/runner/install/icat.server/run.properties.tmp
- name: Apply rootUserNames change
run: |
mv -f /home/runner/install/icat.server/run.properties.tmp /home/runner/install/icat.server/run.properties
- name: Reinstall ICAT Server
run: |
cd /home/runner/install/icat.server/ && ./setup -vv install
- name: Checkout datagateway-api
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
with:
repository: ral-facilities/datagateway-api
path: datagateway-api
ref: v9.0.3
# DataGateway API file setup
- name: Create search_api_mapping.json
run: cp datagateway-api/datagateway_api/search_api_mapping.json.example datagateway-api/datagateway_api/search_api_mapping.json
- name: Create log file
run: touch datagateway-api/datagateway_api/logs.log
- name: Create config.yaml
run: cp datagateway-api/datagateway_api/config.yaml.example datagateway-api/datagateway_api/config.yaml
# DataGateway API dependencies
- name: Install Poetry
run: pip install poetry
- name: Install dependencies
run: cd datagateway-api/; poetry install
- name: Add dummy data to icatdb
run: |
cd datagateway-api/; poetry run python -m util.icat_db_generator
- name: Add ICAT rules & public steps
run: cd datagateway-api/; poetry run python ../.github/add_icat_rules.py
# Recreate Lucene indexes
- name: Use icatadmin to reindex all indexes
run: ~/bin/icatadmin https://localhost:8181 simple username root password pw -- populate --delete
- name: Let reindexes complete
run: sleep 30
- name: Start API
run: cd datagateway-api/; nohup poetry run python -m datagateway_api.src.main > api-output.txt &
# E2E tests
- name: Setup Node.js
uses: actions/setup-node@b39b52d1213e96004bfcb1c61a8a6fa8ab84f3e8 # v4
with:
node-version: 20
cache: 'yarn'
- name: Install dependencies
# Ubuntu 16+ does not install libgconf-2-4 by default, so we need to install it ourselves (for Cypress)
run: |
sudo apt-get install libgconf-2-4
yarn --immutable
- name: Run datagateway-search e2e tests
run: yarn workspace datagateway-search run e2e
- name: Upload Cypress screenshots
if: failure()
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4
with:
name: Search-Screenshots
path: packages/datagateway-search/cypress/screenshots
docker:
# This job triggers only if all the other jobs succeed. It builds the Docker image and if successful,
# it pushes it to Harbor.
needs: [lint-and-unit-test, dataview-e2e-tests, download-e2e-tests, search-e2e-tests]
name: Docker
runs-on: ubuntu-20.04
steps:
- name: Checkout repo
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
- name: Login to Harbor
uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # v2.2.0
with:
registry: ${{ secrets.HARBOR_URL }}
username: ${{ secrets.HARBOR_USERNAME }}
password: ${{ secrets.HARBOR_TOKEN }}
- name: Extract metadata (tags, labels) for Docker
id: meta
uses: docker/metadata-action@818d4b7b91585d195f67373fd9cb0332e31a7175 # v4.6.0
with:
images: ${{ secrets.HARBOR_URL }}/plugins
- name: Build and push Docker image to Harbor
uses: docker/build-push-action@0a97817b6ade9f46837855d676c4cca3a2471fc9 # v4.2.1
with:
context: .
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}