Implement sematic/lexcial search feature in datagateway search #3477
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI Build | |
| on: | |
| workflow_dispatch: | |
| pull_request: | |
| # By default, the pull_request event type is not triggered when a PR is merged into main | |
| push: | |
| branches: | |
| - main | |
| - develop | |
| jobs: | |
| lint-and-unit-test: | |
| name: Lint & Unit Tests | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3 | |
| - name: Setup Node.js | |
| uses: actions/setup-node@e33196f7422957bea03ed53f6fbb155025ffc7b8 # v3 | |
| with: | |
| node-version: '16.x' | |
| cache: 'yarn' | |
| - name: Install dependencies | |
| # Ubuntu 16+ does not install libgconf-2-4 by default, so we need to install it ourselves (for Cypress) | |
| run: | | |
| npm config set scripts-prepend-node-path true | |
| sudo apt-get install libgconf-2-4 | |
| yarn --immutable | |
| # Linting and unit testing | |
| - name: Run linting | |
| run: yarn lint | |
| - name: Run unit tests | |
| run: yarn test:unit | |
| # Test coverage upload | |
| - name: Upload unit test coverage for the Common package | |
| if: success() | |
| uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d # v3 | |
| with: | |
| directory: ./packages/datagateway-common/ | |
| flags: common | |
| fail_ci_if_error: true | |
| - name: Upload unit test coverage for the DataView package | |
| if: success() | |
| uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d # v3 | |
| with: | |
| directory: ./packages/datagateway-dataview/ | |
| flags: dataview | |
| fail_ci_if_error: true | |
| - name: Upload unit test coverage for the Search package | |
| if: success() | |
| uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d # v3 | |
| with: | |
| directory: ./packages/datagateway-search/ | |
| flags: search | |
| fail_ci_if_error: true | |
| - name: Upload unit test coverage for the Download package | |
| if: success() | |
| uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d # v3 | |
| with: | |
| directory: ./packages/datagateway-download/ | |
| flags: download | |
| fail_ci_if_error: true | |
| dataview-e2e-tests: | |
| name: DataGateway DataView End to End Tests | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3 | |
| - name: Add apt repo | |
| run: sudo add-apt-repository universe | |
| - name: Setup Java | |
| uses: actions/setup-java@5ffc13f4174014e2d4d4572b3d74c3fa61aeb2c2 # v3 | |
| with: | |
| distribution: 'zulu' | |
| java-version: 8 | |
| java-package: jdk | |
| - name: Setup Python | |
| uses: actions/setup-python@bd6b4b6205c4dbad673328db7b31b7fab9e241c0 # v4 | |
| with: | |
| python-version: 3.6 | |
| architecture: x64 | |
| # ICAT Ansible clone and install dependencies | |
| - name: Checkout icat-ansible | |
| uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3 | |
| with: | |
| repository: icatproject-contrib/icat-ansible | |
| ref: master | |
| path: icat-ansible | |
| - name: Install Ansible | |
| run: pip install -r icat-ansible/requirements.txt | |
| # Prep for running the playbook | |
| - name: Create hosts file | |
| run: echo -e "[icat_test_hosts]\nlocalhost ansible_connection=local" > icat-ansible/hosts | |
| - name: Prepare vault pass | |
| run: echo -e "icattravispw" > icat-ansible/vault_pass.txt | |
| - name: Move vault to directory it'll get detected by Ansible | |
| run: mv icat-ansible/vault.yml icat-ansible/group_vars/all | |
| - name: Replace default payara user with Actions user | |
| run: | | |
| sed -i -e "s/^payara_user: \"glassfish\"/payara_user: \"runner\"/" icat-ansible/group_vars/all/vars.yml | |
| - name: Amending roles | |
| run: | | |
| sed -i 's/role: authn_uows_isis/role: authn_anon/' icat-ansible/icat_test_hosts.yml | |
| # Force hostname to localhost - bug fix for previous ICAT Ansible issues on Actions | |
| - name: Change hostname to localhost | |
| run: sudo hostname -b localhost | |
| # Remove existing MySQL installation so it doesn't interfere with GitHub Actions | |
| - name: Remove existing mysql | |
| run: | | |
| sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld | |
| sudo apt-get remove --purge "mysql*" | |
| sudo rm -rf /var/lib/mysql* /etc/mysql | |
| # Create local instance of ICAT | |
| - name: Run ICAT Ansible Playbook | |
| run: | | |
| ansible-playbook icat-ansible/icat_test_hosts.yml -i icat-ansible/hosts --vault-password-file icat-ansible/vault_pass.txt -vv | |
| # Fixes on ICAT components needed for e2e tests | |
| - name: Add anon user to rootUserNames | |
| run: | | |
| awk -F" =" '/rootUserNames/{$2="= simple/root anon/anon";print;next}1' /home/runner/install/icat.server/run.properties > /home/runner/install/icat.server/run.properties.tmp | |
| - name: Apply rootUserNames change | |
| run: | | |
| mv -f /home/runner/install/icat.server/run.properties.tmp /home/runner/install/icat.server/run.properties | |
| - name: Reinstall ICAT Server | |
| run: | | |
| cd /home/runner/install/icat.server/ && ./setup -vv install | |
| - name: Checkout datagateway-api | |
| uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3 | |
| with: | |
| repository: ral-facilities/datagateway-api | |
| path: datagateway-api | |
| ref: v7.1.0 | |
| # DataGateway API file setup | |
| - name: Create search_api_mapping.json | |
| run: cp datagateway-api/datagateway_api/search_api_mapping.json.example datagateway-api/datagateway_api/search_api_mapping.json | |
| - name: Create log file | |
| run: touch datagateway-api/datagateway_api/logs.log | |
| - name: Create config.yaml | |
| run: cp datagateway-api/datagateway_api/config.yaml.example datagateway-api/datagateway_api/config.yaml | |
| # DataGateway API dependencies | |
| - name: Install Poetry | |
| run: pip install poetry | |
| - name: Install dependencies | |
| run: cd datagateway-api/; poetry install | |
| - name: Add dummy data to icatdb | |
| run: | | |
| cd datagateway-api/; poetry run python -m util.icat_db_generator | |
| - name: Start API | |
| run: cd datagateway-api/; nohup poetry run python -m datagateway_api.src.main > api-output.txt & | |
| # E2E tests | |
| - name: Setup Node.js | |
| uses: actions/setup-node@e33196f7422957bea03ed53f6fbb155025ffc7b8 # v3 | |
| with: | |
| node-version: '14.x' | |
| # Cache yarn dependencies/ restore the cached dependencies during future workflows | |
| - name: Get yarn cache directory path | |
| id: yarn-cache-dir-path | |
| run: echo "::set-output name=dir::$(yarn cache dir)" | |
| - name: Cache yarn dependencies | |
| uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3 | |
| id: yarn-cache | |
| with: | |
| path: ${{ steps.yarn-cache-dir-path.outputs.dir }} | |
| key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-yarn- | |
| - name: Install dependencies | |
| # Ubuntu 16+ does not install libgconf-2-4 by default, so we need to install it ourselves (for Cypress) | |
| run: | | |
| npm config set scripts-prepend-node-path true | |
| sudo apt-get install libgconf-2-4 | |
| yarn --frozen-lockfile | |
| - name: Run datagateway-dataview e2e tests | |
| run: yarn workspace datagateway-dataview run e2e | |
| - name: Upload Cypress screenshots | |
| if: failure() | |
| uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3 | |
| with: | |
| name: DataView-Screenshots | |
| path: packages/datagateway-dataview/cypress/screenshots | |
| download-e2e-tests: | |
| name: DataGateway Download End to End Tests | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3 | |
| - name: Add apt repo | |
| run: sudo add-apt-repository universe | |
| - name: Setup Java | |
| uses: actions/setup-java@5ffc13f4174014e2d4d4572b3d74c3fa61aeb2c2 # v3 | |
| with: | |
| distribution: 'zulu' | |
| java-version: 8 | |
| java-package: jdk | |
| - name: Setup Python | |
| uses: actions/setup-python@bd6b4b6205c4dbad673328db7b31b7fab9e241c0 # v4 | |
| with: | |
| python-version: 3.6 | |
| architecture: x64 | |
| # ICAT Ansible clone and install dependencies | |
| - name: Checkout icat-ansible | |
| uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3 | |
| with: | |
| repository: icatproject-contrib/icat-ansible | |
| ref: master | |
| path: icat-ansible | |
| - name: Install Ansible | |
| run: pip install -r icat-ansible/requirements.txt | |
| # Prep for running the playbook | |
| - name: Create hosts file | |
| run: echo -e "[icat_test_hosts]\nlocalhost ansible_connection=local" > icat-ansible/hosts | |
| - name: Prepare vault pass | |
| run: echo -e "icattravispw" > icat-ansible/vault_pass.txt | |
| - name: Move vault to directory it'll get detected by Ansible | |
| run: mv icat-ansible/vault.yml icat-ansible/group_vars/all | |
| - name: Replace default payara user with Actions user | |
| run: | | |
| sed -i -e "s/^payara_user: \"glassfish\"/payara_user: \"runner\"/" icat-ansible/group_vars/all/vars.yml | |
| - name: Amending roles | |
| run: | | |
| sed -i 's/role: authn_uows_isis/role: authn_anon/' icat-ansible/icat_test_hosts.yml | |
| # Force hostname to localhost - bug fix for previous ICAT Ansible issues on Actions | |
| - name: Change hostname to localhost | |
| run: sudo hostname -b localhost | |
| # Remove existing MySQL installation so it doesn't interfere with GitHub Actions | |
| - name: Remove existing mysql | |
| run: | | |
| sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld | |
| sudo apt-get remove --purge "mysql*" | |
| sudo rm -rf /var/lib/mysql* /etc/mysql | |
| # Create local instance of ICAT | |
| - name: Run ICAT Ansible Playbook | |
| run: | | |
| ansible-playbook icat-ansible/icat_test_hosts.yml -i icat-ansible/hosts --vault-password-file icat-ansible/vault_pass.txt -vv | |
| # Fixes on ICAT components needed for e2e tests | |
| - name: Add anon user to rootUserNames | |
| run: | | |
| awk -F" =" '/rootUserNames/{$2="= simple/root anon/anon";print;next}1' /home/runner/install/icat.server/run.properties > /home/runner/install/icat.server/run.properties.tmp | |
| - name: Apply rootUserNames change | |
| run: | | |
| mv -f /home/runner/install/icat.server/run.properties.tmp /home/runner/install/icat.server/run.properties | |
| - name: Reinstall ICAT Server | |
| run: | | |
| cd /home/runner/install/icat.server/ && ./setup -vv install | |
| - name: Disable Two Level Storage | |
| run: | | |
| sed -i '/plugin.archive.class/,/tidyBlockSize/ s/^/#/' /home/runner/install/ids.server/run.properties | |
| - name: Reinstall IDS Server | |
| run: | | |
| cd /home/runner/install/ids.server/ && python2 ./setup -vv install | |
| # Disable Globus for Download e2e tests | |
| - name: Login to ICAT | |
| run: | | |
| curl -k --request POST 'https://localhost:8181/icat/session' --header 'Content-Type: application/x-www-form-urlencoded' --data-urlencode 'json={"plugin":"simple", "credentials": [{"username":"root"}, {"password":"pw"}]}' > login_output | |
| - name: Disable Globus for testing | |
| run: | | |
| curl -k --request PUT 'https://localhost:8181/topcat/admin/downloadType/globus/status' --header 'Content-Type: application/x-www-form-urlencoded' --data-urlencode 'sessionId='`jq -r '.sessionId' login_output` --data-urlencode 'facilityName=LILS' --data-urlencode 'disabled=True' --data-urlencode 'message="GLOBUS has been disabled for testing"' | |
| - name: Remove session ID data | |
| run: rm -f login_output | |
| - name: Checkout datagateway-api | |
| uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3 | |
| with: | |
| repository: ral-facilities/datagateway-api | |
| path: datagateway-api | |
| ref: v7.1.0 | |
| # DataGateway API file setup | |
| - name: Create search_api_mapping.json | |
| run: cp datagateway-api/datagateway_api/search_api_mapping.json.example datagateway-api/datagateway_api/search_api_mapping.json | |
| - name: Create log file | |
| run: touch datagateway-api/datagateway_api/logs.log | |
| - name: Create config.yaml | |
| run: cp datagateway-api/datagateway_api/config.yaml.example datagateway-api/datagateway_api/config.yaml | |
| # DataGateway API dependencies | |
| - name: Install Poetry | |
| run: pip install poetry | |
| - name: Install dependencies | |
| run: cd datagateway-api/; poetry install | |
| - name: Add dummy data to icatdb | |
| run: | | |
| cd datagateway-api/; poetry run python -m util.icat_db_generator | |
| - name: Start API | |
| run: cd datagateway-api/; nohup poetry run python -m datagateway_api.src.main > api-output.txt & | |
| # E2E tests | |
| - name: Setup Node.js | |
| uses: actions/setup-node@e33196f7422957bea03ed53f6fbb155025ffc7b8 # v3 | |
| with: | |
| node-version: '14.x' | |
| # Cache yarn dependencies/ restore the cached dependencies during future workflows | |
| - name: Get yarn cache directory path | |
| id: yarn-cache-dir-path | |
| run: echo "::set-output name=dir::$(yarn cache dir)" | |
| - name: Cache yarn dependencies | |
| uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3 | |
| id: yarn-cache | |
| with: | |
| path: ${{ steps.yarn-cache-dir-path.outputs.dir }} | |
| key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-yarn- | |
| - name: Install dependencies | |
| # Ubuntu 16+ does not install libgconf-2-4 by default, so we need to install it ourselves (for Cypress) | |
| run: | | |
| npm config set scripts-prepend-node-path true | |
| sudo apt-get install libgconf-2-4 | |
| yarn --frozen-lockfile | |
| - name: Run datagateway-download e2e tests | |
| run: yarn workspace datagateway-download run e2e | |
| - name: Upload Cypress screenshots | |
| if: failure() | |
| uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3 | |
| with: | |
| name: Download-Screenshots | |
| path: packages/datagateway-download/cypress/screenshots | |
| search-e2e-tests: | |
| name: DataGateway Search End to End Tests | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3 | |
| - name: Add apt repo | |
| run: sudo add-apt-repository universe | |
| - name: Setup Java | |
| uses: actions/setup-java@5ffc13f4174014e2d4d4572b3d74c3fa61aeb2c2 # v3 | |
| with: | |
| distribution: 'zulu' | |
| java-version: 8 | |
| java-package: jdk | |
| - name: Setup Python | |
| uses: actions/setup-python@bd6b4b6205c4dbad673328db7b31b7fab9e241c0 # v4 | |
| with: | |
| python-version: 3.6 | |
| architecture: x64 | |
| # ICAT Ansible clone and install dependencies | |
| - name: Checkout icat-ansible | |
| uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3 | |
| with: | |
| repository: icatproject-contrib/icat-ansible | |
| ref: master | |
| path: icat-ansible | |
| - name: Install Ansible | |
| run: pip install -r icat-ansible/requirements.txt | |
| # Prep for running the playbook | |
| - name: Create hosts file | |
| run: echo -e "[icat_test_hosts]\nlocalhost ansible_connection=local" > icat-ansible/hosts | |
| - name: Prepare vault pass | |
| run: echo -e "icattravispw" > icat-ansible/vault_pass.txt | |
| - name: Move vault to directory it'll get detected by Ansible | |
| run: mv icat-ansible/vault.yml icat-ansible/group_vars/all | |
| - name: Replace default payara user with Actions user | |
| run: | | |
| sed -i -e "s/^payara_user: \"glassfish\"/payara_user: \"runner\"/" icat-ansible/group_vars/all/vars.yml | |
| - name: Amending roles | |
| run: | | |
| sed -i 's/role: authn_uows_isis/role: authn_anon/' icat-ansible/icat_test_hosts.yml | |
| # Force hostname to localhost - bug fix for previous ICAT Ansible issues on Actions | |
| - name: Change hostname to localhost | |
| run: sudo hostname -b localhost | |
| # Remove existing MySQL installation so it doesn't interfere with GitHub Actions | |
| - name: Remove existing mysql | |
| run: | | |
| sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld | |
| sudo apt-get remove --purge "mysql*" | |
| sudo rm -rf /var/lib/mysql* /etc/mysql | |
| # Create local instance of ICAT | |
| - name: Run ICAT Ansible Playbook | |
| run: | | |
| ansible-playbook icat-ansible/icat_test_hosts.yml -i icat-ansible/hosts --vault-password-file icat-ansible/vault_pass.txt -vv | |
| - name: Add anon user to rootUserNames | |
| run: | | |
| awk -F" =" '/rootUserNames/{$2="= simple/root anon/anon";print;next}1' /home/runner/install/icat.server/run.properties > /home/runner/install/icat.server/run.properties.tmp | |
| - name: Apply rootUserNames change | |
| run: | | |
| mv -f /home/runner/install/icat.server/run.properties.tmp /home/runner/install/icat.server/run.properties | |
| - name: Reinstall ICAT Server | |
| run: | | |
| cd /home/runner/install/icat.server/ && ./setup -vv install | |
| - name: Checkout datagateway-api | |
| uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3 | |
| with: | |
| repository: ral-facilities/datagateway-api | |
| path: datagateway-api | |
| ref: v7.1.0 | |
| # DataGateway API file setup | |
| - name: Create search_api_mapping.json | |
| run: cp datagateway-api/datagateway_api/search_api_mapping.json.example datagateway-api/datagateway_api/search_api_mapping.json | |
| - name: Create log file | |
| run: touch datagateway-api/datagateway_api/logs.log | |
| - name: Create config.yaml | |
| run: cp datagateway-api/datagateway_api/config.yaml.example datagateway-api/datagateway_api/config.yaml | |
| # DataGateway API dependencies | |
| - name: Install Poetry | |
| run: pip install poetry | |
| - name: Install dependencies | |
| run: cd datagateway-api/; poetry install | |
| - name: Add dummy data to icatdb | |
| run: | | |
| cd datagateway-api/; poetry run python -m util.icat_db_generator | |
| # Recreate Lucene indexes | |
| - name: Login to ICAT | |
| run: | | |
| curl -k --request POST 'https://localhost:8181/icat/session' --header 'Content-Type: application/x-www-form-urlencoded' --data-urlencode 'json={"plugin":"simple", "credentials": [{"username":"root"}, {"password":"pw"}]}' > login_output | |
| - name: Recreate Datafile index | |
| run: | | |
| curl -k --request POST 'https://localhost:8181/icat/lucene/db/Datafile/0' --header 'Content-Type: application/x-www-form-urlencoded' --data-urlencode 'sessionId='`jq -r '.sessionId' login_output` | |
| - name: Recreate Dataset index | |
| run: | | |
| curl -k --request POST 'https://localhost:8181/icat/lucene/db/Dataset/1' --header 'Content-Type: application/x-www-form-urlencoded' --data-urlencode 'sessionId='`jq -r '.sessionId' login_output` | |
| - name: Recreate Investigation index | |
| run: | | |
| curl -k --request POST 'https://localhost:8181/icat/lucene/db/Investigation/1' --header 'Content-Type: application/x-www-form-urlencoded' --data-urlencode 'sessionId='`jq -r '.sessionId' login_output` | |
| - name: Let reindexes complete | |
| run: sleep 30 | |
| - name: Remove session ID data | |
| run: rm -f login_output | |
| - name: Start API | |
| run: cd datagateway-api/; nohup poetry run python -m datagateway_api.src.main > api-output.txt & | |
| # E2E tests | |
| - name: Setup Node.js | |
| uses: actions/setup-node@e33196f7422957bea03ed53f6fbb155025ffc7b8 # v3 | |
| with: | |
| node-version: '14.x' | |
| # Cache yarn dependencies/ restore the cached dependencies during future workflows | |
| - name: Get yarn cache directory path | |
| id: yarn-cache-dir-path | |
| run: echo "::set-output name=dir::$(yarn cache dir)" | |
| - name: Cache yarn dependencies | |
| uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3 | |
| id: yarn-cache | |
| with: | |
| path: ${{ steps.yarn-cache-dir-path.outputs.dir }} | |
| key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-yarn- | |
| - name: Install dependencies | |
| # Ubuntu 16+ does not install libgconf-2-4 by default, so we need to install it ourselves (for Cypress) | |
| run: | | |
| npm config set scripts-prepend-node-path true | |
| sudo apt-get install libgconf-2-4 | |
| yarn --frozen-lockfile | |
| - name: Run datagateway-search e2e tests | |
| run: yarn workspace datagateway-search run e2e | |
| - name: Upload Cypress screenshots | |
| if: failure() | |
| uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3 | |
| with: | |
| name: Search-Screenshots | |
| path: packages/datagateway-search/cypress/screenshots |