From 49aca6c83eb570ea8975d8ccf3bc3f63112c1f7e Mon Sep 17 00:00:00 2001 From: Vincent Emonet Date: Sun, 26 Mar 2023 15:40:56 +0200 Subject: [PATCH 1/5] Now uses docker to deploy the bioregistry, a virtuoso triplestore, and a blazegraph triplestore to test federating querying to the bioregistry SPARQL mapping service. Added a simpler SPARQL query to test federated querying, since the original query using VALUES fails on Virtuoso The docker stack is now started in the tests github action A section to describe how to run the tests has been added at the bottom of the README, you might want to move it to a more consistent place --- .github/workflows/tests.yml | 8 +++++ README.md | 50 +++++++++++++++++++++++++++++ docker-compose.yml | 19 +++++++++++ tests/test_sparql.py | 63 ++++++++++++++++++++++++++----------- 4 files changed, 122 insertions(+), 18 deletions(-) create mode 100644 docker-compose.yml diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 737914bd2..c0f9020bf 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,6 +1,7 @@ name: Tests on: + workflow_dispatch: push: branches: [ main ] pull_request: @@ -23,6 +24,13 @@ jobs: run: | pip install --upgrade pip setuptools wheel pip install tox + + - name: Start blazegraph and virtuoso triplestores with docker + run: | + docker-compose up -d + sleep(5) + docker compose exec virtuoso isql -U dba -P dba exec='GRANT "SPARQL_SELECT_FED" TO "SPARQL";' + - name: Check manifest run: tox -e manifest - name: Check code quality with flake8 diff --git a/README.md b/README.md index a06b1eac1..47c38cfeb 100644 --- a/README.md +++ b/README.md @@ -475,3 +475,53 @@ at [Harvard Medical School](https://hms.harvard.edu/). ### 💰 Funding The development of the Bioregistry is funded by the DARPA Young Faculty Award W911NF2010255 (PI: Benjamin M. Gyori). + + +## 🛠️ For Developers + +
+ See developer instructions + + +The final section of the README is for if you want to get involved by making a code contribution. + +### 📥️ Development Installation + +To install in development mode, consider enabling a virtual environment, and use the following: + +```bash +$ git clone git+https://github.com/biopragmatics/bioregistry.git +$ cd bioregistry +$ pip install -e ".[tests,web]" +``` + +### 🥼 Testing + +To run all tests you will need to start the Bioregistry, a Virtuoso endpoint, and a Blazegraph endpoint using `docker compose`: + +```bash +$ docker compose up +``` + +The first time you start it you will need start another terminal, and go to this repository root folder, to run an additional command to enable federated query in Virtuoso: + +```bash +$ docker compose exec virtuoso isql -U dba -P dba exec='GRANT "SPARQL_SELECT_FED" TO "SPARQL";' +``` + +After cloning the repository and installing the dependencies, the unit tests in the `tests/` folder can be +run reproducibly with: + +```shell +$ tox +``` + +You can also run tests files individually to quickly test a specific feature: + +```bash +$ pytest tests/test_sparql.py +``` + +Additionally, these tests are automatically re-run with each commit in a [GitHub Action](https://github.com/cthoyt/curies/actions?query=workflow%3ATests). + +
diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..9aad3cbf9 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,19 @@ +version: "3" +services: + + bioregistry: + build: . + ports: + - 8888:8766 + + blazegraph: + image: metaphacts/blazegraph-basic:2.2.0-20160908.003514-6-jetty9.4.44-jre8-45dbfff + ports: + - 8889:8080 + + virtuoso: + image: openlink/virtuoso-opensource-7:latest + ports: + - 8890:8890 + environment: + - DBA_PASSWORD=${VIRTUOSO_PASSWORD:-dba} diff --git a/tests/test_sparql.py b/tests/test_sparql.py index 4976e464c..cb5eb6533 100644 --- a/tests/test_sparql.py +++ b/tests/test_sparql.py @@ -1,5 +1,6 @@ """Tests for the local SPARQL endpoint.""" +import csv import unittest from typing import Set, Tuple from xml import etree @@ -7,8 +8,11 @@ import requests PING_SPARQL = 'SELECT ?s ?o WHERE { BIND("hello" as ?s) . BIND("there" as ?o) . }' -LOCAL_BIOREGISTRY = "http://localhost:5000/sparql" -LOCAL_BLAZEGRAPH = "http://192.168.2.30:9999/blazegraph/sparql" +# NOTE: federated queries need to use docker internal URL +DOCKER_BIOREGISTRY = "http://bioregistry:8766/sparql" +LOCAL_BIOREGISTRY = "http://localhost:8888/sparql" +LOCAL_BLAZEGRAPH = "http://localhost:8889/blazegraph/namespace/kb/sparql" +LOCAL_VIRTUOSO = "http://localhost:8890/sparql" def _handle_res_xml(res: requests.Response) -> Set[Tuple[str, str]]: @@ -32,14 +36,13 @@ def _handle_res_json(res: requests.Response) -> Set[Tuple[str, str]]: def _handle_res_csv(res: requests.Response) -> Set[Tuple[str, str]]: - header, *lines = (line.strip().split(",") for line in res.text.splitlines()) - records = (dict(zip(header, line)) for line in lines) - return {(record["s"], record["o"]) for record in records} + reader = csv.DictReader(res.text.splitlines()) + return {(record["s"], record["o"]) for record in reader} HANDLERS = { "application/json": _handle_res_json, - "application/xml": _handle_res_xml, + "application/sparql-results+xml": _handle_res_xml, "text/csv": _handle_res_csv, } @@ -64,12 +67,22 @@ def sparql_service_available(endpoint: str) -> bool: return list(records) == [("hello", "there")] -SPARQL = f"""\ +SPARQL_VALUES = f"""\ PREFIX owl: SELECT DISTINCT ?s ?o WHERE {{ - SERVICE <{LOCAL_BIOREGISTRY}> {{ - VALUES ?s {{ }} - ?s owl:sameAs ?o + SERVICE <{DOCKER_BIOREGISTRY}> {{ + VALUES ?s {{ }} . + ?s owl:sameAs ?o . + }} +}} +""".rstrip() + +SPARQL_SIMPLE = f"""\ +PREFIX owl: +SELECT DISTINCT ?s ?o WHERE {{ + SERVICE <{DOCKER_BIOREGISTRY}> {{ + owl:sameAs ?o . + ?s owl:sameAs ?o . }} }} """.rstrip() @@ -79,11 +92,11 @@ def sparql_service_available(endpoint: str) -> bool: sparql_service_available(LOCAL_BIOREGISTRY), reason="No local Bioregistry is running" ) class TestSPARQL(unittest.TestCase): - """Tests for SPARQL.""" + """Tests for federated SPARQL queries to the Bioregistry mapping service.""" - def assert_endpoint(self, endpoint: str, *, accept: str): + def assert_endpoint(self, endpoint: str, query: str, *, accept: str): """Assert the endpoint returns favorable results.""" - records = get(endpoint, SPARQL, accept=accept) + records = get(endpoint, query, accept=accept) self.assertIn( ("http://purl.obolibrary.org/obo/CHEBI_24867", "https://bioregistry.io/chebi:24867"), records, @@ -93,13 +106,27 @@ def assert_endpoint(self, endpoint: str, *, accept: str): sparql_service_available(LOCAL_BLAZEGRAPH), reason="No local BlazeGraph is running" ) def test_federate_blazegraph(self): - """Test federating on a blazegraph. + """Test federating on a Blazegraph triplestore. - How to run blazegraph locally: + To run blazegraph locally: docker compose up + """ + for mimetype in HANDLERS: + with self.subTest(mimetype=mimetype): + self.assert_endpoint(LOCAL_BLAZEGRAPH, SPARQL_SIMPLE, accept=mimetype) + self.assert_endpoint(LOCAL_BLAZEGRAPH, SPARQL_VALUES, accept=mimetype) + + @unittest.skipUnless( + sparql_service_available(LOCAL_VIRTUOSO), reason="No local Virtuoso is running" + ) + def test_federate_virtuoso(self): + """Test federating on a OpenLink Virtuoso triplestore. - 1. Get: https://github.com/blazegraph/database/releases/download/BLAZEGRAPH_2_1_6_RC/blazegraph.jar - 2. Run: java -jar blazegraph.jar + To run Virtuoso locally: + 1. docker compose up + 2. docker compose exec virtuoso isql -U dba -P dba exec='GRANT "SPARQL_SELECT_FED" TO "SPARQL";' """ for mimetype in HANDLERS: with self.subTest(mimetype=mimetype): - self.assert_endpoint(LOCAL_BLAZEGRAPH, accept=mimetype) + self.assert_endpoint(LOCAL_VIRTUOSO, SPARQL_SIMPLE, accept=mimetype) + # TODO: Virtuoso fails to resolves VALUES in federated query + # self.assert_endpoint(LOCAL_VIRTUOSO, SPARQL_VALUES, accept=mimetype) From 51fd98df683ef0f5c90f4ebb30ce4f700172d890 Mon Sep 17 00:00:00 2001 From: Vincent Emonet Date: Sun, 26 Mar 2023 15:45:01 +0200 Subject: [PATCH 2/5] fix test workflow --- .github/workflows/tests.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c0f9020bf..5f6308e22 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -24,13 +24,6 @@ jobs: run: | pip install --upgrade pip setuptools wheel pip install tox - - - name: Start blazegraph and virtuoso triplestores with docker - run: | - docker-compose up -d - sleep(5) - docker compose exec virtuoso isql -U dba -P dba exec='GRANT "SPARQL_SELECT_FED" TO "SPARQL";' - - name: Check manifest run: tox -e manifest - name: Check code quality with flake8 @@ -78,6 +71,13 @@ jobs: run: | pip install --upgrade pip setuptools wheel pip install "tox<4.0.0" + + - name: Start blazegraph and virtuoso triplestores with docker + run: | + docker-compose up -d + sleep 5 + docker compose exec virtuoso isql -U dba -P dba exec='GRANT "SPARQL_SELECT_FED" TO "SPARQL";' + - name: Test with pytest run: tox -e py From c7e4220b7ee66a4ce3468b3ec5b83757cbc7b388 Mon Sep 17 00:00:00 2001 From: Vincent Emonet Date: Sun, 26 Mar 2023 15:49:49 +0200 Subject: [PATCH 3/5] increase sleep to wait for virtuoso to start in test workflow --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5f6308e22..20fb648cf 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -75,7 +75,7 @@ jobs: - name: Start blazegraph and virtuoso triplestores with docker run: | docker-compose up -d - sleep 5 + sleep 20 docker compose exec virtuoso isql -U dba -P dba exec='GRANT "SPARQL_SELECT_FED" TO "SPARQL";' - name: Test with pytest From 8c36dca3d05bbce473900fb3051f6d8eaf40a56a Mon Sep 17 00:00:00 2001 From: Vincent Emonet Date: Sun, 26 Mar 2023 15:56:16 +0200 Subject: [PATCH 4/5] fix test workflow --- .github/workflows/tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 20fb648cf..02cd21e4f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -58,6 +58,7 @@ jobs: name: Tests runs-on: ${{ matrix.os }} strategy: + fail-fast: false matrix: os: [ ubuntu-latest, windows-latest ] python-version: [ "3.11", "3.7" ] @@ -73,6 +74,7 @@ jobs: pip install "tox<4.0.0" - name: Start blazegraph and virtuoso triplestores with docker + if: matrix.os == 'ubuntu-latest' run: | docker-compose up -d sleep 20 From 4508d79aaa08ddb054fff76658a617e2cb19be95 Mon Sep 17 00:00:00 2001 From: Vincent Emonet Date: Sun, 26 Mar 2023 16:39:48 +0200 Subject: [PATCH 5/5] Add docker-compose.yml to MANIFEST.in ignore, remove fail-fast false from workflow, and improve readme docs for tests --- .github/workflows/tests.yml | 3 -- MANIFEST.in | 2 +- README.md | 75 +++++++++++++------------------------ 3 files changed, 26 insertions(+), 54 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 02cd21e4f..6be2b41f3 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -58,7 +58,6 @@ jobs: name: Tests runs-on: ${{ matrix.os }} strategy: - fail-fast: false matrix: os: [ ubuntu-latest, windows-latest ] python-version: [ "3.11", "3.7" ] @@ -72,14 +71,12 @@ jobs: run: | pip install --upgrade pip setuptools wheel pip install "tox<4.0.0" - - name: Start blazegraph and virtuoso triplestores with docker if: matrix.os == 'ubuntu-latest' run: | docker-compose up -d sleep 20 docker compose exec virtuoso isql -U dba -P dba exec='GRANT "SPARQL_SELECT_FED" TO "SPARQL";' - - name: Test with pytest run: tox -e py diff --git a/MANIFEST.in b/MANIFEST.in index c3c7f697f..f5f75bb35 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -19,5 +19,5 @@ prune tests/.pytest_cache exclude docs/*.md exclude docs/_config.yml -exclude .appveyor.yml .bumpversion.cfg .coveragerc .flake8 .travis.yml .readthedocs.yml tox.ini .pre-commit-config.yaml Dockerfile CITATION.cff +exclude .appveyor.yml .bumpversion.cfg .coveragerc .flake8 .travis.yml .readthedocs.yml tox.ini .pre-commit-config.yaml Dockerfile CITATION.cff docker-compose.yml include LICENSE *.md *.rst diff --git a/README.md b/README.md index 47c38cfeb..fc6125976 100644 --- a/README.md +++ b/README.md @@ -127,6 +127,31 @@ $ pip install --editable . Build the docs locally with `tox -e ldocs` then view by opening `docs/build/html/index.html`. +
+ See the instructions to run the tests + +Having [`tox`](https://tox.wiki/en/latest/installation.html) installed is required to run the tests. Optionally to run all tests you will need to start the Bioregistry, a Virtuoso endpoint, and a Blazegraph endpoint using `docker compose`: + +```shell +$ docker compose up +``` + +The first time you deploy the docker compose stack, you will need to run an additional command from another terminal, in this repository root folder, to enable federated query in Virtuoso: + +```shell +$ docker compose exec virtuoso isql -U dba -P dba exec='GRANT "SPARQL_SELECT_FED" TO "SPARQL";' +``` + +After cloning the repository and installing the dependencies, the unit tests in the `tests/` folder can be run reproducibly with: + +```shell +$ tox +``` + +Additionally, these tests are automatically re-run with each commit in a [GitHub Action](https://github.com/biopragmatics/bioregistry/actions/workflows/tests.yml). + +
+ ## 💪 Usage ### Normalizing Prefixes @@ -475,53 +500,3 @@ at [Harvard Medical School](https://hms.harvard.edu/). ### 💰 Funding The development of the Bioregistry is funded by the DARPA Young Faculty Award W911NF2010255 (PI: Benjamin M. Gyori). - - -## 🛠️ For Developers - -
- See developer instructions - - -The final section of the README is for if you want to get involved by making a code contribution. - -### 📥️ Development Installation - -To install in development mode, consider enabling a virtual environment, and use the following: - -```bash -$ git clone git+https://github.com/biopragmatics/bioregistry.git -$ cd bioregistry -$ pip install -e ".[tests,web]" -``` - -### 🥼 Testing - -To run all tests you will need to start the Bioregistry, a Virtuoso endpoint, and a Blazegraph endpoint using `docker compose`: - -```bash -$ docker compose up -``` - -The first time you start it you will need start another terminal, and go to this repository root folder, to run an additional command to enable federated query in Virtuoso: - -```bash -$ docker compose exec virtuoso isql -U dba -P dba exec='GRANT "SPARQL_SELECT_FED" TO "SPARQL";' -``` - -After cloning the repository and installing the dependencies, the unit tests in the `tests/` folder can be -run reproducibly with: - -```shell -$ tox -``` - -You can also run tests files individually to quickly test a specific feature: - -```bash -$ pytest tests/test_sparql.py -``` - -Additionally, these tests are automatically re-run with each commit in a [GitHub Action](https://github.com/cthoyt/curies/actions?query=workflow%3ATests). - -