diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 737914bd2..6be2b41f3 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,6 +1,7 @@ name: Tests on: + workflow_dispatch: push: branches: [ main ] pull_request: @@ -70,6 +71,12 @@ jobs: run: | pip install --upgrade pip setuptools wheel pip install "tox<4.0.0" + - name: Start blazegraph and virtuoso triplestores with docker + if: matrix.os == 'ubuntu-latest' + run: | + docker-compose up -d + sleep 20 + docker compose exec virtuoso isql -U dba -P dba exec='GRANT "SPARQL_SELECT_FED" TO "SPARQL";' - name: Test with pytest run: tox -e py diff --git a/MANIFEST.in b/MANIFEST.in index c3c7f697f..f5f75bb35 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -19,5 +19,5 @@ prune tests/.pytest_cache exclude docs/*.md exclude docs/_config.yml -exclude .appveyor.yml .bumpversion.cfg .coveragerc .flake8 .travis.yml .readthedocs.yml tox.ini .pre-commit-config.yaml Dockerfile CITATION.cff +exclude .appveyor.yml .bumpversion.cfg .coveragerc .flake8 .travis.yml .readthedocs.yml tox.ini .pre-commit-config.yaml Dockerfile CITATION.cff docker-compose.yml include LICENSE *.md *.rst diff --git a/README.md b/README.md index a06b1eac1..fc6125976 100644 --- a/README.md +++ b/README.md @@ -127,6 +127,31 @@ $ pip install --editable . Build the docs locally with `tox -e ldocs` then view by opening `docs/build/html/index.html`. +
+ See the instructions to run the tests + +Having [`tox`](https://tox.wiki/en/latest/installation.html) installed is required to run the tests. Optionally to run all tests you will need to start the Bioregistry, a Virtuoso endpoint, and a Blazegraph endpoint using `docker compose`: + +```shell +$ docker compose up +``` + +The first time you deploy the docker compose stack, you will need to run an additional command from another terminal, in this repository root folder, to enable federated query in Virtuoso: + +```shell +$ docker compose exec virtuoso isql -U dba -P dba exec='GRANT "SPARQL_SELECT_FED" TO "SPARQL";' +``` + +After cloning the repository and installing the dependencies, the unit tests in the `tests/` folder can be run reproducibly with: + +```shell +$ tox +``` + +Additionally, these tests are automatically re-run with each commit in a [GitHub Action](https://github.com/biopragmatics/bioregistry/actions/workflows/tests.yml). + +
+ ## 💪 Usage ### Normalizing Prefixes diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..9aad3cbf9 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,19 @@ +version: "3" +services: + + bioregistry: + build: . + ports: + - 8888:8766 + + blazegraph: + image: metaphacts/blazegraph-basic:2.2.0-20160908.003514-6-jetty9.4.44-jre8-45dbfff + ports: + - 8889:8080 + + virtuoso: + image: openlink/virtuoso-opensource-7:latest + ports: + - 8890:8890 + environment: + - DBA_PASSWORD=${VIRTUOSO_PASSWORD:-dba} diff --git a/tests/test_sparql.py b/tests/test_sparql.py index 4976e464c..cb5eb6533 100644 --- a/tests/test_sparql.py +++ b/tests/test_sparql.py @@ -1,5 +1,6 @@ """Tests for the local SPARQL endpoint.""" +import csv import unittest from typing import Set, Tuple from xml import etree @@ -7,8 +8,11 @@ import requests PING_SPARQL = 'SELECT ?s ?o WHERE { BIND("hello" as ?s) . BIND("there" as ?o) . }' -LOCAL_BIOREGISTRY = "http://localhost:5000/sparql" -LOCAL_BLAZEGRAPH = "http://192.168.2.30:9999/blazegraph/sparql" +# NOTE: federated queries need to use docker internal URL +DOCKER_BIOREGISTRY = "http://bioregistry:8766/sparql" +LOCAL_BIOREGISTRY = "http://localhost:8888/sparql" +LOCAL_BLAZEGRAPH = "http://localhost:8889/blazegraph/namespace/kb/sparql" +LOCAL_VIRTUOSO = "http://localhost:8890/sparql" def _handle_res_xml(res: requests.Response) -> Set[Tuple[str, str]]: @@ -32,14 +36,13 @@ def _handle_res_json(res: requests.Response) -> Set[Tuple[str, str]]: def _handle_res_csv(res: requests.Response) -> Set[Tuple[str, str]]: - header, *lines = (line.strip().split(",") for line in res.text.splitlines()) - records = (dict(zip(header, line)) for line in lines) - return {(record["s"], record["o"]) for record in records} + reader = csv.DictReader(res.text.splitlines()) + return {(record["s"], record["o"]) for record in reader} HANDLERS = { "application/json": _handle_res_json, - "application/xml": _handle_res_xml, + "application/sparql-results+xml": _handle_res_xml, "text/csv": _handle_res_csv, } @@ -64,12 +67,22 @@ def sparql_service_available(endpoint: str) -> bool: return list(records) == [("hello", "there")] -SPARQL = f"""\ +SPARQL_VALUES = f"""\ PREFIX owl: SELECT DISTINCT ?s ?o WHERE {{ - SERVICE <{LOCAL_BIOREGISTRY}> {{ - VALUES ?s {{ }} - ?s owl:sameAs ?o + SERVICE <{DOCKER_BIOREGISTRY}> {{ + VALUES ?s {{ }} . + ?s owl:sameAs ?o . + }} +}} +""".rstrip() + +SPARQL_SIMPLE = f"""\ +PREFIX owl: +SELECT DISTINCT ?s ?o WHERE {{ + SERVICE <{DOCKER_BIOREGISTRY}> {{ + owl:sameAs ?o . + ?s owl:sameAs ?o . }} }} """.rstrip() @@ -79,11 +92,11 @@ def sparql_service_available(endpoint: str) -> bool: sparql_service_available(LOCAL_BIOREGISTRY), reason="No local Bioregistry is running" ) class TestSPARQL(unittest.TestCase): - """Tests for SPARQL.""" + """Tests for federated SPARQL queries to the Bioregistry mapping service.""" - def assert_endpoint(self, endpoint: str, *, accept: str): + def assert_endpoint(self, endpoint: str, query: str, *, accept: str): """Assert the endpoint returns favorable results.""" - records = get(endpoint, SPARQL, accept=accept) + records = get(endpoint, query, accept=accept) self.assertIn( ("http://purl.obolibrary.org/obo/CHEBI_24867", "https://bioregistry.io/chebi:24867"), records, @@ -93,13 +106,27 @@ def assert_endpoint(self, endpoint: str, *, accept: str): sparql_service_available(LOCAL_BLAZEGRAPH), reason="No local BlazeGraph is running" ) def test_federate_blazegraph(self): - """Test federating on a blazegraph. + """Test federating on a Blazegraph triplestore. - How to run blazegraph locally: + To run blazegraph locally: docker compose up + """ + for mimetype in HANDLERS: + with self.subTest(mimetype=mimetype): + self.assert_endpoint(LOCAL_BLAZEGRAPH, SPARQL_SIMPLE, accept=mimetype) + self.assert_endpoint(LOCAL_BLAZEGRAPH, SPARQL_VALUES, accept=mimetype) + + @unittest.skipUnless( + sparql_service_available(LOCAL_VIRTUOSO), reason="No local Virtuoso is running" + ) + def test_federate_virtuoso(self): + """Test federating on a OpenLink Virtuoso triplestore. - 1. Get: https://github.com/blazegraph/database/releases/download/BLAZEGRAPH_2_1_6_RC/blazegraph.jar - 2. Run: java -jar blazegraph.jar + To run Virtuoso locally: + 1. docker compose up + 2. docker compose exec virtuoso isql -U dba -P dba exec='GRANT "SPARQL_SELECT_FED" TO "SPARQL";' """ for mimetype in HANDLERS: with self.subTest(mimetype=mimetype): - self.assert_endpoint(LOCAL_BLAZEGRAPH, accept=mimetype) + self.assert_endpoint(LOCAL_VIRTUOSO, SPARQL_SIMPLE, accept=mimetype) + # TODO: Virtuoso fails to resolves VALUES in federated query + # self.assert_endpoint(LOCAL_VIRTUOSO, SPARQL_VALUES, accept=mimetype)