From 20844fe7c4ec2a4fa1863e18cbf890a833841302 Mon Sep 17 00:00:00 2001 From: tarilabs Date: Tue, 30 Jul 2024 10:12:54 +0200 Subject: [PATCH] core: add end-to-end testing with distribution Signed-off-by: tarilabs --- .github/workflows/build.yaml | 2 +- .github/workflows/e2e.yaml | 34 +++++++++++ docs/demos/demo.ipynb | 2 +- e2e/deploy_distribution_registry.sh | 16 +++++ .../distribution-registry.yaml | 59 +++++++++++++++++++ omlmd/helpers.py | 2 +- pyproject.toml | 5 ++ tests/conftest.py | 26 ++++++++ tests/test_helpers.py | 39 ++++++++++++ 9 files changed, 182 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/e2e.yaml create mode 100755 e2e/deploy_distribution_registry.sh create mode 100644 e2e/distribution-registry/distribution-registry.yaml create mode 100644 tests/conftest.py diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index c39107b..13f5d3a 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -20,4 +20,4 @@ jobs: poetry install - name: Run tests run: | - poetry run pytest + poetry run pytest -s -x diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml new file mode 100644 index 0000000..91d73d4 --- /dev/null +++ b/.github/workflows/e2e.yaml @@ -0,0 +1,34 @@ +name: end-to-end testing + +on: [push, pull_request] + +jobs: + prepare: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.10' + - name: Install Poetry + run: | + pipx install poetry + - name: Install dependencies + run: | + poetry install + - name: Start Kind Cluster + uses: helm/kind-action@v1 + with: + cluster_name: kind-kind + e2e-distribution-registry: + runs-on: ubuntu-latest + needs: prepare + steps: + - name: Start distribution-registry + run: | + e2e/deploy_distribution_registry.sh + - name: Run tests + run: | + poetry run pytest --e2e -s -x diff --git a/docs/demos/demo.ipynb b/docs/demos/demo.ipynb index 913d520..7777d46 100644 --- a/docs/demos/demo.ipynb +++ b/docs/demos/demo.ipynb @@ -149,7 +149,7 @@ } ], "source": [ - "omlmd.pull(target=\"localhost:8080/matteo/ml-artifact:latest\", outdir=\"tmp/b\", media_types=[\"application/x-artifact\"])\n", + "omlmd.pull(target=\"localhost:8080/matteo/ml-artifact:latest\", outdir=\"tmp/b\", media_types=[\"application/x-mlmodel\"])\n", "\n", "%ls -lA tmp/b" ] diff --git a/e2e/deploy_distribution_registry.sh b/e2e/deploy_distribution_registry.sh new file mode 100755 index 0000000..9613ed6 --- /dev/null +++ b/e2e/deploy_distribution_registry.sh @@ -0,0 +1,16 @@ +#! /bin/bash + +SCRIPT_DIR="$(dirname "$(realpath "$BASH_SOURCE")")" + +kubectl apply -f "${SCRIPT_DIR}/distribution-registry/" + +echo "Waiting for Deployment..." +kubectl wait --for=condition=available deployment/distribution-registry-test-deployment --timeout=5m +kubectl logs deployment/distribution-registry-test-deployment +echo "Deployment looks ready." + +echo "Starting port-forward..." +kubectl port-forward service/distribution-registry-test-service 5001:5001 & +PID=$! +sleep 2 +echo "I have launched port-forward in background with: $PID." diff --git a/e2e/distribution-registry/distribution-registry.yaml b/e2e/distribution-registry/distribution-registry.yaml new file mode 100644 index 0000000..4b7d51e --- /dev/null +++ b/e2e/distribution-registry/distribution-registry.yaml @@ -0,0 +1,59 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: distribution-registry-test-deployment + labels: + app: distribution-registry-test +spec: + replicas: 1 + selector: + matchLabels: + app: distribution-registry-test + template: + metadata: + labels: + app: distribution-registry-test + spec: + containers: + - name: distribution-registry-test + image: docker.io/library/registry:2 + args: + - /etc/docker/registry/config.yml + env: + - name: REGISTRY_HTTP_ADDR + value: 0.0.0.0:5001 + ports: + - containerPort: 5001 + volumeMounts: + - mountPath: /var/lib/registry + name: distribution-registry-storage + volumes: + - name: distribution-registry-storage + persistentVolumeClaim: + claimName: distribution-registry-pvc +--- +apiVersion: v1 +kind: Service +metadata: + name: distribution-registry-test-service + labels: + app: distribution-registry-test +spec: + selector: + app: distribution-registry-test + ports: + - protocol: TCP + port: 5001 + targetPort: 5001 +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: distribution-registry-pvc +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 100Mi diff --git a/omlmd/helpers.py b/omlmd/helpers.py index eec2ae8..52189be 100644 --- a/omlmd/helpers.py +++ b/omlmd/helpers.py @@ -77,7 +77,7 @@ def pull( self, target: str, outdir: str, - media_types: List[str] + media_types: Optional[List[str]] = [] ): self._registry.download_layers(target, outdir, media_types) diff --git a/pyproject.toml b/pyproject.toml index 4063b59..db149e8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,3 +25,8 @@ omlmd = "omlmd.cli:cli" [build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" + +[tool.pytest.ini_options] +markers = [ + "e2e: end-to-end testing with localhost:5001", +] diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..ae420c5 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,26 @@ +import pytest + + +def pytest_collection_modifyitems(config, items): + if config.getoption("--e2e"): + skip_not_e2e = pytest.mark.skip(reason="skipping non-e2e tests") + for item in items: + if "e2e" not in item.keywords: + item.add_marker(skip_not_e2e) + return + skip_e2e = pytest.mark.skip(reason="test requires --e2e option to run") + for item in items: + if "e2e" in item.keywords: + item.add_marker(skip_e2e) + + +def pytest_addoption(parser): + parser.addoption( + "--e2e", action="store_true", default=False, help="opt-in to run tests marked with e2e" + ) + + +@pytest.fixture +def target() -> str: + return "localhost:5001/mmortari/mlartifact:v1" + diff --git a/tests/test_helpers.py b/tests/test_helpers.py index ab0536b..a88b838 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -2,6 +2,8 @@ from omlmd.model_metadata import deserialize_mdfile import tempfile import json +import pytest +from pathlib import Path def test_call_push_using_md_from_file(mocker): helper = Helper() @@ -27,3 +29,40 @@ def test_call_push_using_md_from_file(mocker): author="John Doe", accuracy=0.987 ) + + +@pytest.mark.e2e +def test_e2e_push_pull(tmp_path, target): + omlmd = Helper() + omlmd.push( + target, + Path(__file__).parent / ".." / "README.md", + name="mnist", + description="Lorem ipsum", + author="John Doe", + accuracy=0.987 + ) + omlmd.pull( + target, + tmp_path + ) + assert len(list(tmp_path.iterdir())) == 3 + + +@pytest.mark.e2e +def test_e2e_push_pull_with_filters(tmp_path, target): + omlmd = Helper() + omlmd.push( + target, + Path(__file__).parent / ".." / "README.md", + name="mnist", + description="Lorem ipsum", + author="John Doe", + accuracy=0.987 + ) + omlmd.pull( + target, + tmp_path, + media_types=["application/x-mlmodel"] + ) + assert len(list(tmp_path.iterdir())) == 1