core: add end-to-end testing with distribution

Signed-off-by: tarilabs <[email protected]>

.github/workflows/build.yaml

@@ -20,4 +20,4 @@ jobs:
         poetry install
     - name: Run tests
       run: |
-        poetry run pytest
+        poetry run pytest -s -x

.github/workflows/e2e.yaml

@@ -0,0 +1,34 @@
+name: end-to-end testing
+
+on: [push, pull_request]
+
+jobs:
+  prepare:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.10' 
+    - name: Install Poetry
+      run: |
+        pipx install poetry
+    - name: Install dependencies
+      run: |
+        poetry install
+    - name: Start Kind Cluster
+      uses: helm/kind-action@v1
+      with:
+        cluster_name: kind-kind
+  e2e-distribution-registry:
+    runs-on: ubuntu-latest
+    needs: prepare
+    steps:
+    - name: Start distribution-registry
+      run: |
+        e2e/deploy_distribution_registry.sh
+    - name: Run tests
+      run: |
+        poetry run pytest --e2e -s -x

docs/demos/demo.ipynb

@@ -149,7 +149,7 @@
     }
    ],
    "source": [
-    "omlmd.pull(target=\"localhost:8080/matteo/ml-artifact:latest\", outdir=\"tmp/b\", media_types=[\"application/x-artifact\"])\n",
+    "omlmd.pull(target=\"localhost:8080/matteo/ml-artifact:latest\", outdir=\"tmp/b\", media_types=[\"application/x-mlmodel\"])\n",
     "\n",
     "%ls -lA tmp/b"
    ]

e2e/deploy_distribution_registry.sh

@@ -0,0 +1,16 @@
+#! /bin/bash
+
+SCRIPT_DIR="$(dirname "$(realpath "$BASH_SOURCE")")"
+
+kubectl apply -f "${SCRIPT_DIR}/distribution-registry/"
+
+echo "Waiting for Deployment..."
+kubectl wait --for=condition=available deployment/distribution-registry-test-deployment --timeout=5m
+kubectl logs deployment/distribution-registry-test-deployment
+echo "Deployment looks ready."
+
+echo "Starting port-forward..."
+kubectl port-forward service/distribution-registry-test-service 5001:5001 &
+PID=$!
+sleep 2
+echo "I have launched port-forward in background with: $PID."

e2e/distribution-registry/distribution-registry.yaml

@@ -0,0 +1,59 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: distribution-registry-test-deployment
+  labels:
+    app: distribution-registry-test
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: distribution-registry-test
+  template:
+    metadata:
+      labels:
+        app: distribution-registry-test
+    spec:
+      containers:
+      - name: distribution-registry-test
+        image: docker.io/library/registry:2
+        args:
+        - /etc/docker/registry/config.yml
+        env:
+        - name: REGISTRY_HTTP_ADDR
+          value: 0.0.0.0:5001
+        ports:
+        - containerPort: 5001
+        volumeMounts:
+        - mountPath: /var/lib/registry
+          name: distribution-registry-storage
+      volumes:
+      - name: distribution-registry-storage
+        persistentVolumeClaim:
+          claimName: distribution-registry-pvc
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: distribution-registry-test-service
+  labels:
+    app: distribution-registry-test
+spec:
+  selector:
+    app: distribution-registry-test
+  ports:
+  - protocol: TCP
+    port: 5001
+    targetPort: 5001
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: distribution-registry-pvc
+spec:
+  accessModes:
+  - ReadWriteOnce
+  resources:
+    requests:
+      storage: 100Mi

omlmd/helpers.py

@@ -77,7 +77,7 @@ def pull(
         self,
         target: str,
         outdir: str,
-        media_types: List[str]
+        media_types: Optional[List[str]] = []
     ):
         self._registry.download_layers(target, outdir, media_types)
 

pyproject.toml

@@ -25,3 +25,8 @@ omlmd = "omlmd.cli:cli"
 [build-system]
 requires = ["poetry-core"]
 build-backend = "poetry.core.masonry.api"
+
+[tool.pytest.ini_options]
+markers = [
+    "e2e: end-to-end testing with localhost:5001",
+]

tests/conftest.py

@@ -0,0 +1,26 @@
+import pytest
+
+
+def pytest_collection_modifyitems(config, items):
+    if config.getoption("--e2e"):
+        skip_not_e2e = pytest.mark.skip(reason="skipping non-e2e tests")
+        for item in items:
+            if "e2e" not in item.keywords:
+                item.add_marker(skip_not_e2e)
+        return
+    skip_e2e = pytest.mark.skip(reason="test requires --e2e option to run")
+    for item in items:
+        if "e2e" in item.keywords:
+            item.add_marker(skip_e2e)
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--e2e", action="store_true", default=False, help="opt-in to run tests marked with e2e"
+    )
+
+
+@pytest.fixture
+def target() -> str:
+    return "localhost:5001/mmortari/mlartifact:v1"
+

tests/test_helpers.py

@@ -2,6 +2,8 @@
 from omlmd.model_metadata import deserialize_mdfile
 import tempfile
 import json
+import pytest
+from pathlib import Path
 
 def test_call_push_using_md_from_file(mocker):
     helper = Helper()
@@ -27,3 +29,40 @@ def test_call_push_using_md_from_file(mocker):
         author="John Doe",
         accuracy=0.987
     )
+
+
+@pytest.mark.e2e
+def test_e2e_push_pull(tmp_path, target):
+    omlmd = Helper()
+    omlmd.push(
+        target,
+        Path(__file__).parent / ".." / "README.md",
+        name="mnist",
+        description="Lorem ipsum",
+        author="John Doe",
+        accuracy=0.987
+    )
+    omlmd.pull(
+        target,
+        tmp_path
+    )
+    assert len(list(tmp_path.iterdir())) == 3
+
+
+@pytest.mark.e2e
+def test_e2e_push_pull_with_filters(tmp_path, target):
+    omlmd = Helper()
+    omlmd.push(
+        target,
+        Path(__file__).parent / ".." / "README.md",
+        name="mnist",
+        description="Lorem ipsum",
+        author="John Doe",
+        accuracy=0.987
+    )
+    omlmd.pull(
+        target,
+        tmp_path,
+        media_types=["application/x-mlmodel"]
+    )
+    assert len(list(tmp_path.iterdir())) == 1