diff --git a/.github/workflows/integrate.yaml b/.github/workflows/integrate.yaml index 7ade8d1d..5b77a721 100644 --- a/.github/workflows/integrate.yaml +++ b/.github/workflows/integrate.yaml @@ -171,6 +171,10 @@ jobs: juju add-model kubeflow sg snap_microk8s -c "tox -e bundle-integration-${{ matrix.sdk }} -- --model kubeflow --bundle=./tests/integration/bundles/kfp_latest_edge.yaml.j2" --charmcraft-clean + - name: Display df + run: df -h + if: always() + - name: Get all run: kubectl get all -A if: failure() diff --git a/requirements-integration-v1.in b/requirements-integration-v1.in index b4bd6e73..62eb5fb5 100644 --- a/requirements-integration-v1.in +++ b/requirements-integration-v1.in @@ -12,3 +12,4 @@ pytest-operator pyyaml tenacity sh +jq diff --git a/requirements-integration-v1.txt b/requirements-integration-v1.txt index 5ba4a321..b7de1809 100644 --- a/requirements-integration-v1.txt +++ b/requirements-integration-v1.txt @@ -12,8 +12,6 @@ aiosignal==1.3.1 # via aiohttp anyio==4.4.0 # via httpx -appnope==0.1.4 - # via ipython asttokens==2.4.1 # via stack-data async-timeout==4.0.3 @@ -130,6 +128,8 @@ jedi==0.19.1 # via ipython jinja2==3.1.4 # via pytest-operator +jq==1.8.0 + # via -r requirements-integration-v1.in jsonschema==4.17.3 # via # -r requirements-integration-v1.in diff --git a/requirements-integration-v2.in b/requirements-integration-v2.in index dc095562..1aef6db5 100644 --- a/requirements-integration-v2.in +++ b/requirements-integration-v2.in @@ -12,3 +12,4 @@ pytest-operator pyyaml sh tenacity +jq diff --git a/requirements-integration-v2.txt b/requirements-integration-v2.txt index 1c49ff25..b49dd671 100644 --- a/requirements-integration-v2.txt +++ b/requirements-integration-v2.txt @@ -109,6 +109,8 @@ jedi==0.19.1 # via ipython jinja2==3.1.4 # via pytest-operator +jq==1.8.0 + # via -r requirements-integration-v2.in jsonschema==4.17.3 # via -r requirements-integration-v2.in juju==3.5.2.0 diff --git a/tests/README.md b/tests/README.md index 908f2b5c..474baee6 100644 --- a/tests/README.md +++ b/tests/README.md @@ -20,6 +20,7 @@ This directory has the following structure: │   ├── charmcraft.py │   ├── k8s_resources.py │   └── localize_bundle.py + │   └── lxc.py ├── kfp_globals.py ├── pipelines/ │   └── ... # Sample pipelines diff --git a/tests/integration/bundles/kfp_1.8_stable_install.yaml.j2 b/tests/integration/bundles/kfp_1.8_stable_install.yaml.j2 index cadabae7..b2b13720 100644 --- a/tests/integration/bundles/kfp_1.8_stable_install.yaml.j2 +++ b/tests/integration/bundles/kfp_1.8_stable_install.yaml.j2 @@ -4,7 +4,6 @@ applications: argo-controller: { charm: ch:argo-controller, channel: 3.3.10/stable, scale: 1, trust: true } metacontroller-operator: { charm: ch:metacontroller-operator, channel: 3.0/stable, scale: 1, trust: true } minio: { charm: ch:minio, channel: ckf-1.8/stable, scale: 1 } - kfp-db: { charm: ch:mysql-k8s, channel: 8.0/stable, scale: 1, constraints: mem=2G, trust: true } mlmd: { charm: ch:mlmd, channel: 1.14/stable, scale: 1 } envoy: { charm: ch:envoy, channel: 2.0/stable, scale: 1 } kubeflow-profiles: { charm: ch:kubeflow-profiles, channel: 1.8/stable, scale: 1, trust: true } @@ -22,6 +21,13 @@ applications: options: default-gateway: kubeflow-gateway trust: true + kfp-db: + charm: mysql-k8s + channel: 8.0/stable + scale: 1 + options: + profile: testing + trust: true kubeflow-roles: charm: kubeflow-roles channel: 1.8/stable diff --git a/tests/integration/bundles/kfp_latest_edge.yaml.j2 b/tests/integration/bundles/kfp_latest_edge.yaml.j2 index 4d4dabc7..bd93e27d 100644 --- a/tests/integration/bundles/kfp_latest_edge.yaml.j2 +++ b/tests/integration/bundles/kfp_latest_edge.yaml.j2 @@ -4,7 +4,6 @@ applications: argo-controller: { charm: ch:argo-controller, channel: latest/edge, scale: 1, trust: true } metacontroller-operator: { charm: ch:metacontroller-operator, channel: latest/edge, scale: 1, trust: true } minio: { charm: ch:minio, channel: latest/edge, scale: 1 } - kfp-db: { charm: ch:mysql-k8s, channel: 8.0/stable, scale: 1, constraints: mem=2G, trust: true } mlmd: { charm: ch:mlmd, channel: latest/edge, scale: 1, trust: true} envoy: { charm: ch:envoy, channel: latest/edge, scale: 1 } kubeflow-profiles: { charm: ch:kubeflow-profiles, channel: latest/edge, scale: 1, trust: true } @@ -22,6 +21,13 @@ applications: options: default-gateway: kubeflow-gateway trust: true + kfp-db: + charm: mysql-k8s + channel: 8.0/stable + scale: 1 + options: + profile: testing + trust: true kubeflow-roles: charm: kubeflow-roles channel: latest/edge diff --git a/tests/integration/helpers/lxc.py b/tests/integration/helpers/lxc.py new file mode 100644 index 00000000..aab0107d --- /dev/null +++ b/tests/integration/helpers/lxc.py @@ -0,0 +1,14 @@ +import sh +import jq + +def clean_charmcraft_lxc_instances() -> None: + """ + Delete lxc instances in project "charmcraft" that are prefixed with "charmcraft-". + + Based on https://discourse.charmhub.io/t/how-to-quickly-clean-unused-lxd-instances-from-charmcraft-pack/15975 + """ + lxc_instances = sh.lxc.list(project="charmcraft", format="json") + lxc_instances_charmcraft = jq.compile('.[] | select(.name | startswith("charmcraft-")) | .name').input_text(lxc_instances).all() + for instance in lxc_instances_charmcraft: + print(f"Deleting lxc instance '{instance}'") + sh.lxc.delete(instance, project="charmcraft") diff --git a/tests/integration/test_kfp_functional_v1.py b/tests/integration/test_kfp_functional_v1.py index 9562e1f0..3e7e394c 100644 --- a/tests/integration/test_kfp_functional_v1.py +++ b/tests/integration/test_kfp_functional_v1.py @@ -10,6 +10,7 @@ from helpers.k8s_resources import apply_manifests, fetch_response from helpers.localize_bundle import get_resources_from_charm_file from helpers.charmcraft import charmcraft_clean +from helpers.lxc import clean_charmcraft_lxc_instances from kfp_globals import ( CHARM_PATH_TEMPLATE, KFP_CHARMS, @@ -19,7 +20,6 @@ SAMPLE_VIEWER, ) -import sh import kfp import lightkube import pytest @@ -97,7 +97,8 @@ async def test_build_and_deploy(ops_test: OpsTest, request, lightkube_client): context.update([(f"{charm.replace('-', '_')}", charm_file)]) if charmcraft_clean_flag == True: - charmcraft_clean(charms_to_build) + # charmcraft_clean(charms_to_build) + clean_charmcraft_lxc_instances() # Render kfp-operators bundle file with locally built charms and their resources rendered_bundle = render_bundle( @@ -107,11 +108,16 @@ async def test_build_and_deploy(ops_test: OpsTest, request, lightkube_client): # Deploy the kfp-operators bundle from the rendered bundle file await deploy_bundle(ops_test, bundle_path=rendered_bundle, trust=True) - # Use `juju wait-for` instead of `wait_for_idle()` - # due to https://github.com/canonical/kfp-operators/issues/601 - # and https://github.com/juju/python-libjuju/issues/1204 - log.info("Waiting on model applications to be active") - sh.juju("wait-for","model","kubeflow", query="forEach(applications, app => app.status == 'active')", timeout="30m") + # Wait for everything to be up. Note, at time of writing these charms would naturally go + # into blocked during deploy while waiting for each other to satisfy relations, so we don't + # raise_on_blocked. + await ops_test.model.wait_for_idle( + status="active", + raise_on_blocked=False, # These apps block while waiting for each other to deploy/relate + raise_on_error=True, + timeout=3600, + idle_period=30, + ) # ---- KFP API Server focused test cases diff --git a/tests/integration/test_kfp_functional_v2.py b/tests/integration/test_kfp_functional_v2.py index 8d891425..dd7a08e9 100644 --- a/tests/integration/test_kfp_functional_v2.py +++ b/tests/integration/test_kfp_functional_v2.py @@ -10,6 +10,7 @@ from helpers.k8s_resources import apply_manifests, fetch_response from helpers.localize_bundle import get_resources_from_charm_file from helpers.charmcraft import charmcraft_clean +from helpers.lxc import clean_charmcraft_lxc_instances from kfp_globals import ( CHARM_PATH_TEMPLATE, KFP_CHARMS, @@ -99,7 +100,8 @@ async def test_build_and_deploy(ops_test: OpsTest, request, lightkube_client): context.update([(f"{charm.replace('-', '_')}", charm_file)]) if charmcraft_clean_flag == True: - charmcraft_clean(charms_to_build) + # charmcraft_clean(charms_to_build) + clean_charmcraft_lxc_instances() # Render kfp-operators bundle file with locally built charms and their resources rendered_bundle = render_bundle( @@ -109,11 +111,17 @@ async def test_build_and_deploy(ops_test: OpsTest, request, lightkube_client): # Deploy the kfp-operators bundle from the rendered bundle file await deploy_bundle(ops_test, bundle_path=rendered_bundle, trust=True) - # Use `juju wait-for` instead of `wait_for_idle()` - # due to https://github.com/canonical/kfp-operators/issues/601 - # and https://github.com/juju/python-libjuju/issues/1204 - log.info("Waiting on model applications to be active") - sh.juju("wait-for","model","kubeflow", query="forEach(applications, app => app.status == 'active')", timeout="30m") + # Wait for everything to be up. Note, at time of writing these charms would naturally go + # into blocked during deploy while waiting for each other to satisfy relations, so we don't + # raise_on_blocked. + await ops_test.model.wait_for_idle( + status="active", + raise_on_blocked=False, # These apps block while waiting for each other to deploy/relate + raise_on_error=True, + timeout=3600, + idle_period=30, + ) + # ---- KFP API Server focused test cases async def test_upload_pipeline(kfp_client):