From 8e5daeb1d0a8cf6e9cf8db29882bdc8da10ea857 Mon Sep 17 00:00:00 2001 From: Daniela Plascencia Date: Tue, 26 Nov 2024 18:40:51 +0100 Subject: [PATCH 1/5] ci, fix: change profile of mysql-k8s to testing The mysql-k8s charm, used as kfp-db, has a profile configuration option that can be set to testing which sets the scope of deployment. This is recommended for testing environments with limited memory/disk. --- tests/integration/bundles/kfp_1.8_stable_install.yaml.j2 | 8 +++++++- tests/integration/bundles/kfp_latest_edge.yaml.j2 | 8 +++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/tests/integration/bundles/kfp_1.8_stable_install.yaml.j2 b/tests/integration/bundles/kfp_1.8_stable_install.yaml.j2 index cadabae7..b2b13720 100644 --- a/tests/integration/bundles/kfp_1.8_stable_install.yaml.j2 +++ b/tests/integration/bundles/kfp_1.8_stable_install.yaml.j2 @@ -4,7 +4,6 @@ applications: argo-controller: { charm: ch:argo-controller, channel: 3.3.10/stable, scale: 1, trust: true } metacontroller-operator: { charm: ch:metacontroller-operator, channel: 3.0/stable, scale: 1, trust: true } minio: { charm: ch:minio, channel: ckf-1.8/stable, scale: 1 } - kfp-db: { charm: ch:mysql-k8s, channel: 8.0/stable, scale: 1, constraints: mem=2G, trust: true } mlmd: { charm: ch:mlmd, channel: 1.14/stable, scale: 1 } envoy: { charm: ch:envoy, channel: 2.0/stable, scale: 1 } kubeflow-profiles: { charm: ch:kubeflow-profiles, channel: 1.8/stable, scale: 1, trust: true } @@ -22,6 +21,13 @@ applications: options: default-gateway: kubeflow-gateway trust: true + kfp-db: + charm: mysql-k8s + channel: 8.0/stable + scale: 1 + options: + profile: testing + trust: true kubeflow-roles: charm: kubeflow-roles channel: 1.8/stable diff --git a/tests/integration/bundles/kfp_latest_edge.yaml.j2 b/tests/integration/bundles/kfp_latest_edge.yaml.j2 index 4d4dabc7..bd93e27d 100644 --- a/tests/integration/bundles/kfp_latest_edge.yaml.j2 +++ b/tests/integration/bundles/kfp_latest_edge.yaml.j2 @@ -4,7 +4,6 @@ applications: argo-controller: { charm: ch:argo-controller, channel: latest/edge, scale: 1, trust: true } metacontroller-operator: { charm: ch:metacontroller-operator, channel: latest/edge, scale: 1, trust: true } minio: { charm: ch:minio, channel: latest/edge, scale: 1 } - kfp-db: { charm: ch:mysql-k8s, channel: 8.0/stable, scale: 1, constraints: mem=2G, trust: true } mlmd: { charm: ch:mlmd, channel: latest/edge, scale: 1, trust: true} envoy: { charm: ch:envoy, channel: latest/edge, scale: 1 } kubeflow-profiles: { charm: ch:kubeflow-profiles, channel: latest/edge, scale: 1, trust: true } @@ -22,6 +21,13 @@ applications: options: default-gateway: kubeflow-gateway trust: true + kfp-db: + charm: mysql-k8s + channel: 8.0/stable + scale: 1 + options: + profile: testing + trust: true kubeflow-roles: charm: kubeflow-roles channel: latest/edge From 7d1429d02356fe757e665543fec3d2fa9acf60ff Mon Sep 17 00:00:00 2001 From: Daniela Plascencia Date: Wed, 27 Nov 2024 02:08:03 +0100 Subject: [PATCH 2/5] debug: check disk after --- .github/workflows/integrate.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/integrate.yaml b/.github/workflows/integrate.yaml index 7ade8d1d..5b77a721 100644 --- a/.github/workflows/integrate.yaml +++ b/.github/workflows/integrate.yaml @@ -171,6 +171,10 @@ jobs: juju add-model kubeflow sg snap_microk8s -c "tox -e bundle-integration-${{ matrix.sdk }} -- --model kubeflow --bundle=./tests/integration/bundles/kfp_latest_edge.yaml.j2" --charmcraft-clean + - name: Display df + run: df -h + if: always() + - name: Get all run: kubectl get all -A if: failure() From 8eacbbeac3c68045ced45e2ac09fd772e3e23591 Mon Sep 17 00:00:00 2001 From: Orfeas Kourkakis Date: Wed, 20 Nov 2024 20:12:44 +0200 Subject: [PATCH 3/5] fix(ci): Clean lxc instances on bundle integration tests --- requirements-integration-v2.in | 1 + requirements-integration-v2.txt | 2 ++ tests/README.md | 1 + tests/integration/helpers/lxc.py | 14 ++++++++++++++ tests/integration/test_kfp_functional_v1.py | 4 +++- tests/integration/test_kfp_functional_v2.py | 4 +++- 6 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 tests/integration/helpers/lxc.py diff --git a/requirements-integration-v2.in b/requirements-integration-v2.in index dc095562..1aef6db5 100644 --- a/requirements-integration-v2.in +++ b/requirements-integration-v2.in @@ -12,3 +12,4 @@ pytest-operator pyyaml sh tenacity +jq diff --git a/requirements-integration-v2.txt b/requirements-integration-v2.txt index 1c49ff25..b49dd671 100644 --- a/requirements-integration-v2.txt +++ b/requirements-integration-v2.txt @@ -109,6 +109,8 @@ jedi==0.19.1 # via ipython jinja2==3.1.4 # via pytest-operator +jq==1.8.0 + # via -r requirements-integration-v2.in jsonschema==4.17.3 # via -r requirements-integration-v2.in juju==3.5.2.0 diff --git a/tests/README.md b/tests/README.md index 908f2b5c..474baee6 100644 --- a/tests/README.md +++ b/tests/README.md @@ -20,6 +20,7 @@ This directory has the following structure: │   ├── charmcraft.py │   ├── k8s_resources.py │   └── localize_bundle.py + │   └── lxc.py ├── kfp_globals.py ├── pipelines/ │   └── ... # Sample pipelines diff --git a/tests/integration/helpers/lxc.py b/tests/integration/helpers/lxc.py new file mode 100644 index 00000000..aab0107d --- /dev/null +++ b/tests/integration/helpers/lxc.py @@ -0,0 +1,14 @@ +import sh +import jq + +def clean_charmcraft_lxc_instances() -> None: + """ + Delete lxc instances in project "charmcraft" that are prefixed with "charmcraft-". + + Based on https://discourse.charmhub.io/t/how-to-quickly-clean-unused-lxd-instances-from-charmcraft-pack/15975 + """ + lxc_instances = sh.lxc.list(project="charmcraft", format="json") + lxc_instances_charmcraft = jq.compile('.[] | select(.name | startswith("charmcraft-")) | .name').input_text(lxc_instances).all() + for instance in lxc_instances_charmcraft: + print(f"Deleting lxc instance '{instance}'") + sh.lxc.delete(instance, project="charmcraft") diff --git a/tests/integration/test_kfp_functional_v1.py b/tests/integration/test_kfp_functional_v1.py index 9562e1f0..ce09b7d8 100644 --- a/tests/integration/test_kfp_functional_v1.py +++ b/tests/integration/test_kfp_functional_v1.py @@ -10,6 +10,7 @@ from helpers.k8s_resources import apply_manifests, fetch_response from helpers.localize_bundle import get_resources_from_charm_file from helpers.charmcraft import charmcraft_clean +from helpers.lxc import clean_charmcraft_lxc_instances from kfp_globals import ( CHARM_PATH_TEMPLATE, KFP_CHARMS, @@ -97,7 +98,8 @@ async def test_build_and_deploy(ops_test: OpsTest, request, lightkube_client): context.update([(f"{charm.replace('-', '_')}", charm_file)]) if charmcraft_clean_flag == True: - charmcraft_clean(charms_to_build) + # charmcraft_clean(charms_to_build) + clean_charmcraft_lxc_instances() # Render kfp-operators bundle file with locally built charms and their resources rendered_bundle = render_bundle( diff --git a/tests/integration/test_kfp_functional_v2.py b/tests/integration/test_kfp_functional_v2.py index 8d891425..2e75f0cc 100644 --- a/tests/integration/test_kfp_functional_v2.py +++ b/tests/integration/test_kfp_functional_v2.py @@ -10,6 +10,7 @@ from helpers.k8s_resources import apply_manifests, fetch_response from helpers.localize_bundle import get_resources_from_charm_file from helpers.charmcraft import charmcraft_clean +from helpers.lxc import clean_charmcraft_lxc_instances from kfp_globals import ( CHARM_PATH_TEMPLATE, KFP_CHARMS, @@ -99,7 +100,8 @@ async def test_build_and_deploy(ops_test: OpsTest, request, lightkube_client): context.update([(f"{charm.replace('-', '_')}", charm_file)]) if charmcraft_clean_flag == True: - charmcraft_clean(charms_to_build) + # charmcraft_clean(charms_to_build) + clean_charmcraft_lxc_instances() # Render kfp-operators bundle file with locally built charms and their resources rendered_bundle = render_bundle( From 5547a10aba6e2677136307892f288bfee3e7c960 Mon Sep 17 00:00:00 2001 From: Orfeas Kourkakis Date: Wed, 27 Nov 2024 17:20:38 +0200 Subject: [PATCH 4/5] Revert "fix(ci): Use `juju wait-for` instead wait_for_idle() (#614)" This reverts commit ce22b7588916b23d36091b6e5dfa5e68009cdb08. --- requirements-integration-v1.in | 1 - requirements-integration-v1.txt | 2 -- tests/integration/test_kfp_functional_v1.py | 16 ++++++++++------ tests/integration/test_kfp_functional_v2.py | 16 +++++++++++----- 4 files changed, 21 insertions(+), 14 deletions(-) diff --git a/requirements-integration-v1.in b/requirements-integration-v1.in index b4bd6e73..619721a9 100644 --- a/requirements-integration-v1.in +++ b/requirements-integration-v1.in @@ -11,4 +11,3 @@ pytest pytest-operator pyyaml tenacity -sh diff --git a/requirements-integration-v1.txt b/requirements-integration-v1.txt index 5ba4a321..25f9623b 100644 --- a/requirements-integration-v1.txt +++ b/requirements-integration-v1.txt @@ -272,8 +272,6 @@ rich==13.7.1 # via typer rsa==4.9 # via google-auth -sh==2.1.0 - # via -r requirements-integration-v1.in shellingham==1.5.4 # via typer six==1.16.0 diff --git a/tests/integration/test_kfp_functional_v1.py b/tests/integration/test_kfp_functional_v1.py index ce09b7d8..3e7e394c 100644 --- a/tests/integration/test_kfp_functional_v1.py +++ b/tests/integration/test_kfp_functional_v1.py @@ -20,7 +20,6 @@ SAMPLE_VIEWER, ) -import sh import kfp import lightkube import pytest @@ -109,11 +108,16 @@ async def test_build_and_deploy(ops_test: OpsTest, request, lightkube_client): # Deploy the kfp-operators bundle from the rendered bundle file await deploy_bundle(ops_test, bundle_path=rendered_bundle, trust=True) - # Use `juju wait-for` instead of `wait_for_idle()` - # due to https://github.com/canonical/kfp-operators/issues/601 - # and https://github.com/juju/python-libjuju/issues/1204 - log.info("Waiting on model applications to be active") - sh.juju("wait-for","model","kubeflow", query="forEach(applications, app => app.status == 'active')", timeout="30m") + # Wait for everything to be up. Note, at time of writing these charms would naturally go + # into blocked during deploy while waiting for each other to satisfy relations, so we don't + # raise_on_blocked. + await ops_test.model.wait_for_idle( + status="active", + raise_on_blocked=False, # These apps block while waiting for each other to deploy/relate + raise_on_error=True, + timeout=3600, + idle_period=30, + ) # ---- KFP API Server focused test cases diff --git a/tests/integration/test_kfp_functional_v2.py b/tests/integration/test_kfp_functional_v2.py index 2e75f0cc..dd7a08e9 100644 --- a/tests/integration/test_kfp_functional_v2.py +++ b/tests/integration/test_kfp_functional_v2.py @@ -111,11 +111,17 @@ async def test_build_and_deploy(ops_test: OpsTest, request, lightkube_client): # Deploy the kfp-operators bundle from the rendered bundle file await deploy_bundle(ops_test, bundle_path=rendered_bundle, trust=True) - # Use `juju wait-for` instead of `wait_for_idle()` - # due to https://github.com/canonical/kfp-operators/issues/601 - # and https://github.com/juju/python-libjuju/issues/1204 - log.info("Waiting on model applications to be active") - sh.juju("wait-for","model","kubeflow", query="forEach(applications, app => app.status == 'active')", timeout="30m") + # Wait for everything to be up. Note, at time of writing these charms would naturally go + # into blocked during deploy while waiting for each other to satisfy relations, so we don't + # raise_on_blocked. + await ops_test.model.wait_for_idle( + status="active", + raise_on_blocked=False, # These apps block while waiting for each other to deploy/relate + raise_on_error=True, + timeout=3600, + idle_period=30, + ) + # ---- KFP API Server focused test cases async def test_upload_pipeline(kfp_client): From 2bdf314c70510bdd001e1e3f2fbbd82a328794b7 Mon Sep 17 00:00:00 2001 From: Orfeas Kourkakis Date: Wed, 27 Nov 2024 17:26:02 +0200 Subject: [PATCH 5/5] fix requirements --- requirements-integration-v1.in | 2 ++ requirements-integration-v1.txt | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/requirements-integration-v1.in b/requirements-integration-v1.in index 619721a9..62eb5fb5 100644 --- a/requirements-integration-v1.in +++ b/requirements-integration-v1.in @@ -11,3 +11,5 @@ pytest pytest-operator pyyaml tenacity +sh +jq diff --git a/requirements-integration-v1.txt b/requirements-integration-v1.txt index 25f9623b..b7de1809 100644 --- a/requirements-integration-v1.txt +++ b/requirements-integration-v1.txt @@ -12,8 +12,6 @@ aiosignal==1.3.1 # via aiohttp anyio==4.4.0 # via httpx -appnope==0.1.4 - # via ipython asttokens==2.4.1 # via stack-data async-timeout==4.0.3 @@ -130,6 +128,8 @@ jedi==0.19.1 # via ipython jinja2==3.1.4 # via pytest-operator +jq==1.8.0 + # via -r requirements-integration-v1.in jsonschema==4.17.3 # via # -r requirements-integration-v1.in @@ -272,6 +272,8 @@ rich==13.7.1 # via typer rsa==4.9 # via google-auth +sh==2.1.0 + # via -r requirements-integration-v1.in shellingham==1.5.4 # via typer six==1.16.0