bump rocks from 0.16.0 to 0.17.0 (#44)

* bump rocks from 0.16.0 to 0.17.0 * Updated rocks * fixed tests * rolled back tests fixes * fixed tests
canonical · Jul 29, 2024 · 92cd6d9 · 92cd6d9
1 parent 0b77988
commit 92cd6d9
Show file tree

Hide file tree

Showing 36 changed files with 1,337 additions and 57 deletions.
diff --git a/earlystopping-medianstop/rockcraft.yaml b/earlystopping-medianstop/rockcraft.yaml
@@ -1,7 +1,7 @@
-# Based on https://github.com/kubeflow/katib/blob/v0.16.0/cmd/earlystopping/medianstop/v1beta1/Dockerfile
+# Based on https://github.com/kubeflow/katib/blob/v0.17.0/cmd/earlystopping/medianstop/v1beta1/Dockerfile
 name: earlystopping-medianstop
 base: [email protected]
-version: 'v0.16.0-22.04-2'
+version: 'v0.17.0'
 summary: Early stopping algorithm for katib.
 description: |
     The median stopping rule stops a pending trial X at step S if the trial’s best 
@@ -26,7 +26,7 @@ parts:
     requirements-and-deps:
       plugin: python
       source: https://github.com/kubeflow/katib.git
-      source-tag: "v0.16.0"
+      source-tag: "v0.17.0"
       source-depth: 1
       source-type: git
       source-subdir: cmd/earlystopping/medianstop/v1beta1
@@ -45,7 +45,7 @@ parts:
     earlystopping-medianstop:
       plugin: dump
       source: https://github.com/kubeflow/katib.git
-      source-tag: "v0.16.0"
+      source-tag: "v0.17.0"
       source-depth: 1
       source-type: git
       organize:

diff --git a/earlystopping-medianstop/tests/test_rock.py b/earlystopping-medianstop/tests/test_rock.py
@@ -0,0 +1,56 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+#
+#
+
+from pathlib import Path
+
+import os
+import logging
+import random
+import pytest
+import string
+import subprocess
+import yaml
+
+from charmed_kubeflow_chisme.rock import CheckRock
+
+
+@pytest.fixture()
+def rock_test_env(tmpdir):
+    """Yields a temporary directory and random docker container name, then cleans them up after."""
+    container_name = "".join(
+        [str(i) for i in random.choices(string.ascii_lowercase, k=8)]
+    )
+    yield tmpdir, container_name
+
+    try:
+        subprocess.run(["docker", "rm", container_name])
+    except Exception:
+        pass
+    # tmpdir fixture we use here should clean up the other files for us
+
+
+@pytest.mark.abort_on_fail
+def test_rock(rock_test_env):
+    """Test rock."""
+    temp_dir, container_name = rock_test_env
+    check_rock = CheckRock("rockcraft.yaml")
+    rock_image = check_rock.get_name()
+    rock_version = check_rock.get_version()
+    LOCAL_ROCK_IMAGE = f"{rock_image}:{rock_version}"
+
+    subprocess.run(
+        [
+            "docker",
+            "run",
+            "--rm",
+            LOCAL_ROCK_IMAGE,
+            "exec",
+            "ls",
+            "-la",
+            "/opt/katib/cmd/earlystopping/medianstop/v1beta1/main.py",
+        ],
+        check=True,
+    )
+
diff --git a/earlystopping-medianstop/tox.ini b/earlystopping-medianstop/tox.ini
@@ -0,0 +1,51 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+[tox]
+skipsdist = True
+skip_missing_interpreters = True
+envlist = pack, export-to-docker, sanity, integration
+
+[testenv]
+setenv =
+    PYTHONPATH={toxinidir}
+    PYTHONBREAKPOINT=ipdb.set_trace
+
+[testenv:pack]
+passenv = *
+allowlist_externals =
+    rockcraft
+commands =
+    rockcraft pack
+
+[testenv:export-to-docker]
+passenv = *
+allowlist_externals =
+    bash
+    skopeo
+    yq
+commands =
+    # export rock to docker
+    bash -c 'NAME=$(yq eval .name rockcraft.yaml) && \
+             VERSION=$(yq eval .version rockcraft.yaml) && \
+             ARCH=$(yq eval ".platforms | keys | .[0]" rockcraft.yaml) && \
+             ROCK="$\{NAME\}_$\{VERSION\}_$\{ARCH\}.rock" && \
+             DOCKER_IMAGE=$NAME:$VERSION && \
+             echo "Exporting $ROCK to docker as $DOCKER_IMAGE" && \
+             skopeo --insecure-policy copy oci-archive:$ROCK docker-daemon:$DOCKER_IMAGE'
+
+[testenv:sanity]
+passenv = *
+deps =
+    pytest
+    charmed-kubeflow-chisme
+commands =
+    # run rock tests
+    pytest -s -v --tb native --show-capture=all --log-cli-level=INFO {posargs} {toxinidir}/tests
+
+[testenv:integration]
+passenv = *
+allowlist_externals =
+    echo
+commands =
+    # TODO: Implement integration tests here
+    echo "WARNING: This is a placeholder test - no test is implemented here."
diff --git a/file-metrics-collector/rockcraft.yaml b/file-metrics-collector/rockcraft.yaml
@@ -1,12 +1,11 @@
-# Based on https://github.com/kubeflow/katib/blob/v0.16.0/cmd/metricscollector/v1beta1/file-metricscollector/Dockerfile
+# Based on https://github.com/kubeflow/katib/blob/v0.17.0/cmd/metricscollector/v1beta1/file-metricscollector/Dockerfile
 name: file-metrics-collector
 summary: Metrics collector for file info for Katib.
 description: |
   Collects metrics from specified file.
-version: v0.16.0-22.04-2
+version: v0.17.0
 license: Apache-2.0
-build-base: [email protected]
-base: bare
+base: [email protected]
 run-user: _daemon_
 services:
   file-metrics-collector:
@@ -22,7 +21,7 @@ parts:
     plugin: go
     source: https://github.com/kubeflow/katib
     source-type: git
-    source-tag: v0.16.0
+    source-tag: v0.17.0
     build-snaps:
       - go
     stage-packages:
@@ -41,8 +40,7 @@ parts:
   security-team-requirement:
     plugin: nil
     override-build: |
-      # security requirement
-      # there are no packages installed in `bare` base which is used in this rock
       mkdir -p ${CRAFT_PART_INSTALL}/usr/share/rocks
-      (echo "# os-release" && cat /etc/os-release && echo "# dpkg-query") \
-       > ${CRAFT_PART_INSTALL}/usr/share/rocks/dpkg.query
+      (echo "# os-release" && cat /etc/os-release && echo "# dpkg-query" && \
+      dpkg-query --root=${CRAFT_PROJECT_DIR}/../bundles/ubuntu-22.04/rootfs/ -f '${db:Status-Abbrev},${binary:Package},${Version},${source:Package},${Source:Version}\n' -W) \
+      > ${CRAFT_PART_INSTALL}/usr/share/rocks/dpkg.query
diff --git a/file-metrics-collector/tests/test_rock.py b/file-metrics-collector/tests/test_rock.py
@@ -0,0 +1,56 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+#
+#
+
+from pathlib import Path
+
+import os
+import logging
+import random
+import pytest
+import string
+import subprocess
+import yaml
+
+from charmed_kubeflow_chisme.rock import CheckRock
+
+
+@pytest.fixture()
+def rock_test_env(tmpdir):
+    """Yields a temporary directory and random docker container name, then cleans them up after."""
+    container_name = "".join(
+        [str(i) for i in random.choices(string.ascii_lowercase, k=8)]
+    )
+    yield tmpdir, container_name
+
+    try:
+        subprocess.run(["docker", "rm", container_name])
+    except Exception:
+        pass
+    # tmpdir fixture we use here should clean up the other files for us
+
+
+@pytest.mark.abort_on_fail
+def test_rock(rock_test_env):
+    """Test rock."""
+    temp_dir, container_name = rock_test_env
+    check_rock = CheckRock("rockcraft.yaml")
+    rock_image = check_rock.get_name()
+    rock_version = check_rock.get_version()
+    LOCAL_ROCK_IMAGE = f"{rock_image}:{rock_version}"
+
+    subprocess.run(
+        [
+            "docker",
+            "run",
+            "--rm",
+            LOCAL_ROCK_IMAGE,
+            "exec",
+            "ls",
+            "-la",
+            "/app/file-metricscollector",
+        ],
+        check=True,
+    )
+
diff --git a/file-metrics-collector/tox.ini b/file-metrics-collector/tox.ini
@@ -0,0 +1,51 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+[tox]
+skipsdist = True
+skip_missing_interpreters = True
+envlist = pack, export-to-docker, sanity, integration
+
+[testenv]
+setenv =
+    PYTHONPATH={toxinidir}
+    PYTHONBREAKPOINT=ipdb.set_trace
+
+[testenv:pack]
+passenv = *
+allowlist_externals =
+    rockcraft
+commands =
+    rockcraft pack
+
+[testenv:export-to-docker]
+passenv = *
+allowlist_externals =
+    bash
+    skopeo
+    yq
+commands =
+    # export rock to docker
+    bash -c 'NAME=$(yq eval .name rockcraft.yaml) && \
+             VERSION=$(yq eval .version rockcraft.yaml) && \
+             ARCH=$(yq eval ".platforms | keys | .[0]" rockcraft.yaml) && \
+             ROCK="$\{NAME\}_$\{VERSION\}_$\{ARCH\}.rock" && \
+             DOCKER_IMAGE=$NAME:$VERSION && \
+             echo "Exporting $ROCK to docker as $DOCKER_IMAGE" && \
+             skopeo --insecure-policy copy oci-archive:$ROCK docker-daemon:$DOCKER_IMAGE'
+
+[testenv:sanity]
+passenv = *
+deps =
+    pytest
+    charmed-kubeflow-chisme
+commands =
+    # run rock tests
+    pytest -s -v --tb native --show-capture=all --log-cli-level=INFO {posargs} {toxinidir}/tests
+
+[testenv:integration]
+passenv = *
+allowlist_externals =
+    echo
+commands =
+    # TODO: Implement integration tests here
+    echo "WARNING: This is a placeholder test - no test is implemented here."
diff --git a/katib-db-manager/rockcraft.yaml b/katib-db-manager/rockcraft.yaml
@@ -1,12 +1,11 @@
-# Based on https://github.com/kubeflow/katib/blob/v0.16.0/cmd/db-manager/v1beta1/Dockerfile
+# Based on https://github.com/kubeflow/katib/blob/v0.17.0/cmd/db-manager/v1beta1/Dockerfile
 name: katib-db-manager
 summary: Katib DB Controller
 description: |
   Katib database manager.
-version: v0.16.0-22.04-2
+version: v0.17.0
 license: Apache-2.0
-build-base: [email protected]
-base: bare
+base: [email protected]
 run-user: _daemon_
 services:
   katib-db-manager:
@@ -23,7 +22,7 @@ parts:
     plugin: go
     source: https://github.com/kubeflow/katib
     source-type: git
-    source-tag: v0.16.0
+    source-tag: v0.17.0
     build-snaps:
       - go
     stage-packages:
@@ -54,8 +53,7 @@ parts:
   security-team-requirement:
     plugin: nil
     override-build: |
-      # security requirement
-      # there are no packages installed in `bare` base which is used in this rock
       mkdir -p ${CRAFT_PART_INSTALL}/usr/share/rocks
-      (echo "# os-release" && cat /etc/os-release && echo "# dpkg-query") \
-       > ${CRAFT_PART_INSTALL}/usr/share/rocks/dpkg.query
+      (echo "# os-release" && cat /etc/os-release && echo "# dpkg-query" && \
+      dpkg-query --root=${CRAFT_PROJECT_DIR}/../bundles/ubuntu-22.04/rootfs/ -f '${db:Status-Abbrev},${binary:Package},${Version},${source:Package},${Source:Version}\n' -W) \
+      > ${CRAFT_PART_INSTALL}/usr/share/rocks/dpkg.query
diff --git a/katib-db-manager/tests/test_rock.py b/katib-db-manager/tests/test_rock.py
@@ -0,0 +1,56 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+#
+#
+
+from pathlib import Path
+
+import os
+import logging
+import random
+import pytest
+import string
+import subprocess
+import yaml
+
+from charmed_kubeflow_chisme.rock import CheckRock
+
+
+@pytest.fixture()
+def rock_test_env(tmpdir):
+    """Yields a temporary directory and random docker container name, then cleans them up after."""
+    container_name = "".join(
+        [str(i) for i in random.choices(string.ascii_lowercase, k=8)]
+    )
+    yield tmpdir, container_name
+
+    try:
+        subprocess.run(["docker", "rm", container_name])
+    except Exception:
+        pass
+    # tmpdir fixture we use here should clean up the other files for us
+
+
+@pytest.mark.abort_on_fail
+def test_rock(rock_test_env):
+    """Test rock."""
+    temp_dir, container_name = rock_test_env
+    check_rock = CheckRock("rockcraft.yaml")
+    rock_image = check_rock.get_name()
+    rock_version = check_rock.get_version()
+    LOCAL_ROCK_IMAGE = f"{rock_image}:{rock_version}"
+
+    subprocess.run(
+        [
+            "docker",
+            "run",
+            "--rm",
+            LOCAL_ROCK_IMAGE,
+            "exec",
+            "ls",
+            "-la",
+            "/app/katib-db-manager",
+        ],
+        check=True,
+    )
+