From 47688422b47ea5ed2266b874dbf51ec37a58414a Mon Sep 17 00:00:00 2001 From: Maciek Golaszewski Date: Thu, 18 Jul 2024 08:53:28 +0200 Subject: [PATCH] integration tests KU-1063 --- tests/.copyright.tmpl | 2 + tests/integration/conftest.py | 136 ++++++++++ tests/integration/test_util/config.py | 64 +++++ .../integration/test_util/harness/__init__.py | 19 ++ tests/integration/test_util/harness/base.py | 107 ++++++++ tests/integration/test_util/harness/juju.py | 204 +++++++++++++++ tests/integration/test_util/harness/local.py | 78 ++++++ tests/integration/test_util/harness/lxd.py | 182 +++++++++++++ .../test_util/harness/multipass.py | 135 ++++++++++ tests/integration/test_util/util.py | 247 ++++++++++++++++++ tests/sanity/test_rock.py | 7 +- tests/templates/httpbin.yaml | 39 +++ tests/tox.ini | 2 +- 13 files changed, 1219 insertions(+), 3 deletions(-) create mode 100644 tests/.copyright.tmpl create mode 100644 tests/integration/conftest.py create mode 100644 tests/integration/test_util/config.py create mode 100644 tests/integration/test_util/harness/__init__.py create mode 100644 tests/integration/test_util/harness/base.py create mode 100644 tests/integration/test_util/harness/juju.py create mode 100644 tests/integration/test_util/harness/local.py create mode 100644 tests/integration/test_util/harness/lxd.py create mode 100644 tests/integration/test_util/harness/multipass.py create mode 100644 tests/integration/test_util/util.py create mode 100644 tests/templates/httpbin.yaml diff --git a/tests/.copyright.tmpl b/tests/.copyright.tmpl new file mode 100644 index 0000000..1eb2340 --- /dev/null +++ b/tests/.copyright.tmpl @@ -0,0 +1,2 @@ +Copyright ${years} ${owner}. +See LICENSE file for licensing details diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py new file mode 100644 index 0000000..64461aa --- /dev/null +++ b/tests/integration/conftest.py @@ -0,0 +1,136 @@ +# +# Copyright 2024 Canonical, Ltd. +# See LICENSE file for licensing details +# +import logging +from pathlib import Path +from typing import Generator, List + +import pytest +from test_util import config, harness, util + +LOG = logging.getLogger(__name__) + + +def _harness_clean(h: harness.Harness): + "Clean up created instances within the test harness." + + if config.SKIP_CLEANUP: + LOG.warning( + "Skipping harness cleanup. " + "It is your job now to clean up cloud resources" + ) + else: + LOG.debug("Cleanup") + h.cleanup() + + +@pytest.fixture(scope="session") +def h() -> harness.Harness: + LOG.debug("Create harness for %s", config.SUBSTRATE) + if config.SUBSTRATE == "local": + h = harness.LocalHarness() + elif config.SUBSTRATE == "lxd": + h = harness.LXDHarness() + elif config.SUBSTRATE == "multipass": + h = harness.MultipassHarness() + elif config.SUBSTRATE == "juju": + h = harness.JujuHarness() + else: + raise harness.HarnessError( + "TEST_SUBSTRATE must be one of: local, lxd, multipass, juju" + ) + + yield h + + _harness_clean(h) + + +def pytest_configure(config): + config.addinivalue_line( + "markers", + "node_count: Mark a test to specify how many instance nodes need to be created\n" + "disable_k8s_bootstrapping: By default, the first k8s node is bootstrapped. This marker disables that.", + ) + + +@pytest.fixture(scope="function") +def node_count(request) -> int: + node_count_marker = request.node.get_closest_marker("node_count") + if not node_count_marker: + return 1 + node_count_arg, *_ = node_count_marker.args + return int(node_count_arg) + + +@pytest.fixture(scope="function") +def disable_k8s_bootstrapping(request) -> int: + return bool(request.node.get_closest_marker("disable_k8s_bootstrapping")) + + +@pytest.fixture(scope="function") +def instances( + h: harness.Harness, node_count: int, tmp_path: Path, disable_k8s_bootstrapping: bool +) -> Generator[List[harness.Instance], None, None]: + """Construct instances for a cluster. + + Bootstrap and setup networking on the first instance, if `disable_k8s_bootstrapping` marker is not set. + """ + if not config.SNAP_CHANNEL: + pytest.fail("Set TEST_SNAP_CHANNEL to the channel of the k8s snap to install.") + + if node_count <= 0: + pytest.xfail("Test requested 0 or fewer instances, skip this test.") + + LOG.info(f"Creating {node_count} instances") + instances: List[harness.Instance] = [] + + for _ in range(node_count): + # Create instances and setup the k8s snap in each. + instance = h.new_instance() + instances.append(instance) + util.setup_k8s_snap(instance) + + if not disable_k8s_bootstrapping: + first_node, *_ = instances + first_node.exec(["k8s", "bootstrap"]) + + yield instances + + if config.SKIP_CLEANUP: + LOG.warning("Skipping clean-up of instances, delete them on your own") + return + + # Cleanup after each test. + # We cannot execute _harness_clean() here as this would also + # remove the session_instance. The harness ensures that everything is cleaned up + # at the end of the test session. + for instance in instances: + h.delete_instance(instance.id) + + +@pytest.fixture(scope="session") +def session_instance( + h: harness.Harness, tmp_path_factory: pytest.TempPathFactory +) -> Generator[harness.Instance, None, None]: + """Constructs and bootstraps an instance that persists over a test session. + + Bootstraps the instance with all k8sd features enabled to reduce testing time. + """ + LOG.info("Setup node and enable all features") + + instance = h.new_instance() + util.setup_k8s_snap(instance) + + bootstrap_config_path = "/home/ubuntu/bootstrap-session.yaml" + instance.send_file( + (config.MANIFESTS_DIR / "bootstrap-session.yaml").as_posix(), + bootstrap_config_path, + ) + + instance.exec(["k8s", "bootstrap", "--file", bootstrap_config_path]) + util.wait_until_k8s_ready(instance, [instance]) + util.wait_for_network(instance) + util.wait_for_dns(instance) + + yield instance diff --git a/tests/integration/test_util/config.py b/tests/integration/test_util/config.py new file mode 100644 index 0000000..82a7384 --- /dev/null +++ b/tests/integration/test_util/config.py @@ -0,0 +1,64 @@ +# +# Copyright 2024 Canonical, Ltd. +# See LICENSE file for licensing details +# +import os +from pathlib import Path + +DIR = Path(__file__).absolute().parent + +MANIFESTS_DIR = DIR / ".." / ".." / "templates" + +# SNAP is the absolute path to the snap against which we run the integration tests. +SNAP_CHANNEL = os.getenv("TEST_SNAP_CHANNEL") + +# SUBSTRATE is the substrate to use for running the integration tests. +# One of 'local' (default), 'lxd', 'juju', or 'multipass'. +SUBSTRATE = os.getenv("TEST_SUBSTRATE") or "local" + +# SKIP_CLEANUP can be used to prevent machines to be automatically destroyed +# after the tests complete. +SKIP_CLEANUP = (os.getenv("TEST_SKIP_CLEANUP") or "") == "1" + +# LXD_PROFILE_NAME is the profile name to use for LXD containers. +LXD_PROFILE_NAME = os.getenv("TEST_LXD_PROFILE_NAME") or "k8s-integration" + +# LXD_PROFILE is the profile to use for LXD containers. +LXD_PROFILE = ( + os.getenv("TEST_LXD_PROFILE") + or (DIR / ".." / ".." / "lxd-profile.yaml").read_text() +) + +# LXD_IMAGE is the image to use for LXD containers. +LXD_IMAGE = os.getenv("TEST_LXD_IMAGE") or "ubuntu:22.04" + +# LXD_SIDELOAD_IMAGES_DIR is an optional directory with OCI images from the host +# that will be mounted at /var/snap/k8s/common/images on the LXD containers. +LXD_SIDELOAD_IMAGES_DIR = os.getenv("TEST_LXD_SIDELOAD_IMAGES_DIR") or "" + +# MULTIPASS_IMAGE is the image to use for Multipass VMs. +MULTIPASS_IMAGE = os.getenv("TEST_MULTIPASS_IMAGE") or "22.04" + +# MULTIPASS_CPUS is the number of cpus for Multipass VMs. +MULTIPASS_CPUS = os.getenv("TEST_MULTIPASS_CPUS") or "2" + +# MULTIPASS_MEMORY is the memory for Multipass VMs. +MULTIPASS_MEMORY = os.getenv("TEST_MULTIPASS_MEMORY") or "2G" + +# MULTIPASS_DISK is the disk size for Multipass VMs. +MULTIPASS_DISK = os.getenv("TEST_MULTIPASS_DISK") or "10G" + +# JUJU_MODEL is the Juju model to use. +JUJU_MODEL = os.getenv("TEST_JUJU_MODEL") + +# JUJU_CONTROLLER is the Juju controller to use. +JUJU_CONTROLLER = os.getenv("TEST_JUJU_CONTROLLER") + +# JUJU_CONSTRAINTS is the constraints to use when creating Juju machines. +JUJU_CONSTRAINTS = os.getenv("TEST_JUJU_CONSTRAINTS", "mem=4G cores=2 root-disk=20G") + +# JUJU_BASE is the base OS to use when creating Juju machines. +JUJU_BASE = os.getenv("TEST_JUJU_BASE") or "ubuntu@22.04" + +# JUJU_MACHINES is a list of existing Juju machines to use. +JUJU_MACHINES = os.getenv("TEST_JUJU_MACHINES") or "" diff --git a/tests/integration/test_util/harness/__init__.py b/tests/integration/test_util/harness/__init__.py new file mode 100644 index 0000000..b952e76 --- /dev/null +++ b/tests/integration/test_util/harness/__init__.py @@ -0,0 +1,19 @@ +# +# Copyright 2024 Canonical, Ltd. +# See LICENSE file for licensing details +# +from test_util.harness.base import Harness, HarnessError, Instance +from test_util.harness.juju import JujuHarness +from test_util.harness.local import LocalHarness +from test_util.harness.lxd import LXDHarness +from test_util.harness.multipass import MultipassHarness + +__all__ = [ + HarnessError, + Harness, + Instance, + JujuHarness, + LocalHarness, + LXDHarness, + MultipassHarness, +] diff --git a/tests/integration/test_util/harness/base.py b/tests/integration/test_util/harness/base.py new file mode 100644 index 0000000..f94bc10 --- /dev/null +++ b/tests/integration/test_util/harness/base.py @@ -0,0 +1,107 @@ +# +# Copyright 2024 Canonical, Ltd. +# See LICENSE file for licensing details +# +import subprocess +from functools import partial + + +class HarnessError(Exception): + """Base error for all our harness failures""" + + pass + + +class Instance: + """Reference to a harness and a given instance id. + + Provides convenience methods for an instance to call its harness' methods + """ + + def __init__(self, h: "Harness", id: str) -> None: + self._h = h + self._id = id + + self.send_file = partial(h.send_file, id) + self.pull_file = partial(h.pull_file, id) + self.exec = partial(h.exec, id) + self.delete_instance = partial(h.delete_instance, id) + + @property + def id(self) -> str: + return self._id + + def __str__(self) -> str: + return f"{self._h.name}:{self.id}" + + +class Harness: + """Abstract how integration tests can start and manage multiple machines. This allows + writing integration tests that can run on the local machine, LXD, or Multipass with minimum + effort. + """ + + name: str + + def new_instance(self) -> Instance: + """Creates a new instance on the infrastructure and returns an object + which can be used to interact with it. + + If the operation fails, a HarnessError is raised. + """ + raise NotImplementedError + + def send_file(self, instance_id: str, source: str, destination: str): + """Send a local file to the instance. + + :param instance_id: The instance_id, as returned by new_instance() + :param source: Path to the file that will be copied to the instance + :param destination: Path in the instance where the file will be copied. + This must always be an absolute path. + + + If the operation fails, a HarnessError is raised. + """ + raise NotImplementedError + + def pull_file(self, instance_id: str, source: str, destination: str): + """Pull a file from the instance and save it on the local machine + + :param instance_id: The instance_id, as returned by new_instance() + :param source: Path to the file that will be copied from the instance. + This must always be an absolute path. + :param destination: Path on the local machine the file will be saved. + + If the operation fails, a HarnessError is raised. + """ + raise NotImplementedError + + def exec( + self, instance_id: str, command: list, **kwargs + ) -> subprocess.CompletedProcess: + """Run a command as root on the instance. + + :param instance_id: The instance_id, as returned by new_instance() + :param command: Command for subprocess.run() + :param kwargs: Keyword args compatible with subprocess.run() + + If the operation fails, a subprocesss.CalledProcessError is raised. + """ + raise NotImplementedError + + def delete_instance(self, instance_id: str): + """Delete a previously created instance. + + :param instance_id: The instance_id, as returned by new_instance() + + If the operation fails, a HarnessError is raised. + """ + raise NotImplementedError + + def cleanup(self): + """Delete any leftover resources after the tests are done, e.g. delete any + instances that might still be running. + + If the operation fails, a HarnessError is raised. + """ + raise NotImplementedError diff --git a/tests/integration/test_util/harness/juju.py b/tests/integration/test_util/harness/juju.py new file mode 100644 index 0000000..a2e7b9d --- /dev/null +++ b/tests/integration/test_util/harness/juju.py @@ -0,0 +1,204 @@ +# +# Copyright 2024 Canonical, Ltd. +# See LICENSE file for licensing details +# +import json +import logging +import shlex +import subprocess +from pathlib import Path + +from test_util import config +from test_util.harness import Harness, HarnessError, Instance +from test_util.util import run + +LOG = logging.getLogger(__name__) + + +class JujuHarness(Harness): + """A Harness that creates an Juju machine for each instance.""" + + name = "juju" + + def __init__(self): + super(JujuHarness, self).__init__() + + self.model = config.JUJU_MODEL + if not self.model: + raise HarnessError("Set JUJU_MODEL to the Juju model to use") + + if config.JUJU_CONTROLLER: + self.model = f"{config.JUJU_CONTROLLER}:{self.model}" + + self.constraints = config.JUJU_CONSTRAINTS + self.base = config.JUJU_BASE + self.existing_machines = {} + self.instances = set() + + if config.JUJU_MACHINES: + self.existing_machines = { + instance_id.strip(): False + for instance_id in config.JUJU_MACHINES.split() + } + LOG.debug( + "Configured Juju substrate (model %s, machines %s)", + self.model, + config.JUJU_MACHINES, + ) + + else: + LOG.debug( + "Configured Juju substrate (model %s, base %s, constraints %s)", + self.model, + self.base, + self.constraints, + ) + + def new_instance(self) -> Instance: + for instance_id in self.existing_machines: + if not self.existing_machines[instance_id]: + LOG.debug("Reusing existing machine %s", instance_id) + self.existing_machines[instance_id] = True + self.instances.add(instance_id) + return Instance(self, instance_id) + + LOG.debug("Creating instance with constraints %s", self.constraints) + try: + p = run( + [ + "juju", + "add-machine", + "-m", + self.model, + "--constraints", + self.constraints, + "--base", + self.base, + ], + capture_output=True, + ) + + output = p.stderr.decode().strip() + if not output.startswith("created machine "): + raise HarnessError(f"failed to parse output from juju add-machine {p=}") + + instance_id = output.split(" ")[2] + except subprocess.CalledProcessError as e: + raise HarnessError("Failed to create Juju machine") from e + + self.instances.add(instance_id) + + self.exec(instance_id, ["snap", "wait", "system", "seed.loaded"]) + return Instance(self, instance_id) + + def send_file(self, instance_id: str, source: str, destination: str): + if instance_id not in self.instances: + raise HarnessError(f"unknown instance {instance_id}") + + if not Path(destination).is_absolute(): + raise HarnessError(f"path {destination} must be absolute") + + LOG.debug( + "Copying file %s to instance %s at %s", source, instance_id, destination + ) + try: + self.exec( + instance_id, + ["mkdir", "-m=0777", "-p", Path(destination).parent.as_posix()], + ) + run(["juju", "scp", source, f"{instance_id}:{destination}"]) + except subprocess.CalledProcessError as e: + raise HarnessError("juju scp command failed") from e + + def pull_file(self, instance_id: str, source: str, destination: str): + if instance_id not in self.instances: + raise HarnessError(f"unknown instance {instance_id}") + + if not Path(source).is_absolute(): + raise HarnessError(f"path {source} must be absolute") + + LOG.debug( + "Copying file %s from instance %s to %s", source, instance_id, destination + ) + try: + run(["juju", "scp", f"{instance_id}:{source}", destination]) + except subprocess.CalledProcessError as e: + raise HarnessError("juju scp command failed") from e + + def exec(self, instance_id: str, command: list, **kwargs): + if instance_id not in self.instances: + raise HarnessError(f"unknown instance {instance_id}") + + LOG.debug("Execute command %s in instance %s", command, instance_id) + capture_output = kwargs.pop("capture_output", False) + check = kwargs.pop("check", True) + stdout = kwargs.pop("stdout", None) + stderr = kwargs.pop("stderr", None) + input = f" < Instance: + if self.initialized: + raise HarnessError("local substrate only supports up to one instance") + + self.initialized = True + LOG.debug("Initializing instance") + try: + self.exec(self.hostname, ["snap", "wait", "system", "seed.loaded"]) + except subprocess.CalledProcessError as e: + raise HarnessError("failed to wait for snapd seed") from e + + return Instance(self, self.hostname) + + def send_file(self, _: str, source: str, destination: str): + if not self.initialized: + raise HarnessError("no instance initialized") + + if not Path(destination).is_absolute(): + raise HarnessError(f"path {destination} must be absolute") + + LOG.debug("Copying file %s to %s", source, destination) + try: + self.exec( + _, ["mkdir", "-m=0777", "-p", Path(destination).parent.as_posix()] + ) + shutil.copy(source, destination) + except subprocess.CalledProcessError as e: + raise HarnessError("failed to copy file") from e + except shutil.SameFileError: + pass + + def pull_file(self, _: str, source: str, destination: str): + return self.send_file(_, destination, source) + + def exec(self, _: str, command: list, **kwargs): + if not self.initialized: + raise HarnessError("no instance initialized") + + LOG.debug("Executing command %s on %s", command, self.hostname) + return run(["sudo", "-E", "bash", "-c", shlex.join(command)], **kwargs) + + def delete_instance(self, _: str): + LOG.debug("Stopping instance") + self.initialized = False + + def cleanup(self): + LOG.debug("Stopping instance") + self.initialized = False diff --git a/tests/integration/test_util/harness/lxd.py b/tests/integration/test_util/harness/lxd.py new file mode 100644 index 0000000..4693350 --- /dev/null +++ b/tests/integration/test_util/harness/lxd.py @@ -0,0 +1,182 @@ +# +# Copyright 2024 Canonical, Ltd. +# See LICENSE file for licensing details +# +import logging +import os +import shlex +import subprocess +from pathlib import Path + +from test_util import config +from test_util.harness import Harness, HarnessError, Instance +from test_util.util import run, stubbornly + +LOG = logging.getLogger(__name__) + + +class LXDHarness(Harness): + """A Harness that creates an LXD container for each instance.""" + + name = "lxd" + + def next_id(self) -> int: + self._next_id += 1 + return self._next_id + + def __init__(self): + super(LXDHarness, self).__init__() + + self._next_id = 0 + + self.profile = config.LXD_PROFILE_NAME + self.sideload_images_dir = config.LXD_SIDELOAD_IMAGES_DIR + self.image = config.LXD_IMAGE + self.instances = set() + + LOG.debug("Checking for LXD profile %s", self.profile) + try: + run(["lxc", "profile", "show", self.profile]) + except subprocess.CalledProcessError: + try: + LOG.debug("Creating LXD profile %s", self.profile) + run(["lxc", "profile", "create", self.profile]) + + except subprocess.CalledProcessError as e: + raise HarnessError( + f"Failed to create LXD profile {self.profile}" + ) from e + + try: + LOG.debug("Configuring LXD profile %s", self.profile) + run( + ["lxc", "profile", "edit", self.profile], + input=config.LXD_PROFILE.encode(), + ) + except subprocess.CalledProcessError as e: + raise HarnessError(f"Failed to configure LXD profile {self.profile}") from e + + LOG.debug( + "Configured LXD substrate (profile %s, image %s)", self.profile, self.image + ) + + def new_instance(self) -> Instance: + instance_id = os.getenv( + "ROCK_OVERWRITE_INSTANCE_NAME", + f"k8s-integration-{os.urandom(3).hex()}-{self.next_id()}", + ) + LOG.debug("Creating instance %s with image %s", instance_id, self.image) + try: + stubbornly(retries=3, delay_s=1).exec( + [ + "lxc", + "launch", + self.image, + instance_id, + "-p", + "default", + "-p", + self.profile, + ] + ) + self.instances.add(instance_id) + + if self.sideload_images_dir: + stubbornly(retries=3, delay_s=1).exec( + [ + "lxc", + "config", + "device", + "add", + instance_id, + "k8s-e2e-images", + "disk", + f"source={self.sideload_images_dir}", + "path=/mnt/images", + "readonly=true", + ] + ) + + self.exec( + instance_id, + ["mkdir", "-p", "/var/snap/k8s/common"], + ) + self.exec( + instance_id, + ["cp", "-rv", "/mnt/images", "/var/snap/k8s/common/images"], + ) + except subprocess.CalledProcessError as e: + raise HarnessError(f"Failed to create LXD container {instance_id}") from e + + self.exec(instance_id, ["snap", "wait", "system", "seed.loaded"]) + return Instance(self, instance_id) + + def send_file(self, instance_id: str, source: str, destination: str): + if instance_id not in self.instances: + raise HarnessError(f"unknown instance {instance_id}") + + if not Path(destination).is_absolute(): + raise HarnessError(f"path {destination} must be absolute") + + LOG.debug( + "Copying file %s to instance %s at %s", source, instance_id, destination + ) + try: + self.exec( + instance_id, + ["mkdir", "-m=0777", "-p", Path(destination).parent.as_posix()], + capture_output=True, + ) + run( + ["lxc", "file", "push", source, f"{instance_id}{destination}"], + capture_output=True, + ) + except subprocess.CalledProcessError as e: + LOG.error("command {e.cmd} failed") + LOG.error(f" {e.returncode=}") + LOG.error(f" {e.stdout.decode()=}") + LOG.error(f" {e.stderr.decode()=}") + raise HarnessError("failed to push file") from e + + def pull_file(self, instance_id: str, source: str, destination: str): + if instance_id not in self.instances: + raise HarnessError(f"unknown instance {instance_id}") + + if not Path(source).is_absolute(): + raise HarnessError(f"path {source} must be absolute") + + LOG.debug( + "Copying file %s from instance %s to %s", source, instance_id, destination + ) + try: + run( + ["lxc", "file", "pull", f"{instance_id}{source}", destination], + stdout=subprocess.DEVNULL, + ) + except subprocess.CalledProcessError as e: + raise HarnessError("lxc file push command failed") from e + + def exec(self, instance_id: str, command: list, **kwargs): + if instance_id not in self.instances: + raise HarnessError(f"unknown instance {instance_id}") + + LOG.debug("Execute command %s in instance %s", " ".join(command), instance_id) + return run( + ["lxc", "shell", instance_id, "--", "bash", "-c", shlex.join(command)], + **kwargs, + ) + + def delete_instance(self, instance_id: str): + if instance_id not in self.instances: + raise HarnessError(f"unknown instance {instance_id}") + + try: + run(["lxc", "rm", instance_id, "--force"]) + except subprocess.CalledProcessError as e: + raise HarnessError(f"failed to delete instance {instance_id}") from e + + self.instances.discard(instance_id) + + def cleanup(self): + for instance_id in self.instances.copy(): + self.delete_instance(instance_id) diff --git a/tests/integration/test_util/harness/multipass.py b/tests/integration/test_util/harness/multipass.py new file mode 100644 index 0000000..b555c67 --- /dev/null +++ b/tests/integration/test_util/harness/multipass.py @@ -0,0 +1,135 @@ +# +# Copyright 2024 Canonical, Ltd. +# See LICENSE file for licensing details +# +import logging +import os +import shlex +import subprocess +from pathlib import Path + +from test_util import config +from test_util.harness import Harness, HarnessError, Instance +from test_util.util import run + +LOG = logging.getLogger(__name__) + + +class MultipassHarness(Harness): + """A Harness that creates a Multipass VM for each instance.""" + + name = "multipass" + + def next_id(self) -> int: + self._next_id += 1 + return self._next_id + + def __init__(self): + super(MultipassHarness, self).__init__() + + self._next_id = 0 + + self.image = config.MULTIPASS_IMAGE + self.cpus = config.MULTIPASS_CPUS + self.memory = config.MULTIPASS_MEMORY + self.disk = config.MULTIPASS_DISK + self.instances = set() + + LOG.debug("Configured Multipass substrate (image %s)", self.image) + + def new_instance(self) -> Instance: + instance_id = f"k8s-integration-{os.urandom(3).hex()}-{self.next_id()}" + + LOG.debug("Creating instance %s with image %s", instance_id, self.image) + try: + run( + [ + "multipass", + "launch", + self.image, + "--name", + instance_id, + "--cpus", + self.cpus, + "--memory", + self.memory, + "--disk", + self.disk, + ] + ) + except subprocess.CalledProcessError as e: + raise HarnessError(f"Failed to create multipass VM {instance_id}") from e + + self.instances.add(instance_id) + + self.exec(instance_id, ["snap", "wait", "system", "seed.loaded"]) + return Instance(self, instance_id) + + def send_file(self, instance_id: str, source: str, destination: str): + if instance_id not in self.instances: + raise HarnessError(f"unknown instance {instance_id}") + + if not Path(destination).is_absolute(): + raise HarnessError(f"path {destination} must be absolute") + + LOG.debug( + "Copying file %s to instance %s at %s", source, instance_id, destination + ) + try: + self.exec( + instance_id, + ["mkdir", "-m=0777", "-p", Path(destination).parent.as_posix()], + ) + run(["multipass", "transfer", source, f"{instance_id}:{destination}"]) + except subprocess.CalledProcessError as e: + raise HarnessError("lxc file push command failed") from e + + def pull_file(self, instance_id: str, source: str, destination: str): + if instance_id not in self.instances: + raise HarnessError(f"unknown instance {instance_id}") + + if not Path(source).is_absolute(): + raise HarnessError(f"path {source} must be absolute") + + LOG.debug( + "Copying file %s from instance %s to %s", source, instance_id, destination + ) + try: + run(["multipass", "transfer", f"{instance_id}:{source}", destination]) + except subprocess.CalledProcessError as e: + raise HarnessError("lxc file push command failed") from e + + def exec(self, instance_id: str, command: list, **kwargs): + if instance_id not in self.instances: + raise HarnessError(f"unknown instance {instance_id}") + + LOG.debug("Execute command %s in instance %s", command, instance_id) + return run( + [ + "multipass", + "exec", + instance_id, + "--", + "sudo", + "bash", + "-c", + shlex.join(command), + ], + **kwargs, + ) + + def delete_instance(self, instance_id: str): + if instance_id not in self.instances: + raise HarnessError(f"unknown instance {instance_id}") + + try: + run(["multipass", "delete", instance_id]) + run(["multipass", "purge"]) + except subprocess.CalledProcessError as e: + raise HarnessError(f"failed to delete instance {instance_id}") from e + + self.instances.discard(instance_id) + + def cleanup(self): + for instance_id in self.instances.copy(): + self.delete_instance(instance_id) diff --git a/tests/integration/test_util/util.py b/tests/integration/test_util/util.py new file mode 100644 index 0000000..51f31c4 --- /dev/null +++ b/tests/integration/test_util/util.py @@ -0,0 +1,247 @@ +# +# Copyright 2024 Canonical, Ltd. +# See LICENSE file for licensing details +# +import json +import logging +import shlex +import subprocess +from functools import partial +from typing import Any, Callable, List, Optional, Union + +from tenacity import ( + RetryCallState, + retry, + retry_if_exception_type, + stop_after_attempt, + stop_never, + wait_fixed, +) +from test_util import config, harness + +LOG = logging.getLogger(__name__) + + +def run(command: list, **kwargs) -> subprocess.CompletedProcess: + """Log and run command.""" + kwargs.setdefault("check", True) + + LOG.debug("Execute command %s (kwargs=%s)", shlex.join(command), kwargs) + return subprocess.run(command, **kwargs) + + +def stubbornly( + retries: Optional[int] = None, + delay_s: Optional[Union[float, int]] = None, + exceptions: Optional[tuple] = None, + **retry_kds, +): + """ + Retry a command for a while, using tenacity + + By default, retry immediately and forever until no exceptions occur. + + Some commands need to execute until they pass some condition + > stubbornly(*retry_args).until(*some_condition).exec(*some_command) + + Some commands need to execute until they complete + > stubbornly(*retry_args).exec(*some_command) + + : param retries int: convenience param to use stop=retry.stop_after_attempt() + : param delay_s float|int: convenience param to use wait=retry.wait_fixed(delay_s) + : param exceptions Tuple[Exception]: convenience param to use retry=retry.retry_if_exception_type(exceptions) + : param retry_kds Mapping: direct interface to all tenacity arguments for retrying + """ + + def _before_sleep(retry_state: RetryCallState): + attempt = retry_state.attempt_number + tries = f"/{retries}" if retries is not None else "" + LOG.info( + f"Attempt {attempt}{tries} failed. Error: {retry_state.outcome.exception()}" + ) + LOG.info(f"Retrying in {delay_s} seconds...") + + _waits = wait_fixed(delay_s) if delay_s is not None else wait_fixed(0) + _stops = stop_after_attempt(retries) if retries is not None else stop_never + _exceptions = exceptions or (Exception,) # default to retry on all exceptions + + _retry_args = dict( + wait=_waits, + stop=_stops, + retry=retry_if_exception_type(_exceptions), + before_sleep=_before_sleep, + ) + # Permit any tenacity retry overrides from these ^defaults + _retry_args.update(retry_kds) + + class Retriable: + def __init__(self) -> None: + self._condition = None + self._run = partial(run, capture_output=True) + + @retry(**_retry_args) + def exec( + self, + command_args: List[str], + **command_kwds, + ): + """ + Execute a command against a harness or locally with subprocess to be retried. + + :param List[str] command_args: The command to be executed, as a str or list of str + :param Mapping[str,str] command_kwds: Additional keyword arguments to be passed to exec + """ + + try: + resp = self._run(command_args, **command_kwds) + except subprocess.CalledProcessError as e: + LOG.warning(f" rc={e.returncode}") + LOG.warning(f" stdout={e.stdout.decode()}") + LOG.warning(f" stderr={e.stderr.decode()}") + raise + if self._condition: + assert self._condition(resp), "Failed to meet condition" + return resp + + def on(self, instance: harness.Instance) -> "Retriable": + """ + Target the command at some instance. + + :param instance Instance: Instance on a test harness. + """ + self._run = partial(instance.exec, capture_output=True) + return self + + def until( + self, condition: Callable[[subprocess.CompletedProcess], bool] = None + ) -> "Retriable": + """ + Test the output of the executed command against an expected response + + :param Callable condition: a callable which returns a truth about the command output + """ + self._condition = condition + return self + + return Retriable() + + +# Installs and setups the k8s snap on the given instance and connects the interfaces. +def setup_k8s_snap(instance: harness.Instance): + LOG.info("Install k8s snap") + instance.exec( + ["snap", "install", "k8s", "--classic", "--channel", config.SNAP_CHANNEL] + ) + + +# Validates that the K8s node is in Ready state. +def wait_until_k8s_ready( + control_node: harness.Instance, instances: List[harness.Instance] +): + for instance in instances: + host = hostname(instance) + result = ( + stubbornly(retries=15, delay_s=5) + .on(control_node) + .until(lambda p: " Ready" in p.stdout.decode()) + .exec(["k8s", "kubectl", "get", "node", host, "--no-headers"]) + ) + LOG.info("Kubelet registered successfully!") + LOG.info("%s", result.stdout.decode()) + + +def wait_for_dns(instance: harness.Instance): + LOG.info("Waiting for DNS to be ready") + instance.exec(["k8s", "x-wait-for", "dns"]) + + +def wait_for_network(instance: harness.Instance): + LOG.info("Waiting for network to be ready") + instance.exec(["k8s", "x-wait-for", "network"]) + + +def hostname(instance: harness.Instance) -> str: + """Return the hostname for a given instance.""" + resp = instance.exec(["hostname"], capture_output=True) + return resp.stdout.decode().strip() + + +def get_local_node_status(instance: harness.Instance) -> str: + resp = instance.exec(["k8s", "local-node-status"], capture_output=True) + return resp.stdout.decode().strip() + + +def get_nodes(control_node: harness.Instance) -> List[Any]: + """Get a list of existing nodes. + + Args: + control_node: instance on which to execute check + + Returns: + list of nodes + """ + result = control_node.exec( + ["k8s", "kubectl", "get", "nodes", "-o", "json"], capture_output=True + ) + assert result.returncode == 0, "Failed to get nodes with kubectl" + node_list = json.loads(result.stdout.decode()) + assert node_list["kind"] == "List", "Should have found a list of nodes" + return [node for node in node_list["items"]] + + +def ready_nodes(control_node: harness.Instance) -> List[Any]: + """Get a list of the ready nodes. + + Args: + control_node: instance on which to execute check + + Returns: + list of nodes + """ + return [ + node + for node in get_nodes(control_node) + if all( + condition["status"] == "False" + for condition in node["status"]["conditions"] + if condition["type"] != "Ready" + ) + ] + + +# Create a token to join a node to an existing cluster +def get_join_token( + initial_node: harness.Instance, joining_cplane_node: harness.Instance, *args: str +) -> str: + out = initial_node.exec( + ["k8s", "get-join-token", joining_cplane_node.id, *args], + capture_output=True, + ) + return out.stdout.decode().strip() + + +# Join an existing cluster. +def join_cluster(instance: harness.Instance, join_token: str): + instance.exec(["k8s", "join-cluster", join_token]) + + +def get_default_cidr(instance: harness.Instance, instance_default_ip: str): + # ---- + # 1: lo inet 127.0.0.1/8 scope host lo ..... + # 28: eth0 inet 10.42.254.197/24 metric 100 brd 10.42.254.255 scope global dynamic eth0 .... + # ---- + # Fetching the cidr for the default interface by matching with instance ip from the output + p = instance.exec(["ip", "-o", "-f", "inet", "addr", "show"], capture_output=True) + out = p.stdout.decode().split(" ") + return [i for i in out if instance_default_ip in i][0] + + +def get_default_ip(instance: harness.Instance): + # --- + # default via 10.42.254.1 dev eth0 proto dhcp src 10.42.254.197 metric 100 + # --- + # Fetching the default IP address from the output, e.g. 10.42.254.197 + p = instance.exec( + ["ip", "-o", "-4", "route", "show", "to", "default"], capture_output=True + ) + return p.stdout.decode().split(" ")[8] diff --git a/tests/sanity/test_rock.py b/tests/sanity/test_rock.py index d4cc750..fd3ba17 100644 --- a/tests/sanity/test_rock.py +++ b/tests/sanity/test_rock.py @@ -1,16 +1,19 @@ +# +# Copyright 2024 Canonical, Ltd. +# See LICENSE file for licensing details +# import os import subprocess def test_sanity(): image_variable = "ROCK_CONTOUR" - entrypoint = "/contour" + entrypoint = "contour" image = os.getenv(image_variable) assert image is not None, f"${image_variable} is not set" docker_run = subprocess.run( ["docker", "run", "--rm", "--entrypoint", entrypoint, image, "--help"], - capture_output=True, text=True, ) diff --git a/tests/templates/httpbin.yaml b/tests/templates/httpbin.yaml new file mode 100644 index 0000000..2df7038 --- /dev/null +++ b/tests/templates/httpbin.yaml @@ -0,0 +1,39 @@ +## +## Copyright 2024 Canonical, Ltd. +## See LICENSE file for licensing details +## +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: httpbin + name: httpbin +spec: + replicas: 3 + selector: + matchLabels: + app: httpbin + template: + metadata: + labels: + app: httpbin + spec: + containers: + - image: docker.io/kennethreitz/httpbin + name: httpbin +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: httpbin + name: httpbin +spec: + ports: + - port: 8888 + protocol: TCP + targetPort: 80 + selector: + app: httpbin + sessionAffinity: None + type: ClusterIP diff --git a/tests/tox.ini b/tests/tox.ini index 0c3cde7..fb562f2 100644 --- a/tests/tox.ini +++ b/tests/tox.ini @@ -15,7 +15,7 @@ pass_env = description = Apply coding style standards to code deps = -r {tox_root}/requirements-dev.txt commands = -; licenseheaders -t {tox_root}/.copyright.tmpl -cy -o 'Canonical, Ltd' -d {tox_root} + licenseheaders -t {tox_root}/.copyright.tmpl -cy -o 'Canonical, Ltd' -d {tox_root} isort {tox_root} --profile=black black {tox_root}