Adds Whereabouts integration test

Whereabouts depends on Multus, which means that the integration
test is deploying Multus as well.

requirements-dev.txt

@@ -0,0 +1,5 @@
+black==24.3.0
+codespell==2.2.4
+flake8==6.0.0
+isort==5.12.0
+licenseheaders==0.8.8

requirements-test.txt

@@ -0,0 +1,4 @@
+coverage[toml]==7.2.5
+pytest==7.3.1
+PyYAML==6.0.1
+tenacity==8.2.3

tests/integration/conftest.py

@@ -0,0 +1,136 @@
+#
+# Copyright 2024 Canonical, Ltd.
+# See LICENSE file for licensing details
+#
+import logging
+from pathlib import Path
+from typing import Generator, List
+
+import pytest
+from test_util import config, harness, util
+
+LOG = logging.getLogger(__name__)
+
+
+def _harness_clean(h: harness.Harness):
+    "Clean up created instances within the test harness."
+
+    if config.SKIP_CLEANUP:
+        LOG.warning(
+            "Skipping harness cleanup. "
+            "It is your job now to clean up cloud resources"
+        )
+    else:
+        LOG.debug("Cleanup")
+        h.cleanup()
+
+
+@pytest.fixture(scope="session")
+def h() -> harness.Harness:
+    LOG.debug("Create harness for %s", config.SUBSTRATE)
+    if config.SUBSTRATE == "local":
+        h = harness.LocalHarness()
+    elif config.SUBSTRATE == "lxd":
+        h = harness.LXDHarness()
+    elif config.SUBSTRATE == "multipass":
+        h = harness.MultipassHarness()
+    elif config.SUBSTRATE == "juju":
+        h = harness.JujuHarness()
+    else:
+        raise harness.HarnessError(
+            "TEST_SUBSTRATE must be one of: local, lxd, multipass, juju"
+        )
+
+    yield h
+
+    _harness_clean(h)
+
+
+def pytest_configure(config):
+    config.addinivalue_line(
+        "markers",
+        "node_count: Mark a test to specify how many instance nodes need to be created\n"
+        "disable_k8s_bootstrapping: By default, the first k8s node is bootstrapped. This marker disables that.",
+    )
+
+
+@pytest.fixture(scope="function")
+def node_count(request) -> int:
+    node_count_marker = request.node.get_closest_marker("node_count")
+    if not node_count_marker:
+        return 1
+    node_count_arg, *_ = node_count_marker.args
+    return int(node_count_arg)
+
+
+@pytest.fixture(scope="function")
+def disable_k8s_bootstrapping(request) -> int:
+    return bool(request.node.get_closest_marker("disable_k8s_bootstrapping"))
+
+
+@pytest.fixture(scope="function")
+def instances(
+    h: harness.Harness, node_count: int, tmp_path: Path, disable_k8s_bootstrapping: bool
+) -> Generator[List[harness.Instance], None, None]:
+    """Construct instances for a cluster.
+
+    Bootstrap and setup networking on the first instance, if `disable_k8s_bootstrapping` marker is not set.
+    """
+    if not config.SNAP_CHANNEL:
+        pytest.fail("Set TEST_SNAP_CHANNEL to the channel of the k8s snap to install.")
+
+    if node_count <= 0:
+        pytest.xfail("Test requested 0 or fewer instances, skip this test.")
+
+    LOG.info(f"Creating {node_count} instances")
+    instances: List[harness.Instance] = []
+
+    for _ in range(node_count):
+        # Create <node_count> instances and setup the k8s snap in each.
+        instance = h.new_instance()
+        instances.append(instance)
+        util.setup_k8s_snap(instance)
+
+    if not disable_k8s_bootstrapping:
+        first_node, *_ = instances
+        first_node.exec(["k8s", "bootstrap"])
+
+    yield instances
+
+    if config.SKIP_CLEANUP:
+        LOG.warning("Skipping clean-up of instances, delete them on your own")
+        return
+
+    # Cleanup after each test.
+    # We cannot execute _harness_clean() here as this would also
+    # remove the session_instance. The harness ensures that everything is cleaned up
+    # at the end of the test session.
+    for instance in instances:
+        h.delete_instance(instance.id)
+
+
+@pytest.fixture(scope="session")
+def session_instance(
+    h: harness.Harness, tmp_path_factory: pytest.TempPathFactory
+) -> Generator[harness.Instance, None, None]:
+    """Constructs and bootstraps an instance that persists over a test session.
+
+    Bootstraps the instance with all k8sd features enabled to reduce testing time.
+    """
+    LOG.info("Setup node and enable all features")
+
+    instance = h.new_instance()
+    util.setup_k8s_snap(instance)
+
+    bootstrap_config_path = "/home/ubuntu/bootstrap-session.yaml"
+    instance.send_file(
+        (config.MANIFESTS_DIR / "bootstrap-session.yaml").as_posix(),
+        bootstrap_config_path,
+    )
+
+    instance.exec(["k8s", "bootstrap", "--file", bootstrap_config_path])
+    util.wait_until_k8s_ready(instance, [instance])
+    util.wait_for_network(instance)
+    util.wait_for_dns(instance)
+
+    yield instance

tests/integration/test_util/config.py

@@ -0,0 +1,64 @@
+#
+# Copyright 2024 Canonical, Ltd.
+# See LICENSE file for licensing details
+#
+import os
+from pathlib import Path
+
+DIR = Path(__file__).absolute().parent
+
+MANIFESTS_DIR = DIR / ".." / ".." / "templates"
+
+# SNAP is the absolute path to the snap against which we run the integration tests.
+SNAP_CHANNEL = os.getenv("TEST_SNAP_CHANNEL")
+
+# SUBSTRATE is the substrate to use for running the integration tests.
+# One of 'local' (default), 'lxd', 'juju', or 'multipass'.
+SUBSTRATE = os.getenv("TEST_SUBSTRATE") or "local"
+
+# SKIP_CLEANUP can be used to prevent machines to be automatically destroyed
+# after the tests complete.
+SKIP_CLEANUP = (os.getenv("TEST_SKIP_CLEANUP") or "") == "1"
+
+# LXD_PROFILE_NAME is the profile name to use for LXD containers.
+LXD_PROFILE_NAME = os.getenv("TEST_LXD_PROFILE_NAME") or "k8s-integration"
+
+# LXD_PROFILE is the profile to use for LXD containers.
+LXD_PROFILE = (
+    os.getenv("TEST_LXD_PROFILE")
+    or (DIR / ".." / ".." / "lxd-profile.yaml").read_text()
+)
+
+# LXD_IMAGE is the image to use for LXD containers.
+LXD_IMAGE = os.getenv("TEST_LXD_IMAGE") or "ubuntu:22.04"
+
+# LXD_SIDELOAD_IMAGES_DIR is an optional directory with OCI images from the host
+# that will be mounted at /var/snap/k8s/common/images on the LXD containers.
+LXD_SIDELOAD_IMAGES_DIR = os.getenv("TEST_LXD_SIDELOAD_IMAGES_DIR") or ""
+
+# MULTIPASS_IMAGE is the image to use for Multipass VMs.
+MULTIPASS_IMAGE = os.getenv("TEST_MULTIPASS_IMAGE") or "22.04"
+
+# MULTIPASS_CPUS is the number of cpus for Multipass VMs.
+MULTIPASS_CPUS = os.getenv("TEST_MULTIPASS_CPUS") or "2"
+
+# MULTIPASS_MEMORY is the memory for Multipass VMs.
+MULTIPASS_MEMORY = os.getenv("TEST_MULTIPASS_MEMORY") or "2G"
+
+# MULTIPASS_DISK is the disk size for Multipass VMs.
+MULTIPASS_DISK = os.getenv("TEST_MULTIPASS_DISK") or "10G"
+
+# JUJU_MODEL is the Juju model to use.
+JUJU_MODEL = os.getenv("TEST_JUJU_MODEL")
+
+# JUJU_CONTROLLER is the Juju controller to use.
+JUJU_CONTROLLER = os.getenv("TEST_JUJU_CONTROLLER")
+
+# JUJU_CONSTRAINTS is the constraints to use when creating Juju machines.
+JUJU_CONSTRAINTS = os.getenv("TEST_JUJU_CONSTRAINTS", "mem=4G cores=2 root-disk=20G")
+
+# JUJU_BASE is the base OS to use when creating Juju machines.
+JUJU_BASE = os.getenv("TEST_JUJU_BASE") or "[email protected]"
+
+# JUJU_MACHINES is a list of existing Juju machines to use.
+JUJU_MACHINES = os.getenv("TEST_JUJU_MACHINES") or ""

tests/integration/test_util/harness/__init__.py

@@ -0,0 +1,19 @@
+#
+# Copyright 2024 Canonical, Ltd.
+# See LICENSE file for licensing details
+#
+from test_util.harness.base import Harness, HarnessError, Instance
+from test_util.harness.juju import JujuHarness
+from test_util.harness.local import LocalHarness
+from test_util.harness.lxd import LXDHarness
+from test_util.harness.multipass import MultipassHarness
+
+__all__ = [
+    HarnessError,
+    Harness,
+    Instance,
+    JujuHarness,
+    LocalHarness,
+    LXDHarness,
+    MultipassHarness,
+]

tests/integration/test_util/harness/base.py

@@ -0,0 +1,107 @@
+#
+# Copyright 2024 Canonical, Ltd.
+# See LICENSE file for licensing details
+#
+import subprocess
+from functools import partial
+
+
+class HarnessError(Exception):
+    """Base error for all our harness failures"""
+
+    pass
+
+
+class Instance:
+    """Reference to a harness and a given instance id.
+
+    Provides convenience methods for an instance to call its harness' methods
+    """
+
+    def __init__(self, h: "Harness", id: str) -> None:
+        self._h = h
+        self._id = id
+
+        self.send_file = partial(h.send_file, id)
+        self.pull_file = partial(h.pull_file, id)
+        self.exec = partial(h.exec, id)
+        self.delete_instance = partial(h.delete_instance, id)
+
+    @property
+    def id(self) -> str:
+        return self._id
+
+    def __str__(self) -> str:
+        return f"{self._h.name}:{self.id}"
+
+
+class Harness:
+    """Abstract how integration tests can start and manage multiple machines. This allows
+    writing integration tests that can run on the local machine, LXD, or Multipass with minimum
+    effort.
+    """
+
+    name: str
+
+    def new_instance(self) -> Instance:
+        """Creates a new instance on the infrastructure and returns an object
+        which can be used to interact with it.
+
+        If the operation fails, a HarnessError is raised.
+        """
+        raise NotImplementedError
+
+    def send_file(self, instance_id: str, source: str, destination: str):
+        """Send a local file to the instance.
+
+        :param instance_id: The instance_id, as returned by new_instance()
+        :param source: Path to the file that will be copied to the instance
+        :param destination: Path in the instance where the file will be copied.
+                                 This must always be an absolute path.
+
+
+        If the operation fails, a HarnessError is raised.
+        """
+        raise NotImplementedError
+
+    def pull_file(self, instance_id: str, source: str, destination: str):
+        """Pull a file from the instance and save it on the local machine
+
+        :param instance_id: The instance_id, as returned by new_instance()
+        :param source: Path to the file that will be copied from the instance.
+                            This must always be an absolute path.
+        :param destination: Path on the local machine the file will be saved.
+
+        If the operation fails, a HarnessError is raised.
+        """
+        raise NotImplementedError
+
+    def exec(
+        self, instance_id: str, command: list, **kwargs
+    ) -> subprocess.CompletedProcess:
+        """Run a command as root on the instance.
+
+        :param instance_id: The instance_id, as returned by new_instance()
+        :param command: Command for subprocess.run()
+        :param kwargs: Keyword args compatible with subprocess.run()
+
+        If the operation fails, a subprocesss.CalledProcessError is raised.
+        """
+        raise NotImplementedError
+
+    def delete_instance(self, instance_id: str):
+        """Delete a previously created instance.
+
+        :param instance_id: The instance_id, as returned by new_instance()
+
+        If the operation fails, a HarnessError is raised.
+        """
+        raise NotImplementedError
+
+    def cleanup(self):
+        """Delete any leftover resources after the tests are done, e.g. delete any
+        instances that might still be running.
+
+        If the operation fails, a HarnessError is raised.
+        """
+        raise NotImplementedError