Cleanup base sources (pt. 1)

Signed-off-by: Christian Berendt <[email protected]>

files/cleanup-base-sources.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+for e in $(find /*-base-source -name '*config-generator*'); do
+    if [[ -d $e ]]; then
+        mkdir -p /x/$e
+        cp -r $e/* /x/$e
+    else
+        f=$(dirname $e)
+        mkdir -p /x/$f
+        cp -r $f/* /x/$f
+    fi
+done
+
+rm -rf /*-base-source
+mv /x/* /
+rm -rf /x

scripts/004-build.sh

@@ -42,6 +42,9 @@ done
 
 # Build & squash base images
 
+# The line can be commented out for tests to build keystone images only.
+KOLLA_IMAGES_BASE=^keystone-base
+
 kolla-build \
   --template-override templates/$OPENSTACK_VERSION/template-overrides.j2 \
   --config-file $KOLLA_CONF \
@@ -52,6 +55,9 @@ kolla-build \
 
 # Build images
 
+# The line can be commented out for tests to build keystone images only.
+KOLLA_IMAGES=^keystone
+
 kolla-build \
   --template-override templates/$OPENSTACK_VERSION/template-overrides.j2 \
   --config-file $KOLLA_CONF \
@@ -65,4 +71,10 @@ if grep -q "Failed with status: error" kolla-build-$BUILD_ID.log; then
     exit 1
 fi
 
+# Cleanup images
+
+python3 src/cleanup-images.py
+
+# List images
+
 docker images

src/cleanup-images.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+
+import os
+import shutil
+import subprocess
+import tempfile
+
+from docker import APIClient, DockerClient
+from loguru import logger
+
+VERSION = os.environ.get("OPENSTACK_VERSION", "zed")
+FILTERS = {"label": f"de.osism.release.openstack={VERSION}"}
+
+SUPPORTED_IMAGES = ["keystone"]
+
+client = DockerClient()
+client_api = APIClient()
+
+for image in client.images.list(filters=FILTERS):
+    # skip images without a tag
+    if not image.tags:
+        continue
+
+    tag = image.tags[0]
+
+    # Check if the image is supported
+    if not [x for x in SUPPORTED_IMAGES if x in image.labels["name"]]:
+        logger.info(f"Image {tag} not supported")
+        continue
+
+    logger.info(f"Cleaning up image {tag}")
+
+    # 0: "sha256:59c56aee1fb4dbaeb334aef06088b49902105d1ea0c15a9e5a2a9ce560fa4c5d" <-- root
+    # 1: "sha256:295ebbacd98b98d173f5fba0371e4fee8bbdf9ff0ad2676670ddb2b521482004" <-- ubuntu:22.04
+    # 2: "sha256:bd9f5e1f11d685268d42d7bf3a7850fdd7b6767bfcfbdb1bcc0b31227dae8f91" <-- openstack-base
+    # 3: "sha256:6ae01f1fd7a8c29837ddc7ed50e78ba70666f5967bdaf8438f07da2971d3d024" <-- service-base
+    # 4: "sha256:591d1c8305854fabc586786673d87f55eb142f0efed7a002f696c6a1c4c26880" <-- service
+
+    inspect = client_api.inspect_image(tag)
+    service_base_layer = inspect["RootFS"]["Layers"][3]
+
+    with tempfile.TemporaryDirectory() as d:
+        shutil.copy("files/cleanup-base-sources.sh", d)
+
+        dockerfile = os.path.join(d, "Dockerfile")
+        logger.info(f"Generating Dockerfile in {d}")
+
+        # Get current username
+        result = client.containers.run(tag, "bash -c 'whoami'")
+        username = result.decode("utf-8")
+
+        with open(dockerfile, "w+") as fp:
+            fp.write(f"FROM {tag}\n")
+
+            # full source directory required in the bifrost image
+            if "bifrost" not in tag:
+                fp.write("COPY cleanup-base-sources.sh /cleanup-base-sources.sh\n")
+                fp.write("USER root\n")
+                fp.write("RUN bash /cleanup-base-sources.sh\n")
+                fp.write("RUN rm /cleanup-base-sources.sh\n")
+                fp.write(f"USER {username}\n")
+
+        client.images.build(path=d, tag=f"{tag}-after-cleanup")
+
+        # Squash the image
+        logger.info(f"Squashing with service-base layer {service_base_layer}")
+        cmd = f"docker-squash --tag {tag}-after-squash --from-layer {service_base_layer} {tag}-after-cleanup"
+        subprocess.check_output(cmds, stderr=subprocess.STDOUT)
+
+        # Re-tag the image
+        client.images.remove(tag)
+        client.images.tag(f"{tag}-after-squash", tag)