Add arm64

scripts/pre-commit-hadolint.sh

@@ -1,8 +1,13 @@
 #!/bin/bash
 set -e -x -o pipefail
 
-URL="https://github.com/hadolint/hadolint/releases/download/v2.12.0/hadolint-Linux-x86_64"
-CHECKSUM="56de6d5e5ec427e17b74fa48d51271c7fc0d61244bf5c90e828aab8362d55010"
+[[ "$(uname -m)" == "aarch64" ]] && ARCH="arm64" || ARCH="$(uname -m)"
+URL="https://github.com/hadolint/hadolint/releases/download/v2.12.0/hadolint-Linux-$ARCH"
+declare -A CHECKSUMS=(
+  ["x86_64"]="56de6d5e5ec427e17b74fa48d51271c7fc0d61244bf5c90e828aab8362d55010"
+  ["arm64"]="5798551bf19f33951881f15eb238f90aef023f11e7ec7e9f4c37961cb87c5df6"
+)
+CHECKSUM=${CHECKSUMS[$ARCH]}
 HADOLINT=~/.cache/orion/hadolint
 
 function checksum () {

services/langfuzz/service.yaml

@@ -1 +1,4 @@
 name: langfuzz
+arch:
+  - amd64
+  - arm64

services/orion-builder/service.yaml

@@ -1,4 +1,7 @@
 name: orion-builder
+arch:
+  - amd64
+  - arm64
 tests:
   - name: lint
     type: tox

services/orion-builder/setup.cfg

@@ -20,6 +20,7 @@ python_requires = >=3.8
 [options.entry_points]
 console_scripts =
     build = orion_builder.build:main
+    combine = orion_builder.combine:main
     push = orion_builder.push:main
     local-registry = orion_builder.stage_deps:registry_main
 

services/orion-builder/src/orion_builder/build.py

@@ -9,6 +9,7 @@
 import sys
 from os import getenv
 from pathlib import Path
+from platform import machine
 from shutil import rmtree
 from typing import List, Optional
 
@@ -33,6 +34,11 @@ def __init__(self) -> None:
             default=getenv("ARCHIVE_PATH"),
             help="Path to the image tar to output (default: ARCHIVE_PATH)",
         )
+        self.parser.add_argument(
+            "--arch",
+            default=getenv("ARCH", "amd64"),
+            help="Architecture for the image build",
+        )
         self.parser.add_argument(
             "--build-tool",
             default=getenv("BUILD_TOOL"),
@@ -53,7 +59,7 @@ def __init__(self) -> None:
         self.parser.add_argument(
             "--image",
             default=getenv("IMAGE_NAME"),
-            help="Docker image name (without repository, default: IMAGE_NAME)",
+            help="Docker image name (without registry, default: IMAGE_NAME)",
         )
         self.parser.add_argument(
             "--load-deps",
@@ -86,6 +92,9 @@ def sanity_check(self, args: argparse.Namespace) -> None:
         if args.write is None:
             self.parser.error("--output (or ARCHIVE_PATH) is required!")
 
+        if args.arch is None:
+            self.parser.error("--arch (or ARCH) is required!")
+
         if args.build_tool is None:
             self.parser.error("--build-tool (or BUILD_TOOL) is required!")
 
@@ -104,6 +113,10 @@ def sanity_check(self, args: argparse.Namespace) -> None:
 def main(argv: Optional[List[str]] = None) -> None:
     """Build entrypoint. Does not return."""
     args = BuildArgs.parse_args(argv)
+    base_tag = "latest"
+    arch = {"x86_64": "amd64", "aarch64": "arm64"}.get(machine(), machine())
+    assert arch == args.arch
+    args.tag = [f"{args.git_revision}-{arch}", base_tag, f"{base_tag}-{arch}"]
     configure_logging(level=args.log_level)
     target = Target(args)
     if args.load_deps:

services/orion-builder/src/orion_builder/combine.py

@@ -0,0 +1,160 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+"""CLI for Orion builder/combine script"""
+
+
+import argparse
+import logging
+import sys
+from os import getenv
+from pathlib import Path
+from shutil import rmtree
+from subprocess import PIPE
+from tempfile import mkdtemp
+from typing import List, Optional
+
+import taskcluster
+from taskboot.config import Configuration
+from taskboot.docker import Docker, Podman
+from taskboot.utils import download_artifact, load_artifacts, zstd_compress
+from yaml import safe_load as yaml_load
+
+from .cli import CommonArgs, configure_logging
+
+LOG = logging.getLogger(__name__)
+
+
+class CombineArgs(CommonArgs):
+    """CLI arguments for Orion combiner"""
+
+    def __init__(self) -> None:
+        super().__init__()
+        self.parser.add_argument(
+            "--output",
+            "-o",
+            dest="write",
+            default=getenv("ARCHIVE_PATH"),
+            help="Path to the image tar to output (default: ARCHIVE_PATH)",
+        )
+        self.parser.add_argument(
+            "--build-tool",
+            default=getenv("BUILD_TOOL"),
+            help="Tool for combining builds into multiarch image (default: BUILD_TOOL)",
+            choices={"podman", "docker"},
+        )
+        self.parser.add_argument(
+            "--archs",
+            action="append",
+            type=yaml_load,
+            default=getenv("ARCHS", ["amd64"]),
+            help="Architectures to be included in the multiarch image",
+        )
+        self.parser.add_argument(
+            "--service-name",
+            action="append",
+            default=getenv("SERVICE_NAME"),
+            help="Name of the service of the multiarch image",
+        )
+        self.parser.add_argument(
+            "--image",
+            default=getenv("IMAGE_NAME"),
+            help="Docker image name (without repository, default: IMAGE_NAME)",
+        )
+        self.parser.add_argument(
+            "--registry",
+            default=getenv("REGISTRY", "docker.io"),
+            help="Docker registry to use in images tags (default: docker.io)",
+        )
+        self.parser.set_defaults(
+            build_arg=[],
+            cache=str(Path.home() / ".local" / "share"),
+            push=False,
+        )
+
+    def sanity_check(self, args: argparse.Namespace) -> None:
+        super().sanity_check(args)
+        args.tag = [args.git_revision, "latest"]
+
+        if args.write is None:
+            self.parser.error("--output (or ARCHIVE_PATH) is required!")
+
+        if args.build_tool is None:
+            self.parser.error("--build-tool (or BUILD_TOOL) is required!")
+
+        if args.archs is None:
+            self.parser.error("--archs is required!")
+
+        if args.service_name is None:
+            self.parser.error("--service-name is required!")
+
+        if args.image is None:
+            self.parser.error("--image (or IMAGE_NAME) is required!")
+
+
+def main(argv: Optional[List[str]] = None) -> None:
+    """Combine entrypoint. Does not return."""
+
+    args = CombineArgs.parse_args(argv)
+    configure_logging(level=args.log_level)
+
+    service_name = args.service_name
+    archs = args.archs
+    base_tag = "latest"
+
+    config = Configuration(argparse.Namespace(secret=None, config=None))
+    queue = taskcluster.Queue(config.get_taskcluster_options())
+    LOG.info(f"Starting the task to combine {service_name} images for archs: {archs}")
+
+    if args.build_tool == "docker":
+        tool = Docker()
+    elif args.build_tool == "podman":
+        tool = Podman()
+    else:
+        raise ValueError(f"Unsupported build tool: {args.build_tool}")
+
+    # retrieve image archives from dependency tasks to /images
+    image_path = Path(mkdtemp(prefix="image-deps-"))
+    try:
+        existing_images = tool.list_images()
+        LOG.debug("Existing images before loading: %s", existing_images)
+
+        artifacts_ids = load_artifacts(args.task_id, queue, "public/**.tar.zst")
+        for task_id, artifact_name in artifacts_ids:
+            img = download_artifact(queue, task_id, artifact_name, image_path)
+            LOG.info(
+                "Task %s artifact %s downloaded to: %s", task_id, artifact_name, img
+            )
+            # load images into the podman image store
+            load_result = tool.run(
+                [
+                    "load",
+                    "--input",
+                    str(img),
+                ],
+                text=True,
+                stdout=PIPE,
+            )
+            LOG.info(f"Loaded: {load_result}")
+
+        existing_images = tool.list_images()
+        LOG.debug("Existing images after loading: %s", existing_images)
+        assert all(
+            f"{base_tag}-{arch}" in [image["tag"] for image in existing_images]
+            for arch in archs
+        ), "Could not find scheduled archs in local tags"
+
+        # save loaded images into a single multiarch .tar
+        save_result = tool.run(
+            ["save", "--multi-image-archive"]
+            + [
+                f"{args.registry}/mozillasecurity/{service_name}:{base_tag}-{arch}"
+                for arch in archs
+            ]
+            + ["--output", f"{args.write}"]
+        )
+        LOG.info(f"Save multiarch image result: {save_result}")
+        zstd_compress(args.write)
+    finally:
+        rmtree(image_path)
+    sys.exit(0)