From a8fdd6302e8d6a6f2b7210f4fd558767fe407dc7 Mon Sep 17 00:00:00 2001 From: asuleimanov Date: Fri, 20 Sep 2024 04:20:00 +0000 Subject: [PATCH 01/13] [pre-commit] Add arm64 linux package for hadolint (Dockerfile linter) --- scripts/pre-commit-hadolint.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/scripts/pre-commit-hadolint.sh b/scripts/pre-commit-hadolint.sh index b29dca8b..e812858b 100755 --- a/scripts/pre-commit-hadolint.sh +++ b/scripts/pre-commit-hadolint.sh @@ -1,8 +1,13 @@ #!/bin/bash set -e -x -o pipefail -URL="https://github.com/hadolint/hadolint/releases/download/v2.12.0/hadolint-Linux-x86_64" -CHECKSUM="56de6d5e5ec427e17b74fa48d51271c7fc0d61244bf5c90e828aab8362d55010" +[[ "$(uname -m)" == "aarch64" ]] && ARCH="arm64" || ARCH="$(uname -m)" +URL="https://github.com/hadolint/hadolint/releases/download/v2.12.0/hadolint-Linux-$ARCH" +declare -A CHECKSUMS=( + ["x86_64"]="56de6d5e5ec427e17b74fa48d51271c7fc0d61244bf5c90e828aab8362d55010" + ["arm64"]="5798551bf19f33951881f15eb238f90aef023f11e7ec7e9f4c37961cb87c5df6" +) +CHECKSUM=${CHECKSUMS[$ARCH]} HADOLINT=~/.cache/orion/hadolint function checksum () { From 397a4fc5e54fe71b72508dd54bd32b966f88aff6 Mon Sep 17 00:00:00 2001 From: asuleimanov Date: Mon, 23 Sep 2024 13:37:00 -0420 Subject: [PATCH 02/13] [orion-builder] Build arch aware --- services/orion-builder/src/orion_builder/build.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/services/orion-builder/src/orion_builder/build.py b/services/orion-builder/src/orion_builder/build.py index 39093698..25de06d1 100644 --- a/services/orion-builder/src/orion_builder/build.py +++ b/services/orion-builder/src/orion_builder/build.py @@ -9,6 +9,7 @@ import sys from os import getenv from pathlib import Path +from platform import machine from shutil import rmtree from typing import List, Optional @@ -33,6 +34,11 @@ def __init__(self) -> None: default=getenv("ARCHIVE_PATH"), help="Path to the image tar to output (default: ARCHIVE_PATH)", ) + self.parser.add_argument( + "--arch", + default=getenv("ARCH", "amd64"), + help="Architecture for the image build", + ) self.parser.add_argument( "--build-tool", default=getenv("BUILD_TOOL"), @@ -53,7 +59,7 @@ def __init__(self) -> None: self.parser.add_argument( "--image", default=getenv("IMAGE_NAME"), - help="Docker image name (without repository, default: IMAGE_NAME)", + help="Docker image name (without registry, default: IMAGE_NAME)", ) self.parser.add_argument( "--load-deps", @@ -86,6 +92,9 @@ def sanity_check(self, args: argparse.Namespace) -> None: if args.write is None: self.parser.error("--output (or ARCHIVE_PATH) is required!") + if args.arch is None: + self.parser.error("--arch (or ARCH) is required!") + if args.build_tool is None: self.parser.error("--build-tool (or BUILD_TOOL) is required!") @@ -104,6 +113,10 @@ def sanity_check(self, args: argparse.Namespace) -> None: def main(argv: Optional[List[str]] = None) -> None: """Build entrypoint. Does not return.""" args = BuildArgs.parse_args(argv) + base_tag = "latest" + arch = {"x86_64": "amd64", "aarch64": "arm64"}.get(machine(), machine()) + assert arch == args.arch + args.tag = [f"{args.git_revision}-{arch}", base_tag, f"{base_tag}-{arch}"] configure_logging(level=args.log_level) target = Target(args) if args.load_deps: From c8db958106b566d08f1825b8af1ab796f634f2ca Mon Sep 17 00:00:00 2001 From: asuleimanov Date: Thu, 26 Sep 2024 04:20:00 +0000 Subject: [PATCH 03/13] [orion-decision] Schedule builds in parallel for multiarch (arm/amd64) linux services --- .../src/orion_decision/__init__.py | 1 + .../src/orion_decision/orion.py | 18 +++--- .../src/orion_decision/scheduler.py | 63 ++++++++++++------- .../orion_decision/task_templates/build.yaml | 3 +- 4 files changed, 50 insertions(+), 35 deletions(-) diff --git a/services/orion-decision/src/orion_decision/__init__.py b/services/orion-decision/src/orion_decision/__init__.py index 46c6696f..037a8aab 100644 --- a/services/orion-decision/src/orion_decision/__init__.py +++ b/services/orion-decision/src/orion_decision/__init__.py @@ -22,6 +22,7 @@ PROVISIONER_ID = "proj-fuzzing" SOURCE_URL = "https://github.com/MozillaSecurity/orion" WORKER_TYPE = "ci" +WORKER_TYPE_ARM64 = "ci-arm64" WORKER_TYPE_MSYS = "ci-windows" WORKER_TYPE_BREW = "ci-osx" del os, relativedelta, timedelta, TaskclusterConfig diff --git a/services/orion-decision/src/orion_decision/orion.py b/services/orion-decision/src/orion_decision/orion.py index 131d3a52..712ec4c3 100644 --- a/services/orion-decision/src/orion_decision/orion.py +++ b/services/orion-decision/src/orion_decision/orion.py @@ -8,7 +8,6 @@ from itertools import chain from logging import getLogger from pathlib import Path -from platform import machine from typing import Any, Dict, Generator, Iterable, List, Optional, Pattern, Set, Union from dockerfile_parse import DockerfileParser @@ -199,6 +198,7 @@ class Service: dirty: Whether or not this image needs to be rebuilt tests: Tests to run against this service root: Path where service is defined + archs: List of Linux architectures (e.g. amd64, arm64) """ def __init__( @@ -208,6 +208,7 @@ def __init__( name: str, tests: List[ServiceTest], root: Path, + archs: Optional[List[str]] = None, ) -> None: """Initialize a Service instance. @@ -217,6 +218,7 @@ def __init__( name: Image name (Docker tag) tests: Tests to run against this service root: Path where service is defined + archs: List of Linux architectures (e.g. amd64, arm64) """ self.dockerfile = dockerfile self.context = context @@ -228,6 +230,7 @@ def __init__( self.dirty = False self.tests = tests self.root = root + self.archs = archs or ["amd64"] @classmethod def from_metadata_yaml(cls, metadata_path: Path, context: Path) -> "Service": @@ -262,17 +265,10 @@ def from_metadata_yaml(cls, metadata_path: Path, context: Path) -> "Service": elif metadata.get("type") == "test": result = ServiceTestOnly(context, name, tests, metadata_path.parent) else: - cpu = {"x86_64": "amd64"}.get(machine(), machine()) - if ( - "arch" in metadata - and cpu in metadata["arch"] - and "dockerfile" in metadata["arch"][cpu] - ): - dockerfile = metadata_path.parent / metadata["arch"][cpu]["dockerfile"] - else: - dockerfile = metadata_path.parent / "Dockerfile" + dockerfile = metadata_path.parent / "Dockerfile" assert dockerfile.is_file() - result = cls(dockerfile, context, name, tests, metadata_path.parent) + archs = metadata["arch"] if "arch" in metadata else ["amd64"] + result = cls(dockerfile, context, name, tests, metadata_path.parent, archs) result.service_deps |= set(metadata.get("force_deps", [])) result.weak_deps |= set(metadata.get("force_dirty", [])) return result diff --git a/services/orion-decision/src/orion_decision/scheduler.py b/services/orion-decision/src/orion_decision/scheduler.py index 488343bc..59af5e34 100644 --- a/services/orion-decision/src/orion_decision/scheduler.py +++ b/services/orion-decision/src/orion_decision/scheduler.py @@ -10,7 +10,7 @@ from os import getenv from pathlib import Path from string import Template -from typing import Dict, List, Set, Union +from typing import Dict, List, Set, Tuple, Union from taskcluster.exceptions import TaskclusterFailure from taskcluster.utils import slugId, stringDate @@ -24,6 +24,7 @@ PROVISIONER_ID, SOURCE_URL, WORKER_TYPE, + WORKER_TYPE_ARM64, WORKER_TYPE_BREW, WORKER_TYPE_MSYS, Taskcluster, @@ -47,6 +48,7 @@ PUSH_TASK = Template((TEMPLATES / "push.yaml").read_text()) TEST_TASK = Template((TEMPLATES / "test.yaml").read_text()) RECIPE_TEST_TASK = Template((TEMPLATES / "recipe_test.yaml").read_text()) +WORKERS_ARCHS = {"amd64": WORKER_TYPE, "arm64": WORKER_TYPE_ARM64} class Scheduler: @@ -174,7 +176,7 @@ def mark_services_for_rebuild(self) -> None: self.services.mark_changed_dirty(self.github_event.list_changed_paths()) def _create_build_task( - self, service, dirty_dep_tasks, test_tasks, service_build_tasks + self, service, dirty_dep_tasks, test_tasks, arch, service_build_tasks ): if isinstance(service, ServiceMsys): task_template = MSYS_TASK @@ -238,11 +240,12 @@ def _create_build_task( service_name=service.name, source_url=SOURCE_URL, task_group=self.task_group, - worker=WORKER_TYPE, + worker=WORKERS_ARCHS[arch], + arch=arch, ) ) build_task["dependencies"].extend(dirty_dep_tasks + test_tasks) - task_id = service_build_tasks[service.name] + task_id = service_build_tasks[(service.name, arch)] LOG.info( "%s task %s: %s", self._create_str, task_id, build_task["metadata"]["name"] ) @@ -291,17 +294,18 @@ def _create_svc_test_task( self, service: Service, test: ToxServiceTest, - service_build_tasks: Dict[str, str], + service_build_tasks: Dict[Tuple[str, str], str], + arch: str, ): image: Union[str, Dict[str, str]] = test.image deps = [] - if image in service_build_tasks: + if (image, arch) in service_build_tasks: if self.services[image].dirty: assert isinstance(image, str) - deps.append(service_build_tasks[image]) + deps.append(service_build_tasks[(image, arch)]) image = { "type": "task-image", - "taskId": service_build_tasks[image], + "taskId": service_build_tasks[(image, arch)], } else: image = { @@ -402,7 +406,11 @@ def create_tasks(self) -> None: if self._skip_tasks(): return None should_push = self._should_push() - service_build_tasks = {service: slugId() for service in self.services} + service_build_tasks = { + (service, arch): slugId() + for service in self.services + for arch in WORKERS_ARCHS + } recipe_test_tasks = {recipe: slugId() for recipe in self.services.recipes} test_tasks_created: Set[str] = set() build_tasks_created: Set[str] = set() @@ -419,8 +427,9 @@ def create_tasks(self) -> None: LOG.info("Service %s doesn't need to be rebuilt", obj.name) continue dirty_dep_tasks = [ - service_build_tasks[dep] + service_build_tasks[(dep, arch)] for dep in obj.service_deps + for arch in getattr(obj, "archs", ["amd64"]) if self.services[dep].dirty ] if is_svc: @@ -428,11 +437,13 @@ def create_tasks(self) -> None: dirty_test_dep_tasks = [] for test in obj.tests: assert isinstance(test, ToxServiceTest) - if ( - test.image in service_build_tasks - and self.services[test.image].dirty - ): - dirty_test_dep_tasks.append(service_build_tasks[test.image]) + for arch in obj.archs: + if (test.image, arch) in service_build_tasks and self.services[ + test.image + ].dirty: + dirty_test_dep_tasks.append( + service_build_tasks[(test.image, arch)] + ) else: dirty_test_dep_tasks = [] dirty_recipe_test_tasks = [ @@ -447,7 +458,8 @@ def create_tasks(self) -> None: pending_deps |= set(dirty_recipe_test_tasks) - test_tasks_created if pending_deps: if is_svc: - task_id = service_build_tasks[obj.name] + for arch in obj.archs: + task_id = service_build_tasks[(obj.name, arch)] else: task_id = recipe_test_tasks[obj.name] @@ -466,20 +478,25 @@ def create_tasks(self) -> None: assert isinstance(obj, Service) for test in obj.tests: assert isinstance(test, ToxServiceTest) - task_id = self._create_svc_test_task(obj, test, service_build_tasks) - test_tasks_created.add(task_id) - test_tasks.append(task_id) + for arch in obj.archs: + task_id = self._create_svc_test_task( + obj, test, service_build_tasks, arch + ) + test_tasks_created.add(task_id) + test_tasks.append(task_id) test_tasks.extend(dirty_recipe_test_tasks) if isinstance(obj, ServiceTestOnly): assert obj.tests continue - build_tasks_created.add( - self._create_build_task( - obj, dirty_dep_tasks, test_tasks, service_build_tasks + for arch in obj.archs: + build_tasks_created.add( + self._create_build_task( + obj, dirty_dep_tasks, test_tasks, arch, service_build_tasks + ) ) - ) + if should_push: push_tasks_created.add( self._create_push_task(obj, service_build_tasks) diff --git a/services/orion-decision/src/orion_decision/task_templates/build.yaml b/services/orion-decision/src/orion_decision/task_templates/build.yaml index 927d4adf..d798ef55 100644 --- a/services/orion-decision/src/orion_decision/task_templates/build.yaml +++ b/services/orion-decision/src/orion_decision/task_templates/build.yaml @@ -17,6 +17,7 @@ payload: - "uname -a && exec build" env: ARCHIVE_PATH: /image.tar + ARCH: "${arch}" BUILD_TOOL: podman DOCKERFILE: "${dockerfile}" GIT_REPOSITORY: "${clone_url}" @@ -34,6 +35,6 @@ scopes: - "queue:scheduler-id:${scheduler}" metadata: description: "Build the docker image for ${service_name} tasks" - name: "Orion ${service_name} docker build" + name: "Orion ${service_name} docker build on ${arch}" owner: "${owner_email}" source: "${source_url}" From 99c2798d8332e7ca40ae9dc744a3fe7cb0642fdf Mon Sep 17 00:00:00 2001 From: asuleimanov Date: Tue, 1 Oct 2024 04:20:00 +0000 Subject: [PATCH 04/13] [orion-decision] Fix tests for parallel builds --- .../fixtures/services02/test/{monkey => }/Dockerfile | 0 .../tests/fixtures/services02/test/service.yaml | 3 +-- services/orion-decision/tests/test_cron.py | 4 ++++ services/orion-decision/tests/test_orion.py | 5 ++--- services/orion-decision/tests/test_scheduler.py | 9 +++++++++ 5 files changed, 16 insertions(+), 5 deletions(-) rename services/orion-decision/tests/fixtures/services02/test/{monkey => }/Dockerfile (100%) diff --git a/services/orion-decision/tests/fixtures/services02/test/monkey/Dockerfile b/services/orion-decision/tests/fixtures/services02/test/Dockerfile similarity index 100% rename from services/orion-decision/tests/fixtures/services02/test/monkey/Dockerfile rename to services/orion-decision/tests/fixtures/services02/test/Dockerfile diff --git a/services/orion-decision/tests/fixtures/services02/test/service.yaml b/services/orion-decision/tests/fixtures/services02/test/service.yaml index 71dc7ac9..2d5d73bf 100644 --- a/services/orion-decision/tests/fixtures/services02/test/service.yaml +++ b/services/orion-decision/tests/fixtures/services02/test/service.yaml @@ -1,4 +1,3 @@ name: test-image2 arch: - monkey: - dockerfile: monkey/Dockerfile + - arm64 diff --git a/services/orion-decision/tests/test_cron.py b/services/orion-decision/tests/test_cron.py index 4afae974..9949bdbd 100644 --- a/services/orion-decision/tests/test_cron.py +++ b/services/orion-decision/tests/test_cron.py @@ -184,6 +184,7 @@ def test_cron_create_02(mocker: MockerFixture) -> None: source_url=SOURCE_URL, task_group="group", worker=WORKER_TYPE, + arch="amd64", ) ) _, push_task = queue.createTask.call_args_list[1][0] @@ -204,6 +205,7 @@ def test_cron_create_02(mocker: MockerFixture) -> None: task_group="group", task_index="project.fuzzing.orion.test1.push", worker=WORKER_TYPE, + archs=str(["amd64"]), ) ) push_expected["dependencies"].append(build_task_id) @@ -251,6 +253,7 @@ def test_cron_create_03(mocker: MockerFixture) -> None: source_url=SOURCE_URL, task_group="group", worker=WORKER_TYPE, + arch="amd64", ) ) _, task2 = queue.createTask.call_args_list[1][0] @@ -293,6 +296,7 @@ def test_cron_create_03(mocker: MockerFixture) -> None: source_url=SOURCE_URL, task_group="group", worker=WORKER_TYPE, + arch="amd64", ) ) expected3["dependencies"].append(task1_id) diff --git a/services/orion-decision/tests/test_orion.py b/services/orion-decision/tests/test_orion.py index a6b17bb0..f28b2882 100644 --- a/services/orion-decision/tests/test_orion.py +++ b/services/orion-decision/tests/test_orion.py @@ -37,12 +37,11 @@ def test_service_load01() -> None: assert not svc.dirty -def test_service_load02(mocker: MockerFixture) -> None: +def test_service_load02() -> None: """test that service is loaded from metadata""" - mocker.patch("orion_decision.orion.machine", autospec=True, return_value="monkey") root = FIXTURES / "services02" svc = Service.from_metadata_yaml(root / "test" / "service.yaml", root) - assert svc.dockerfile == root / "test" / "monkey" / "Dockerfile" + assert svc.dockerfile == root / "test" / "Dockerfile" assert svc.context == root assert svc.name == "test-image2" # these are calculated by `Services`, so should be clear diff --git a/services/orion-decision/tests/test_scheduler.py b/services/orion-decision/tests/test_scheduler.py index 0a9d0ae9..2fde68e0 100644 --- a/services/orion-decision/tests/test_scheduler.py +++ b/services/orion-decision/tests/test_scheduler.py @@ -166,6 +166,7 @@ def test_create_02(mocker: MockerFixture) -> None: source_url=SOURCE_URL, task_group="group", worker=WORKER_TYPE, + arch="amd64", ) ) @@ -210,6 +211,7 @@ def test_create_03(mocker: MockerFixture) -> None: source_url=SOURCE_URL, task_group="group", worker=WORKER_TYPE, + arch="amd64", ) ) _, push_task = queue.createTask.call_args_list[1][0] @@ -275,6 +277,7 @@ def test_create_04(mocker: MockerFixture) -> None: source_url=SOURCE_URL, task_group="group", worker=WORKER_TYPE, + arch="amd64", ) ) _, task2 = queue.createTask.call_args_list[1][0] @@ -295,6 +298,7 @@ def test_create_04(mocker: MockerFixture) -> None: source_url=SOURCE_URL, task_group="group", worker=WORKER_TYPE, + arch="amd64", ) ) expected2["dependencies"].append(task1_id) @@ -378,6 +382,7 @@ def test_create_07(mocker: MockerFixture) -> None: source_url=SOURCE_URL, task_group="group", worker=WORKER_TYPE, + arch="amd64", ) ) @@ -449,6 +454,7 @@ def test_create_08( source_url=SOURCE_URL, task_group="group", worker=WORKER_TYPE, + arch="amd64", ) ) svc = "svc1" if svc1_dirty else "svc2" @@ -500,6 +506,7 @@ def test_create_08( source_url=SOURCE_URL, task_group="group", worker=WORKER_TYPE, + arch="amd64", ) ) expected3["dependencies"].append(task2_id) @@ -546,6 +553,7 @@ def test_create_09(mocker: MockerFixture) -> None: source_url=SOURCE_URL, task_group="group", worker=WORKER_TYPE, + arch="amd64", ) ) task2_id, task2 = queue.createTask.call_args_list[1][0] @@ -586,6 +594,7 @@ def test_create_09(mocker: MockerFixture) -> None: source_url=SOURCE_URL, task_group="group", worker=WORKER_TYPE, + arch="amd64", ) ) expected3["dependencies"].append(task2_id) From 1f9981aeb025b234191cc764c8cdb65ad9bbbe39 Mon Sep 17 00:00:00 2001 From: asuleimanov Date: Wed, 2 Oct 2024 04:20:00 +0000 Subject: [PATCH 05/13] [orion-decision] Enable parallel arm64 builds for langfuzz --- services/langfuzz/service.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/services/langfuzz/service.yaml b/services/langfuzz/service.yaml index e9707061..65fba67e 100644 --- a/services/langfuzz/service.yaml +++ b/services/langfuzz/service.yaml @@ -1 +1,4 @@ name: langfuzz +arch: + - amd64 + - arm64 From 3f173c4118ed3bca3de709e1fde58fa706a1c2f6 Mon Sep 17 00:00:00 2001 From: asuleimanov Date: Thu, 3 Oct 2024 04:20:00 +0000 Subject: [PATCH 06/13] [orion-decision] Enable parallel arm64 builds for orion-builder --- services/orion-builder/service.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/services/orion-builder/service.yaml b/services/orion-builder/service.yaml index 0c7ba0fd..a7bdcf8e 100644 --- a/services/orion-builder/service.yaml +++ b/services/orion-builder/service.yaml @@ -1,4 +1,7 @@ name: orion-builder +arch: + - amd64 + - arm64 tests: - name: lint type: tox From 609c382110d9da486a0eea7fad1e67e228577f7f Mon Sep 17 00:00:00 2001 From: asuleimanov Date: Thu, 10 Oct 2024 04:20:00 +0000 Subject: [PATCH 07/13] [orion-builder] Create combine task to save linux builds as a multiarch image --- services/orion-builder/setup.cfg | 1 + .../src/orion_builder/combine.py | 160 ++++++++++++++++++ 2 files changed, 161 insertions(+) create mode 100644 services/orion-builder/src/orion_builder/combine.py diff --git a/services/orion-builder/setup.cfg b/services/orion-builder/setup.cfg index 51ff2b4d..f05bcf24 100644 --- a/services/orion-builder/setup.cfg +++ b/services/orion-builder/setup.cfg @@ -20,6 +20,7 @@ python_requires = >=3.8 [options.entry_points] console_scripts = build = orion_builder.build:main + combine = orion_builder.combine:main push = orion_builder.push:main local-registry = orion_builder.stage_deps:registry_main diff --git a/services/orion-builder/src/orion_builder/combine.py b/services/orion-builder/src/orion_builder/combine.py new file mode 100644 index 00000000..501da229 --- /dev/null +++ b/services/orion-builder/src/orion_builder/combine.py @@ -0,0 +1,160 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. +"""CLI for Orion builder/combine script""" + + +import argparse +import logging +import sys +from os import getenv +from pathlib import Path +from shutil import rmtree +from subprocess import PIPE +from tempfile import mkdtemp +from typing import List, Optional + +import taskcluster +from taskboot.config import Configuration +from taskboot.docker import Docker, Podman +from taskboot.utils import download_artifact, load_artifacts, zstd_compress +from yaml import safe_load as yaml_load + +from .cli import CommonArgs, configure_logging + +LOG = logging.getLogger(__name__) + + +class CombineArgs(CommonArgs): + """CLI arguments for Orion combiner""" + + def __init__(self) -> None: + super().__init__() + self.parser.add_argument( + "--output", + "-o", + dest="write", + default=getenv("ARCHIVE_PATH"), + help="Path to the image tar to output (default: ARCHIVE_PATH)", + ) + self.parser.add_argument( + "--build-tool", + default=getenv("BUILD_TOOL"), + help="Tool for combining builds into multiarch image (default: BUILD_TOOL)", + choices={"podman", "docker"}, + ) + self.parser.add_argument( + "--archs", + action="append", + type=yaml_load, + default=getenv("ARCHS", ["amd64"]), + help="Architectures to be included in the multiarch image", + ) + self.parser.add_argument( + "--service-name", + action="append", + default=getenv("SERVICE_NAME"), + help="Name of the service of the multiarch image", + ) + self.parser.add_argument( + "--image", + default=getenv("IMAGE_NAME"), + help="Docker image name (without repository, default: IMAGE_NAME)", + ) + self.parser.add_argument( + "--registry", + default=getenv("REGISTRY", "docker.io"), + help="Docker registry to use in images tags (default: docker.io)", + ) + self.parser.set_defaults( + build_arg=[], + cache=str(Path.home() / ".local" / "share"), + push=False, + ) + + def sanity_check(self, args: argparse.Namespace) -> None: + super().sanity_check(args) + args.tag = [args.git_revision, "latest"] + + if args.write is None: + self.parser.error("--output (or ARCHIVE_PATH) is required!") + + if args.build_tool is None: + self.parser.error("--build-tool (or BUILD_TOOL) is required!") + + if args.archs is None: + self.parser.error("--archs is required!") + + if args.service_name is None: + self.parser.error("--service-name is required!") + + if args.image is None: + self.parser.error("--image (or IMAGE_NAME) is required!") + + +def main(argv: Optional[List[str]] = None) -> None: + """Combine entrypoint. Does not return.""" + + args = CombineArgs.parse_args(argv) + configure_logging(level=args.log_level) + + service_name = args.service_name + archs = args.archs + base_tag = "latest" + + config = Configuration(argparse.Namespace(secret=None, config=None)) + queue = taskcluster.Queue(config.get_taskcluster_options()) + LOG.info(f"Starting the task to combine {service_name} images for archs: {archs}") + + if args.build_tool == "docker": + tool = Docker() + elif args.build_tool == "podman": + tool = Podman() + else: + raise ValueError(f"Unsupported build tool: {args.build_tool}") + + # retrieve image archives from dependency tasks to /images + image_path = Path(mkdtemp(prefix="image-deps-")) + try: + existing_images = tool.list_images() + LOG.debug("Existing images before loading: %s", existing_images) + + artifacts_ids = load_artifacts(args.task_id, queue, "public/**.tar.zst") + for task_id, artifact_name in artifacts_ids: + img = download_artifact(queue, task_id, artifact_name, image_path) + LOG.info( + "Task %s artifact %s downloaded to: %s", task_id, artifact_name, img + ) + # load images into the podman image store + load_result = tool.run( + [ + "load", + "--input", + str(img), + ], + text=True, + stdout=PIPE, + ) + LOG.info(f"Loaded: {load_result}") + + existing_images = tool.list_images() + LOG.debug("Existing images after loading: %s", existing_images) + assert all( + f"{base_tag}-{arch}" in [image["tag"] for image in existing_images] + for arch in archs + ), "Could not find scheduled archs in local tags" + + # save loaded images into a single multiarch .tar + save_result = tool.run( + ["save", "--multi-image-archive"] + + [ + f"{args.registry}/mozillasecurity/{service_name}:{base_tag}-{arch}" + for arch in archs + ] + + ["--output", f"{args.write}"] + ) + LOG.info(f"Save multiarch image result: {save_result}") + zstd_compress(args.write) + finally: + rmtree(image_path) + sys.exit(0) From bd6ccbe9b22cc46633776c0c9ba5c9d29841c48e Mon Sep 17 00:00:00 2001 From: asuleimanov Date: Tue, 15 Oct 2024 04:20:00 +0000 Subject: [PATCH 08/13] [orion-decision] Schedule combine task to save multiarch service builds --- .../src/orion_decision/scheduler.py | 52 ++++++++++++++++++- .../task_templates/combine.yaml | 40 ++++++++++++++ 2 files changed, 90 insertions(+), 2 deletions(-) create mode 100644 services/orion-decision/src/orion_decision/task_templates/combine.yaml diff --git a/services/orion-decision/src/orion_decision/scheduler.py b/services/orion-decision/src/orion_decision/scheduler.py index 59af5e34..ab5e22d0 100644 --- a/services/orion-decision/src/orion_decision/scheduler.py +++ b/services/orion-decision/src/orion_decision/scheduler.py @@ -45,6 +45,7 @@ BUILD_TASK = Template((TEMPLATES / "build.yaml").read_text()) MSYS_TASK = Template((TEMPLATES / "build_msys.yaml").read_text()) HOMEBREW_TASK = Template((TEMPLATES / "build_homebrew.yaml").read_text()) +COMBINE_TASK = Template((TEMPLATES / "combine.yaml").read_text()) PUSH_TASK = Template((TEMPLATES / "push.yaml").read_text()) TEST_TASK = Template((TEMPLATES / "test.yaml").read_text()) RECIPE_TEST_TASK = Template((TEMPLATES / "recipe_test.yaml").read_text()) @@ -257,6 +258,48 @@ def _create_build_task( raise return task_id + def _create_combine_task(self, service, service_build_tasks): + combine_task = yaml_load( + COMBINE_TASK.substitute( + clone_url=self._clone_url(), + commit=self._commit(), + deadline=stringDate(self.now + DEADLINE), + expires=stringDate(self.now + ARTIFACTS_EXPIRE), + max_run_time=int(MAX_RUN_TIME.total_seconds()), + now=stringDate(self.now), + owner_email=OWNER_EMAIL, + provisioner=PROVISIONER_ID, + scheduler=self.scheduler_id, + service_name=service.name, + source_url=SOURCE_URL, + task_group=self.task_group, + worker=WORKER_TYPE, + archs=str(service.archs), + ) + ) + for arch in service.archs: + LOG.debug( + "Adding combine task dependency: %s", + service_build_tasks[(service.name, arch)], + ) + combine_task["dependencies"].append( + service_build_tasks[(service.name, arch)] + ) + task_id = slugId() + LOG.info( + "%s task %s: %s", + self._create_str, + task_id, + combine_task["metadata"]["name"], + ) + if not self.dry_run: + try: + Taskcluster.get_service("queue").createTask(task_id, combine_task) + except TaskclusterFailure as exc: # pragma: no cover + LOG.error("Error creating combine task: %s", exc) + raise + return task_id + def _create_push_task(self, service, service_build_tasks): push_task = yaml_load( PUSH_TASK.substitute( @@ -414,6 +457,7 @@ def create_tasks(self) -> None: recipe_test_tasks = {recipe: slugId() for recipe in self.services.recipes} test_tasks_created: Set[str] = set() build_tasks_created: Set[str] = set() + combine_tasks_created: Dict[str, str] = {} push_tasks_created: Set[str] = set() to_create = sorted( self.services.recipes.values(), key=lambda x: x.name @@ -496,7 +540,10 @@ def create_tasks(self) -> None: obj, dirty_dep_tasks, test_tasks, arch, service_build_tasks ) ) - + if len(obj.archs) > 1: + combine_tasks_created[obj.name] = self._create_combine_task( + obj, service_build_tasks + ) if should_push: push_tasks_created.add( self._create_push_task(obj, service_build_tasks) @@ -510,10 +557,11 @@ def create_tasks(self) -> None: ) ) LOG.info( - "%s %d test tasks, %d build tasks and %d push tasks", + "%s %d test tasks, %d build tasks, %d combine tasks and %d push tasks", self._created_str, len(test_tasks_created), len(build_tasks_created), + len(combine_tasks_created), len(push_tasks_created), ) diff --git a/services/orion-decision/src/orion_decision/task_templates/combine.yaml b/services/orion-decision/src/orion_decision/task_templates/combine.yaml new file mode 100644 index 00000000..f30a7c31 --- /dev/null +++ b/services/orion-decision/src/orion_decision/task_templates/combine.yaml @@ -0,0 +1,40 @@ +taskGroupId: "${task_group}" +dependencies: [] +created: "${now}" +deadline: "${deadline}" +provisionerId: "${provisioner}" +schedulerId: "${scheduler}" +workerType: "${worker}" +payload: + artifacts: + "public/${service_name}.tar.zst": + expires: "${expires}" + path: /image.tar.zst + type: file + command: + - "sh" + - "-c" + - "exec combine" + env: + ARCHIVE_PATH: /image.tar + BUILD_TOOL: podman + GIT_REPOSITORY: "${clone_url}" + GIT_REVISION: "${commit}" + IMAGE_NAME: "mozillasecurity/${service_name}" + SERVICE_NAME: "${service_name}" + ARCHS: "${archs}" + capabilities: + privileged: true + image: "mozillasecurity/orion-builder:latest" + maxRunTime: !!int "${max_run_time}" +routes: + - "index.project.fuzzing.orion.${service_name}.rev.${commit}" +scopes: + - "docker-worker:capability:privileged" + - "queue:scheduler-id:${scheduler}" +metadata: + description: "Combine all docker images for ${service_name} into a multiarch image" + name: "Orion ${service_name} builds combined into multiarch image" + owner: "${owner_email}" + source: "${source_url}" + From 72ec6a41c9802ef46c336845c60afbc796d38779 Mon Sep 17 00:00:00 2001 From: asuleimanov Date: Wed, 16 Oct 2024 04:20:00 +0000 Subject: [PATCH 09/13] [orion-decision] Add a test for combine task creation --- .../tests/fixtures/services12/Dockerfile | 2 + .../tests/fixtures/services12/script.sh | 1 + .../tests/fixtures/services12/service.yaml | 4 + .../orion-decision/tests/test_scheduler.py | 88 +++++++++++++++++++ 4 files changed, 95 insertions(+) create mode 100644 services/orion-decision/tests/fixtures/services12/Dockerfile create mode 100644 services/orion-decision/tests/fixtures/services12/script.sh create mode 100644 services/orion-decision/tests/fixtures/services12/service.yaml diff --git a/services/orion-decision/tests/fixtures/services12/Dockerfile b/services/orion-decision/tests/fixtures/services12/Dockerfile new file mode 100644 index 00000000..1ef505f0 --- /dev/null +++ b/services/orion-decision/tests/fixtures/services12/Dockerfile @@ -0,0 +1,2 @@ +# hadolint ignore=DL3061 +RUN ./script.sh diff --git a/services/orion-decision/tests/fixtures/services12/script.sh b/services/orion-decision/tests/fixtures/services12/script.sh new file mode 100644 index 00000000..0ed147e8 --- /dev/null +++ b/services/orion-decision/tests/fixtures/services12/script.sh @@ -0,0 +1 @@ +echo "this script only prints this message" \ No newline at end of file diff --git a/services/orion-decision/tests/fixtures/services12/service.yaml b/services/orion-decision/tests/fixtures/services12/service.yaml new file mode 100644 index 00000000..df3e3088 --- /dev/null +++ b/services/orion-decision/tests/fixtures/services12/service.yaml @@ -0,0 +1,4 @@ +name: test1 +arch: + - amd64 + - arm64 \ No newline at end of file diff --git a/services/orion-decision/tests/test_scheduler.py b/services/orion-decision/tests/test_scheduler.py index 2fde68e0..f0e4bbaa 100644 --- a/services/orion-decision/tests/test_scheduler.py +++ b/services/orion-decision/tests/test_scheduler.py @@ -21,12 +21,14 @@ PROVISIONER_ID, SOURCE_URL, WORKER_TYPE, + WORKER_TYPE_ARM64, WORKER_TYPE_BREW, WORKER_TYPE_MSYS, ) from orion_decision.git import GithubEvent from orion_decision.scheduler import ( BUILD_TASK, + COMBINE_TASK, HOMEBREW_TASK, MSYS_TASK, PUSH_TASK, @@ -754,3 +756,89 @@ def test_create_13(mocker: MockerFixture, branch: str, tasks: int) -> None: sched.services["test1"].dirty = True sched.create_tasks() assert queue.createTask.call_count == tasks + + +@freeze_time() +def test_create_14(mocker: MockerFixture) -> None: + """test combine task creation""" + taskcluster = mocker.patch("orion_decision.scheduler.Taskcluster", autospec=True) + queue = taskcluster.get_service.return_value + now = datetime.now(timezone.utc) + root = FIXTURES / "services12" + evt = mocker.Mock(spec=GithubEvent()) + evt.repo.path = root + evt.repo.git = mocker.Mock( + return_value="\n".join(str(p) for p in root.glob("**/*")) + ) + evt.commit = "commit" + evt.branch = "main" + evt.http_url = "https://example.com" + evt.pull_request = None + sched = Scheduler(evt, "group", "scheduler", "secret", "push") + sched.services["test1"].dirty = True + sched.create_tasks() + assert queue.createTask.call_count == 3 + build_task1_id, build_task1 = queue.createTask.call_args_list[0][0] + assert build_task1 == yaml_load( + BUILD_TASK.substitute( + clone_url="https://example.com", + commit="commit", + deadline=stringDate(now + DEADLINE), + dockerfile="Dockerfile", + expires=stringDate(now + ARTIFACTS_EXPIRE), + load_deps="0", + max_run_time=int(MAX_RUN_TIME.total_seconds()), + now=stringDate(now), + owner_email=OWNER_EMAIL, + provisioner=PROVISIONER_ID, + scheduler="scheduler", + service_name="test1", + source_url=SOURCE_URL, + task_group="group", + worker=WORKER_TYPE, + arch="amd64", + ) + ) + build_task2_id, build_task2 = queue.createTask.call_args_list[1][0] + assert build_task2 == yaml_load( + BUILD_TASK.substitute( + clone_url="https://example.com", + commit="commit", + deadline=stringDate(now + DEADLINE), + dockerfile="Dockerfile", + expires=stringDate(now + ARTIFACTS_EXPIRE), + load_deps="0", + max_run_time=int(MAX_RUN_TIME.total_seconds()), + now=stringDate(now), + owner_email=OWNER_EMAIL, + provisioner=PROVISIONER_ID, + scheduler="scheduler", + service_name="test1", + source_url=SOURCE_URL, + task_group="group", + worker=WORKER_TYPE_ARM64, + arch="arm64", + ) + ) + _, combine_task = queue.createTask.call_args_list[2][0] + combine_expected = yaml_load( + COMBINE_TASK.substitute( + clone_url="https://example.com", + commit="commit", + deadline=stringDate(now + DEADLINE), + expires=stringDate(now + ARTIFACTS_EXPIRE), + max_run_time=int(MAX_RUN_TIME.total_seconds()), + now=stringDate(now), + owner_email=OWNER_EMAIL, + provisioner=PROVISIONER_ID, + scheduler="scheduler", + service_name="test1", + source_url=SOURCE_URL, + task_group="group", + worker=WORKER_TYPE, + archs=["amd64", "arm64"], + ) + ) + combine_expected["dependencies"].append(build_task1_id) + combine_expected["dependencies"].append(build_task2_id) + assert combine_task == combine_expected From 1bfc5354172e69d00d5b573a5ae35c0d77687da2 Mon Sep 17 00:00:00 2001 From: asuleimanov Date: Thu, 17 Oct 2024 04:20:00 +0000 Subject: [PATCH 10/13] [orion-builder] Push multiarch manifest to dockerhub --- .../orion-builder/src/orion_builder/push.py | 146 +++++++++++++++++- 1 file changed, 138 insertions(+), 8 deletions(-) diff --git a/services/orion-builder/src/orion_builder/push.py b/services/orion-builder/src/orion_builder/push.py index 7aa092ab..a8a9e09a 100644 --- a/services/orion-builder/src/orion_builder/push.py +++ b/services/orion-builder/src/orion_builder/push.py @@ -5,17 +5,25 @@ import argparse +import logging import sys from os import getenv +from pathlib import Path +from shutil import rmtree +from subprocess import PIPE +from tempfile import mkdtemp from typing import List, Optional import taskcluster from taskboot.config import Configuration -from taskboot.push import push_artifacts -from taskboot.utils import load_artifacts +from taskboot.docker import Podman +from taskboot.utils import download_artifact, load_artifacts +from yaml import safe_load as yaml_load from .cli import CommonArgs, configure_logging +LOG = logging.getLogger(__name__) + class PushArgs(CommonArgs): """CLI arguments for Orion pusher""" @@ -27,12 +35,25 @@ def __init__(self) -> None: exclude_filter=None, push_tool="skopeo", ) + self.parser.add_argument( + "--archs", + action="append", + type=yaml_load, + default=getenv("ARCHS", ["amd64"]), + help="Architectures to be included in the multiarch image", + ) self.parser.add_argument( "--index", default=getenv("TASK_INDEX"), metavar="NAMESPACE", help="Publish task-id at the specified namespace", ) + self.parser.add_argument( + "--service-name", + action="append", + default=getenv("SERVICE_NAME"), + help="Name of the service of the multiarch image", + ) self.parser.add_argument( "--skip-docker", action="store_true", @@ -50,20 +71,29 @@ def sanity_check(self, args: argparse.Namespace) -> None: "--task-id (or TASK_ID) is required to load dependency artifacts!" ) + if args.archs is None: + self.parser.error("--archs is required!") + + if args.service_name is None: + self.parser.error("--service-name is required!") + def main(argv: Optional[List[str]] = None) -> None: """Push entrypoint. Does not return.""" args = PushArgs.parse_args(argv) configure_logging(level=args.log_level) + base_tag = "latest" + + config = Configuration(args) + queue = taskcluster.Queue(config.get_taskcluster_options()) + index = taskcluster.Index(config.get_taskcluster_options()) + tasks = load_artifacts(args.task_id, queue, "public/**.tar.*") + assert len(tasks) == 1 # manually add the task to the TC index. # do this now and not via route on the build task so that post-build tests can run if args.index is not None: - config = Configuration(argparse.Namespace(secret=None, config=None)) - queue = taskcluster.Queue(config.get_taskcluster_options()) - index = taskcluster.Index(config.get_taskcluster_options()) - tasks = load_artifacts(args.task_id, queue, "public/**.tar.*") - assert len(tasks) == 1 + LOG.info("Inserting into TC index task: ", tasks[0][0]) index.insertTask( args.index, { @@ -75,6 +105,106 @@ def main(argv: Optional[List[str]] = None) -> None: ) if not args.skip_docker: - push_artifacts(None, args) + service_name = args.service_name + archs = args.archs + + tool = Podman() + image_path = Path(mkdtemp(prefix="image-deps-")) + task_id, artifact_name = tasks[0] + + try: + img = download_artifact(queue, task_id, artifact_name, image_path) + LOG.info( + "Task %s artifact %s downloaded to: %s", task_id, artifact_name, img + ) + existing_images = tool.list_images() + LOG.debug("Existing images before loading: %s", existing_images) + + # 1. Load image/s artifact into the podman image store + load_result = tool.run( + [ + "load", + "--input", + str(img), + ], + text=True, + stdout=PIPE, + ) + + LOG.info(f"Loaded: {load_result}") + existing_images = tool.list_images() + LOG.debug("Existing images after loading: %s", existing_images) + assert all( + f"{base_tag}-{arch}" in [image["tag"] for image in existing_images] + for arch in archs + ), "Could not find scheduled archs in local tags" + + moz_repo = f"mozillasecurity/{service_name}" + + # 2. Create the podman manifest list + manifest_name = f"docker.io/{moz_repo}:{base_tag}" + + # Remove base_tag from images since manifest list has same tag in the name + if base_tag in [image["tag"] for image in existing_images]: + untag_res = tool.run( + ["untag", manifest_name, manifest_name], text=True, stdout=PIPE + ) + LOG.info("Removed tag %s: %s", base_tag, untag_res) + existing_images = tool.list_images() + LOG.debug("Existing images after untagging: %s", existing_images) + + create_result = tool.run( + [ + "manifest", + "create", + "--amend", + manifest_name, + ], + text=True, + stdout=PIPE, + ) + LOG.info(f"Manifest created: {create_result}") + + # 3. Add the loaded images to the manifest + inspect_result = tool.run( + ["manifest", "inspect", manifest_name], text=True, stdout=PIPE + ) + LOG.debug("Manifest before adding images: %s", inspect_result) + for arch in archs: + add_result = tool.run( + [ + "manifest", + "add", + manifest_name, + f"containers-storage:docker.io/{moz_repo}:{base_tag}-{arch}", + ], + text=True, + stdout=PIPE, + ) + LOG.info(f"Added: {add_result}") + inspect_result = tool.run( + ["manifest", "inspect", manifest_name], text=True, stdout=PIPE + ) + LOG.debug("Manifest after adding images: %s", inspect_result) + # 4. Push the manifest (with images) to docker.io + tool.login( + config.docker["registry"], + config.docker["username"], + config.docker["password"], + ) + push_result = tool.run( + [ + "manifest", + "push", + "--all", + manifest_name, + f"docker://{manifest_name}", + ], + text=True, + stdout=PIPE, + ) + LOG.info(f"Push manifest result: {push_result}") + finally: + rmtree(image_path) sys.exit(0) From 4f3a6f22667db64a0b09568c8e4e521769015a0e Mon Sep 17 00:00:00 2001 From: asuleimanov Date: Fri, 18 Oct 2024 04:20:00 +0000 Subject: [PATCH 11/13] [orion-decision] Schedule multiarch push depending on combine task --- .../src/orion_decision/scheduler.py | 18 +++++++++++++----- .../orion_decision/task_templates/push.yaml | 5 +++++ 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/services/orion-decision/src/orion_decision/scheduler.py b/services/orion-decision/src/orion_decision/scheduler.py index ab5e22d0..07a1a783 100644 --- a/services/orion-decision/src/orion_decision/scheduler.py +++ b/services/orion-decision/src/orion_decision/scheduler.py @@ -300,7 +300,7 @@ def _create_combine_task(self, service, service_build_tasks): raise return task_id - def _create_push_task(self, service, service_build_tasks): + def _create_push_task(self, service, dependency_task): push_task = yaml_load( PUSH_TASK.substitute( clone_url=self._clone_url(), @@ -318,9 +318,10 @@ def _create_push_task(self, service, service_build_tasks): task_group=self.task_group, task_index=self._build_index(service.name), worker=WORKER_TYPE, + archs=str(service.archs), ) ) - push_task["dependencies"].append(service_build_tasks[service.name]) + push_task["dependencies"].append(dependency_task) task_id = slugId() LOG.info( "%s task %s: %s", self._create_str, task_id, push_task["metadata"]["name"] @@ -545,9 +546,16 @@ def create_tasks(self) -> None: obj, service_build_tasks ) if should_push: - push_tasks_created.add( - self._create_push_task(obj, service_build_tasks) - ) + if len(obj.archs) > 1: + push_tasks_created.add( + self._create_push_task(obj, combine_tasks_created[obj.name]) + ) + else: + push_tasks_created.add( + self._create_push_task( + obj, service_build_tasks[(obj.name, obj.archs[0])] + ) + ) else: test_tasks_created.add( self._create_recipe_test_task( diff --git a/services/orion-decision/src/orion_decision/task_templates/push.yaml b/services/orion-decision/src/orion_decision/task_templates/push.yaml index 6c615b3f..65c89a47 100644 --- a/services/orion-decision/src/orion_decision/task_templates/push.yaml +++ b/services/orion-decision/src/orion_decision/task_templates/push.yaml @@ -8,18 +8,23 @@ workerType: "${worker}" payload: command: [push] env: + ARCHS: "${archs}" BUILD_TOOL: podman GIT_REPOSITORY: "${clone_url}" GIT_REVISION: "${commit}" IMAGE_NAME: "mozillasecurity/${service_name}" + SERVICE_NAME: "${service_name}" SKIP_DOCKER: "${skip_docker}" TASK_INDEX: ${task_index} TASKCLUSTER_SECRET: "${docker_secret}" + capabilities: + privileged: true features: taskclusterProxy: true image: "mozillasecurity/orion-builder:latest" maxRunTime: !!int "${max_run_time}" scopes: + - "docker-worker:capability:privileged" - "index:insert-task:project.fuzzing.orion.*" - "queue:scheduler-id:${scheduler}" - "secrets:get:${docker_secret}" From 5998e9f10c0825937ebd678f90c1aa42dce6e712 Mon Sep 17 00:00:00 2001 From: asuleimanov Date: Mon, 21 Oct 2024 04:20:00 +0000 Subject: [PATCH 12/13] [orion-decision] Add a test for multiarch push task creation --- services/orion-decision/tests/test_cron.py | 1 + .../orion-decision/tests/test_scheduler.py | 116 +++++++++++++++++- 2 files changed, 116 insertions(+), 1 deletion(-) diff --git a/services/orion-decision/tests/test_cron.py b/services/orion-decision/tests/test_cron.py index 9949bdbd..d8ec2558 100644 --- a/services/orion-decision/tests/test_cron.py +++ b/services/orion-decision/tests/test_cron.py @@ -274,6 +274,7 @@ def test_cron_create_03(mocker: MockerFixture) -> None: task_group="group", task_index="project.fuzzing.orion.test1.push", worker=WORKER_TYPE, + archs=str(["amd64"]), ) ) expected2["dependencies"].append(task1_id) diff --git a/services/orion-decision/tests/test_scheduler.py b/services/orion-decision/tests/test_scheduler.py index f0e4bbaa..88acc526 100644 --- a/services/orion-decision/tests/test_scheduler.py +++ b/services/orion-decision/tests/test_scheduler.py @@ -175,7 +175,7 @@ def test_create_02(mocker: MockerFixture) -> None: @freeze_time() def test_create_03(mocker: MockerFixture) -> None: - """test push task creation""" + """test push task creation for single arch""" taskcluster = mocker.patch("orion_decision.scheduler.Taskcluster", autospec=True) queue = taskcluster.get_service.return_value now = datetime.now(timezone.utc) @@ -234,6 +234,7 @@ def test_create_03(mocker: MockerFixture) -> None: task_group="group", task_index="project.fuzzing.orion.test1.push", worker=WORKER_TYPE, + archs=["amd64"], ) ) push_expected["dependencies"].append(build_task_id) @@ -842,3 +843,116 @@ def test_create_14(mocker: MockerFixture) -> None: combine_expected["dependencies"].append(build_task1_id) combine_expected["dependencies"].append(build_task2_id) assert combine_task == combine_expected + + +@freeze_time() +def test_create_15(mocker: MockerFixture) -> None: + """test push task creation for multiple archs""" + taskcluster = mocker.patch("orion_decision.scheduler.Taskcluster", autospec=True) + queue = taskcluster.get_service.return_value + now = datetime.now(timezone.utc) + root = FIXTURES / "services12" + evt = mocker.Mock(spec=GithubEvent()) + evt.repo.path = root + evt.repo.git = mocker.Mock( + return_value="\n".join(str(p) for p in root.glob("**/*")) + ) + evt.repo.refs.return_value = {} + evt.commit = "commit" + evt.branch = "push" + evt.event_type = "push" + evt.http_url = "https://example.com" + evt.pull_request = None + sched = Scheduler(evt, "group", "scheduler", "secret", "push") + sched.services["test1"].dirty = True + sched.create_tasks() + assert queue.createTask.call_count == 4 + + build_task_id, build_task = queue.createTask.call_args_list[0][0] + build_task1_id, build_task1 = queue.createTask.call_args_list[0][0] + assert build_task1 == yaml_load( + BUILD_TASK.substitute( + clone_url="https://example.com", + commit="commit", + deadline=stringDate(now + DEADLINE), + dockerfile="Dockerfile", + expires=stringDate(now + ARTIFACTS_EXPIRE), + load_deps="0", + max_run_time=int(MAX_RUN_TIME.total_seconds()), + now=stringDate(now), + owner_email=OWNER_EMAIL, + provisioner=PROVISIONER_ID, + scheduler="scheduler", + service_name="test1", + source_url=SOURCE_URL, + task_group="group", + worker=WORKER_TYPE, + arch="amd64", + ) + ) + build_task2_id, build_task2 = queue.createTask.call_args_list[1][0] + assert build_task2 == yaml_load( + BUILD_TASK.substitute( + clone_url="https://example.com", + commit="commit", + deadline=stringDate(now + DEADLINE), + dockerfile="Dockerfile", + expires=stringDate(now + ARTIFACTS_EXPIRE), + load_deps="0", + max_run_time=int(MAX_RUN_TIME.total_seconds()), + now=stringDate(now), + owner_email=OWNER_EMAIL, + provisioner=PROVISIONER_ID, + scheduler="scheduler", + service_name="test1", + source_url=SOURCE_URL, + task_group="group", + worker=WORKER_TYPE_ARM64, + arch="arm64", + ) + ) + combine_task_id, combine_task = queue.createTask.call_args_list[2][0] + combine_expected = yaml_load( + COMBINE_TASK.substitute( + clone_url="https://example.com", + commit="commit", + deadline=stringDate(now + DEADLINE), + expires=stringDate(now + ARTIFACTS_EXPIRE), + max_run_time=int(MAX_RUN_TIME.total_seconds()), + now=stringDate(now), + owner_email=OWNER_EMAIL, + provisioner=PROVISIONER_ID, + scheduler="scheduler", + service_name="test1", + source_url=SOURCE_URL, + task_group="group", + worker=WORKER_TYPE, + archs=str(["amd64", "arm64"]), + ) + ) + combine_expected["dependencies"].append(build_task1_id) + combine_expected["dependencies"].append(build_task2_id) + assert combine_task == combine_expected + _, push_task = queue.createTask.call_args_list[3][0] + push_expected = yaml_load( + PUSH_TASK.substitute( + clone_url="https://example.com", + commit="commit", + deadline=stringDate(now + DEADLINE), + docker_secret="secret", + max_run_time=int(MAX_RUN_TIME.total_seconds()), + now=stringDate(now), + owner_email=OWNER_EMAIL, + provisioner=PROVISIONER_ID, + scheduler="scheduler", + service_name="test1", + skip_docker="0", + source_url=SOURCE_URL, + task_group="group", + task_index="project.fuzzing.orion.test1.push", + worker=WORKER_TYPE, + archs=str(["amd64", "arm64"]), + ) + ) + push_expected["dependencies"].append(combine_task_id) + assert push_task == push_expected From b5c0cf790ab99356d119b825a1c745151e411253 Mon Sep 17 00:00:00 2001 From: asuleimanov Date: Mon, 28 Oct 2024 04:20:00 +0000 Subject: [PATCH 13/13] [orion-decision] Schedule all tasks per separate (service, arch) --- .../src/orion_decision/scheduler.py | 97 +++++++++++-------- 1 file changed, 59 insertions(+), 38 deletions(-) diff --git a/services/orion-decision/src/orion_decision/scheduler.py b/services/orion-decision/src/orion_decision/scheduler.py index 07a1a783..de5c0a30 100644 --- a/services/orion-decision/src/orion_decision/scheduler.py +++ b/services/orion-decision/src/orion_decision/scheduler.py @@ -453,18 +453,27 @@ def create_tasks(self) -> None: service_build_tasks = { (service, arch): slugId() for service in self.services - for arch in WORKERS_ARCHS + for arch in getattr(self.services[service], "archs", ["amd64"]) } + for (service, arch), task_id in service_build_tasks.items(): + LOG.debug("Task %s is a build of %s on %s", task_id, service, arch) recipe_test_tasks = {recipe: slugId() for recipe in self.services.recipes} - test_tasks_created: Set[str] = set() + for recipe, task_id in recipe_test_tasks.items(): + LOG.debug("Task %s is a recipe test for %s", task_id, recipe) + test_tasks_created: Dict[Tuple[str, str], str] = {} + recipe_tasks_created: Set[str] = set() build_tasks_created: Set[str] = set() combine_tasks_created: Dict[str, str] = {} push_tasks_created: Set[str] = set() - to_create = sorted( - self.services.recipes.values(), key=lambda x: x.name - ) + sorted(self.services.values(), key=lambda x: x.name) + to_create = [ + (recipe, "amd64") + for recipe in sorted(self.services.recipes.values(), key=lambda x: x.name) + ] + for service in sorted(self.services.values(), key=lambda x: x.name): + for arch in getattr(service, "archs", ["amd64"]): + to_create.append((service, arch)) while to_create: - obj = to_create.pop(0) + obj, arch = to_create.pop(0) is_svc = isinstance(obj, Service) if not obj.dirty: @@ -477,18 +486,18 @@ def create_tasks(self) -> None: for arch in getattr(obj, "archs", ["amd64"]) if self.services[dep].dirty ] + # TODO: implement tests for all archs in the future if is_svc: assert isinstance(obj, Service) dirty_test_dep_tasks = [] for test in obj.tests: assert isinstance(test, ToxServiceTest) - for arch in obj.archs: - if (test.image, arch) in service_build_tasks and self.services[ - test.image - ].dirty: - dirty_test_dep_tasks.append( - service_build_tasks[(test.image, arch)] - ) + if (test.image, "amd64") in service_build_tasks and self.services[ + test.image + ].dirty: + dirty_test_dep_tasks.append( + service_build_tasks[(test.image, "amd64")] + ) else: dirty_test_dep_tasks = [] dirty_recipe_test_tasks = [ @@ -500,11 +509,12 @@ def create_tasks(self) -> None: pending_deps = ( set(dirty_dep_tasks) | set(dirty_test_dep_tasks) ) - build_tasks_created - pending_deps |= set(dirty_recipe_test_tasks) - test_tasks_created + pending_deps |= ( + set(dirty_recipe_test_tasks) - set(test_tasks_created.values()) + ) - recipe_tasks_created if pending_deps: if is_svc: - for arch in obj.archs: - task_id = service_build_tasks[(obj.name, arch)] + task_id = service_build_tasks[(obj.name, arch)] else: task_id = recipe_test_tasks[obj.name] @@ -515,7 +525,8 @@ def create_tasks(self) -> None: task_id, ", ".join(pending_deps), ) - to_create.append(obj) + assert to_create, f"{obj.name} creates circular dependency" + to_create.append((obj, arch)) continue if is_svc: @@ -523,51 +534,61 @@ def create_tasks(self) -> None: assert isinstance(obj, Service) for test in obj.tests: assert isinstance(test, ToxServiceTest) - for arch in obj.archs: + if arch == "amd64": task_id = self._create_svc_test_task( obj, test, service_build_tasks, arch ) - test_tasks_created.add(task_id) - test_tasks.append(task_id) + test_tasks_created[(obj.name, test.name)] = task_id + else: + task_id = test_tasks_created[(obj.name, test.name)] + test_tasks.append(task_id) test_tasks.extend(dirty_recipe_test_tasks) if isinstance(obj, ServiceTestOnly): assert obj.tests continue - for arch in obj.archs: - build_tasks_created.add( - self._create_build_task( - obj, dirty_dep_tasks, test_tasks, arch, service_build_tasks - ) + build_tasks_created.add( + self._create_build_task( + obj, dirty_dep_tasks, test_tasks, arch, service_build_tasks ) - if len(obj.archs) > 1: + ) + multi_arch = len(obj.archs) > 1 + last_build_for_svc = not any( + obj is pending for (pending, _) in to_create + ) + + if multi_arch and last_build_for_svc: combine_tasks_created[obj.name] = self._create_combine_task( obj, service_build_tasks ) if should_push: - if len(obj.archs) > 1: - push_tasks_created.add( - self._create_push_task(obj, combine_tasks_created[obj.name]) - ) + if multi_arch: + if last_build_for_svc: + push_tasks_created.add( + self._create_push_task( + obj, combine_tasks_created[obj.name] + ) + ) else: push_tasks_created.add( self._create_push_task( - obj, service_build_tasks[(obj.name, obj.archs[0])] + obj, service_build_tasks[(obj.name, arch)] ) ) else: - test_tasks_created.add( - self._create_recipe_test_task( - obj, - dirty_dep_tasks + dirty_recipe_test_tasks, - recipe_test_tasks, + if arch == "amd64": + recipe_tasks_created.add( + self._create_recipe_test_task( + obj, + dirty_dep_tasks + dirty_recipe_test_tasks, + recipe_test_tasks, + ) ) - ) LOG.info( "%s %d test tasks, %d build tasks, %d combine tasks and %d push tasks", self._created_str, - len(test_tasks_created), + len(test_tasks_created) + len(recipe_tasks_created), len(build_tasks_created), len(combine_tasks_created), len(push_tasks_created),