From e89cd15e52eac773e9701f513629ce421f8aaf10 Mon Sep 17 00:00:00 2001 From: Jude Taylor Date: Wed, 12 Oct 2022 18:17:53 +0000 Subject: [PATCH] Build guestos in pipeline using bazel (re-merge) --- .gitignore | 3 +- .../46--guest-os-build--guest-os-diskimg.yml | 27 +- .../47--guest-os-test--e2e-scalability.yml | 9 +- .../47--guest-os-test--guest-os-e2e-test.yml | 68 ++-- .../53--host-os-build--build-setupos.yml | 4 +- gitlab-ci/src/job_scripts/guest_os_diskimg.py | 35 +- .../src/job_scripts/lib/guest-os-diskimg.sh | 52 +-- ic-os/defs.bzl | 299 ++++++++++++++++++ ic-os/guestos/BUILD.bazel | 296 +++-------------- ic-os/guestos/launch_single_vm.py | 8 +- rs/tests/run-system-tests.py | 4 +- toolchains/sysimage/build_disk_image.py | 3 + toolchains/sysimage/docker_tar.py | 8 + toolchains/sysimage/toolchain.bzl | 9 +- 14 files changed, 460 insertions(+), 365 deletions(-) create mode 100644 ic-os/defs.bzl diff --git a/.gitignore b/.gitignore index 78a74aed2ec..a66ebe4e0cf 100644 --- a/.gitignore +++ b/.gitignore @@ -68,12 +68,13 @@ dist-newstyle # JUnit files test_report.xml -# IC-OS disk images +# IC-OS disk image files *-os.iso *os.img disk.img *-img.tar.gz *-img.tar.zst +dev-root-ca.crt # IC-OS binaries infogetty diff --git a/gitlab-ci/config/46--guest-os-build--guest-os-diskimg.yml b/gitlab-ci/config/46--guest-os-build--guest-os-diskimg.yml index 1890ead9c97..35b4b9d2b65 100644 --- a/gitlab-ci/config/46--guest-os-build--guest-os-diskimg.yml +++ b/gitlab-ci/config/46--guest-os-build--guest-os-diskimg.yml @@ -1,26 +1,21 @@ guest-os-diskimg: + needs: [] extends: - .ubuntu-docker-k8s-protected - .rules-parent-pipeline-autorun-on-trigger + tags: + - dfinity + - ubuntu + - zh stage: guest-os-build - needs: - - job: cargo-build-release-linux-native - artifacts: false variables: BUILD_EXTRA_ARGS: "" BUILD_EXTRA_SUFFIX: "" + artifacts: + paths: + - version.txt script: - | - "$CI_PROJECT_DIR"/gitlab-ci/src/artifacts/rclone_download.py \ - --git-rev="$CI_COMMIT_SHA" --remote-path="release" \ - --out="artifacts/release" - - if [[ "$CI_JOB_NAME" == *"-malicious" ]]; then - "$CI_PROJECT_DIR"/gitlab-ci/src/artifacts/rclone_download.py \ - --git-rev="$CI_COMMIT_SHA" --remote-path="release-malicious" \ - --out="artifacts/release-malicious" - fi - "${CI_PROJECT_DIR}/gitlab-ci/tools/job-driver.py" guest-os-diskimg # Build dev image. @@ -28,6 +23,7 @@ guest-os-diskimg: # This is the same as the prod image with the following differences: # - it has password "root" set for root to allow console login guest-os-diskimg-dev: + needs: [] extends: - guest-os-diskimg variables: @@ -41,11 +37,6 @@ guest-os-diskimg-dev: guest-os-diskimg-dev-malicious: extends: - guest-os-diskimg - needs: - - job: cargo-build-release-linux-native - artifacts: false - - job: cargo-build-release-linux-native-malicious - artifacts: false variables: BUILD_EXTRA_ARGS: "-t dev -p root" BUILD_EXTRA_SUFFIX: "-dev-malicious" diff --git a/gitlab-ci/config/47--guest-os-test--e2e-scalability.yml b/gitlab-ci/config/47--guest-os-test--e2e-scalability.yml index ff2b9ebfe10..4e7ac9ca593 100644 --- a/gitlab-ci/config/47--guest-os-test--e2e-scalability.yml +++ b/gitlab-ci/config/47--guest-os-test--e2e-scalability.yml @@ -2,8 +2,7 @@ e2e-scalability-suite-test: extends: - .e2e-test-base-k8s needs: - - job: guest-os-diskimg - artifacts: false + - job: guest-os-diskimg-dev - job: guest-os-updateimg-build artifacts: false artifacts: @@ -11,6 +10,7 @@ e2e-scalability-suite-test: - scalability/ script: - | + ICOS_VER=$(cat version.txt) # release "$CI_PROJECT_DIR"/gitlab-ci/src/artifacts/rclone_download.py \ --git-rev="$CI_COMMIT_SHA" --remote-path="release" \ @@ -20,9 +20,6 @@ e2e-scalability-suite-test: --git-rev="$CI_COMMIT_SHA" --remote-path="canisters" \ --out="artifacts/canisters" - GIT_REV=$(git rev-parse HEAD) - GIT_LATEST_WITH_IC_OS=$("${CI_PROJECT_DIR}/gitlab-ci/src/artifacts/newest_sha_with_disk_image.sh" "$GIT_REV") - mkdir -p gitlab-runner-tmp; cd gitlab-runner-tmp ls "${CI_PROJECT_DIR}/artifacts/canisters" @@ -52,7 +49,7 @@ e2e-scalability-suite-test: pipenv run python3 "common/tests/e2e-scalability-tests.py" \ --timeout "$GUEST_OS_TEST_TIMEOUT" \ - --ic_os_version "$GIT_LATEST_WITH_IC_OS" \ + --ic_os_version "$ICOS_VER" \ --ic_admin_bin "${CI_PROJECT_DIR}/artifacts/release/ic-admin" \ --nns_canisters "${CI_PROJECT_DIR}/artifacts/canisters/" \ --ic_prep_bin "${CI_PROJECT_DIR}/artifacts/release/ic-prep" \ diff --git a/gitlab-ci/config/47--guest-os-test--guest-os-e2e-test.yml b/gitlab-ci/config/47--guest-os-test--guest-os-e2e-test.yml index 91096a04c75..b6abfc17d05 100644 --- a/gitlab-ci/config/47--guest-os-test--guest-os-e2e-test.yml +++ b/gitlab-ci/config/47--guest-os-test--guest-os-e2e-test.yml @@ -33,6 +33,8 @@ .run-farm-based-test: &run-farm-based-test | IC_VERSION_ID=$CI_COMMIT_SHA export IC_VERSION_ID + GUESTOS_VERSION_OVERRIDE=$(cat version.txt) + export GUESTOS_VERSION_OVERRIDE #XXX(marko): we need to change this # binaries are needed for run-system-tests.py "$CI_PROJECT_DIR"/gitlab-ci/src/artifacts/rclone_download.py \ @@ -71,7 +73,6 @@ - job: guest-os-diskimg artifacts: false - job: guest-os-diskimg-dev - artifacts: false - job: guest-os-diskimg-dev-malicious artifacts: false - job: guest-os-updateimg-build @@ -139,34 +140,36 @@ tecdsa-pre-master: SUITE_NAME: "tecdsa_pre_master" test-prod-test-driver-e2e: - extends: - - .system-tests - - .rules-test-prod-test-driver-e2e - script: - - | - # IC_VERSION_ID is needed for run-system-tests.py. - IC_VERSION_ID="$CI_COMMIT_SHA" - export IC_VERSION_ID - # required binary artifacts - mkdir -p artifacts/release - pushd artifacts/release - curl -sfS --retry 5 --retry-delay 10 \ - "http://download.proxy.dfinity.systems:8080/ic/$CI_COMMIT_SHA/release/prod-test-driver.gz" -O - popd - # required canisters - "$CI_PROJECT_DIR"/gitlab-ci/src/artifacts/rclone_download.py \ - --git-rev="${IC_VERSION_ID}" --remote-path="canisters" \ - --out="artifacts/canisters" - WORKING_DIR="${CI_PROJECT_DIR}/working_dir/" - mkdir -p "$WORKING_DIR" - # Logs produced during execution of the system tests are stored away in a structured way, i.e. one file per test. - # Unstructured logs, i.e. those which bypassed the logger and were outputed directly to stdout/stderr, - # are forwarded to a separate file. - # All lines produced by the logger share the same structure: e.g. "Nov 10 13:20:30.931 INFO ...". - # Hence, the fourth column can be used to distinguish structured from unstructured logs. - $SHELL_WRAPPER "${CI_PROJECT_DIR}/rs/tests/tests_e2e/test_prod_test_driver_e2e.py" \ - | tee "${WORKING_DIR}/unstructured-logs.log" \ - | awk '$4 ~ /CRIT|ERRO|WARN|INFO|DEBG|TRCE/' + extends: + - .system-tests + - .rules-test-prod-test-driver-e2e + script: + - | + # IC_VERSION_ID is needed for run-system-tests.py. + IC_VERSION_ID="$CI_COMMIT_SHA" + export IC_VERSION_ID + GUESTOS_VERSION_OVERRIDE=$(cat version.txt) + export GUESTOS_VERSION_OVERRIDE + # required binary artifacts + mkdir -p artifacts/release + pushd artifacts/release + curl -sfS --retry 5 --retry-delay 10 \ + "http://download.proxy.dfinity.systems:8080/ic/$CI_COMMIT_SHA/release/prod-test-driver.gz" -O + popd + # required canisters + "$CI_PROJECT_DIR"/gitlab-ci/src/artifacts/rclone_download.py \ + --git-rev="${IC_VERSION_ID}" --remote-path="canisters" \ + --out="artifacts/canisters" + WORKING_DIR="${CI_PROJECT_DIR}/working_dir/" + mkdir -p "$WORKING_DIR" + # Logs produced during execution of the system tests are stored away in a structured way, i.e. one file per test. + # Unstructured logs, i.e. those which bypassed the logger and were outputed directly to stdout/stderr, + # are forwarded to a separate file. + # All lines produced by the logger share the same structure: e.g. "Nov 10 13:20:30.931 INFO ...". + # Hence, the fourth column can be used to distinguish structured from unstructured logs. + $SHELL_WRAPPER "${CI_PROJECT_DIR}/rs/tests/tests_e2e/test_prod_test_driver_e2e.py" \ + | tee "${WORKING_DIR}/unstructured-logs.log" \ + | awk '$4 ~ /CRIT|ERRO|WARN|INFO|DEBG|TRCE/' rosetta-pre-master: extends: .system-tests @@ -192,8 +195,7 @@ upgrade-compatibility-pre-master: needs: - job: cargo-build-release-linux-native artifacts: false - - job: guest-os-diskimg - artifacts: false + - job: guest-os-diskimg-dev - job: guest-os-updateimg-build artifacts: false - job: guest-os-updateimg-build-dev @@ -238,8 +240,10 @@ spec-compliance-pre-master: --out="artifacts/canisters" # requirement of run-system-tests.py - IC_VERSION_ID="$CI_COMMIT_SHA" + IC_VERSION_ID=$CI_COMMIT_SHA export IC_VERSION_ID + GUESTOS_VERSION_OVERRIDE=$(cat version.txt) + export GUESTOS_VERSION_OVERRIDE "$SHELL_WRAPPER" nix-shell -p "(import ./nix {}).ic-ref" --run " ${CI_PROJECT_DIR}/rs/tests/run-system-tests.py --suite=spec_compliance diff --git a/gitlab-ci/config/53--host-os-build--build-setupos.yml b/gitlab-ci/config/53--host-os-build--build-setupos.yml index be7f97eb847..a7576f9b3ad 100644 --- a/gitlab-ci/config/53--host-os-build--build-setupos.yml +++ b/gitlab-ci/config/53--host-os-build--build-setupos.yml @@ -1,7 +1,6 @@ setup-os-diskimg: needs: - job: guest-os-diskimg - artifacts: false - job: host-os-diskimg artifacts: false extends: @@ -15,10 +14,11 @@ setup-os-diskimg: VERSION=$(git rev-parse HEAD) export VERSION echo "Build ID: ${VERSION}" + ICOS_VERSION=$(cat version.txt) cd "${CI_PROJECT_DIR}"/ic-os/setupos - "$CI_PROJECT_DIR"/gitlab-ci/src/artifacts/rclone_download.py --remote-path=guest-os --out=guestos --git-rev="${VERSION}" + "$CI_PROJECT_DIR"/gitlab-ci/src/artifacts/rclone_download.py --remote-path=guest-os --out=guestos --git-rev="${ICOS_VERSION}" "$CI_PROJECT_DIR"/gitlab-ci/src/artifacts/rclone_download.py --remote-path=host-os --out=hostos --git-rev="${VERSION}" BUILD_OUT="build-out/disk-img" diff --git a/gitlab-ci/src/job_scripts/guest_os_diskimg.py b/gitlab-ci/src/job_scripts/guest_os_diskimg.py index 9b4727b0521..ba9a78ca33c 100644 --- a/gitlab-ci/src/job_scripts/guest_os_diskimg.py +++ b/gitlab-ci/src/job_scripts/guest_os_diskimg.py @@ -1,6 +1,6 @@ import logging -from os import environ from os import getenv +from pathlib import Path from ci import buildevent from ci import cwd @@ -11,26 +11,20 @@ def run(): build_extra_suffix = getenv("BUILD_EXTRA_SUFFIX", "") build_out = f"build-out/disk-img{build_extra_suffix}" - build_tmp = f"build-tmp{build_extra_suffix}" upload_target = f"guest-os/disk-img{build_extra_suffix}" version = ENV.build_id logging.info(f"Build ID: {version}") + build_mode = "dev" if "-dev" in build_extra_suffix else "prod" + malicious_mode = "-malicious" if "-malicious" in build_extra_suffix else "" + with cwd("ic-os/guestos"): - # lib/guest-os-diskimg.sh fails if these are not set, which they aren't when running locally - offline_defaults = {"BUILD_EXTRA_ARGS": "", "BUILD_EXTRA_SUFFIX": "", "CI_JOB_ID": ""} - # override those variables with the already-set ones if there are any - script_env = {**offline_defaults, **environ.copy()} - - sh( - f"{ENV.top}/gitlab-ci/src/job_scripts/lib/guest-os-diskimg.sh", - build_out, - build_tmp, - upload_target, - version, - environ.get("CDPRNET", ""), - env=script_env, - ) + sh(f"{ENV.top}/gitlab-ci/src/job_scripts/lib/guest-os-diskimg.sh", build_out, build_mode, malicious_mode) + + version_txt = Path(f"{ENV.top}/version.txt") + if version_txt.is_file(): + legacy_version = version + version = version_txt.read_text().strip() if ENV.is_gitlab: with buildevent("rclone"): @@ -41,5 +35,14 @@ def run(): build_out, upload_target, ) + if legacy_version: + with buildevent("rclone"): + sh( + f"{ENV.top}/gitlab-ci/src/artifacts/rclone_upload.py", + f"--version={legacy_version}", + "--verbose", + build_out, + upload_target, + ) logging.info(f"Build ID (real or fake git revision): {version}") diff --git a/gitlab-ci/src/job_scripts/lib/guest-os-diskimg.sh b/gitlab-ci/src/job_scripts/lib/guest-os-diskimg.sh index c30f7e84713..c6d75a43e0f 100755 --- a/gitlab-ci/src/job_scripts/lib/guest-os-diskimg.sh +++ b/gitlab-ci/src/job_scripts/lib/guest-os-diskimg.sh @@ -8,50 +8,28 @@ set -euo pipefail BUILD_OUT=${1:-"build-out/disk-img"} -BUILD_TMP=${2:-"build-tmp"} -UPLOAD_TARGET=${3:-"guest-os/disk-img"} -VERSION=${4:-$(git rev-parse --verify HEAD)} -CDPRNET=${5:-"cdpr05"} +BUILD_MODE=${2:-"dev"} +MALICIOUS_MODE=${3:-} ROOT_DIR=$(git rev-parse --show-toplevel) groups -cd "$ROOT_DIR" || exit 1 -for f in replica orchestrator canister_sandbox sandbox_launcher vsock_agent state-tool ic-consensus-pool-util ic-crypto-csp ic-regedit ic-recovery ic-btc-adapter ic-canister-http-adapter; do - gunzip -c -d artifacts/release/$f.gz >artifacts/release/$f -done +cd "$ROOT_DIR"/ic-os/guestos || exit 1 +mkdir -p "$BUILD_OUT" -# if we are building the malicious image, use malicious replica version -if [[ "${BUILD_EXTRA_SUFFIX}" =~ "malicious" ]]; then - gunzip -c -d artifacts/release-malicious/replica.gz >artifacts/release/replica - chmod +x artifacts/release/replica +if [ "$BUILD_MODE" = "dev" ]; then + ln -sfv "$DEV_ROOT_CA" "$PWD/dev-root-ca.crt" fi -cd "$ROOT_DIR"/ic-os/guestos || exit 1 -mkdir -p "$BUILD_OUT" "$BUILD_TMP" -echo "$VERSION" >"${BUILD_TMP}/version.txt" - -if [ -z "$CI_JOB_ID" ]; then - # shellcheck disable=SC2086 # Expanding BUILD_EXTRA_ARGS into multiple parameters - ./scripts/build-disk-image.sh -o "${BUILD_TMP}/disk.img" -v "$VERSION" -x ../../artifacts/release/ $BUILD_EXTRA_ARGS - tar --sort=name --owner=root:0 --group=root:0 --mtime='UTC 2020-01-01' --sparse \ - -cvzf "${BUILD_OUT}/disk-img.tar.gz" -C "$BUILD_TMP" disk.img version.txt - tar --sort=name --owner=root:0 --group=root:0 --mtime='UTC 2020-01-01' --sparse \ - -cvf "${BUILD_OUT}/disk-img.tar.zst" --use-compress-program="zstd --threads=0 -10" \ - -C "$BUILD_TMP" disk.img version.txt - ls -lah "$BUILD_TMP" -else - # shellcheck disable=SC2086 # Expanding BUILD_EXTRA_ARGS into multiple parameters - buildevents cmd "${ROOT_PIPELINE_ID}" "${CI_JOB_ID}" build-disk-img -- \ - ./scripts/build-disk-image.sh -o "${BUILD_TMP}/disk.img" -v "$VERSION" -x ../../artifacts/release/ $BUILD_EXTRA_ARGS - buildevents cmd "$ROOT_PIPELINE_ID" "$CI_JOB_ID" tar-build-out -- \ - tar --sort=name --owner=root:0 --group=root:0 --mtime='UTC 2020-01-01' --sparse \ - -cvzf "${BUILD_OUT}/disk-img.tar.gz" -C "$BUILD_TMP" disk.img version.txt - buildevents cmd "$ROOT_PIPELINE_ID" "$CI_JOB_ID" tar-build-out -- \ - tar --sort=name --owner=root:0 --group=root:0 --mtime='UTC 2020-01-01' --sparse \ - -cvf "${BUILD_OUT}/disk-img.tar.zst" --use-compress-program="zstd --threads=0 -10" \ - -C "$BUILD_TMP" disk.img version.txt - ls -lah "$BUILD_TMP" +BUILD_MODE="${BUILD_MODE}${MALICIOUS_MODE}" + +bazel build --config=ci //ic-os/guestos:"$BUILD_MODE"_disk-img.tar_gz //ic-os/guestos:"$BUILD_MODE"_disk-img.tar_zst +cp -fv "$ROOT_DIR"/$(bazel cquery --output=files //ic-os/guestos:"$BUILD_MODE"_disk-img.tar_gz) "$BUILD_OUT"/disk-img.tar.gz +cp -fv "$ROOT_DIR"/$(bazel cquery --output=files //ic-os/guestos:"$BUILD_MODE"_disk-img.tar_zst) "$BUILD_OUT"/disk-img.tar.zst + +if [ -n "${CI_JOB_ID:-}" ]; then "$ROOT_DIR"/gitlab-ci/src/artifacts/openssl-sign.sh "$BUILD_OUT" fi + +cat "$ROOT_DIR"/$(bazel cquery --output=files //ic-os/guestos:"$BUILD_MODE"_version.txt) >"$ROOT_DIR"/version.txt diff --git a/ic-os/defs.bzl b/ic-os/defs.bzl new file mode 100644 index 00000000000..ba7e9bb9b95 --- /dev/null +++ b/ic-os/defs.bzl @@ -0,0 +1,299 @@ +""" +A macro to build multiple versions of the ICOS image (i.e., dev vs prod) +""" + +load("//toolchains/sysimage:toolchain.bzl", "disk_image", "docker_tar", "ext4_image", "sha256sum", "summary_sha256sum", "tar_extract", "upgrade_image") +load("//gitlab-ci/src/artifacts:upload.bzl", "upload_artifacts", "urls_test") + +img_bases = { + "dev": "dfinity/guestos-base-dev@sha256:9a3c0621cc26e5b94fa3517c55c8f57eee901cc0e2d209e0bf9588bfff561709", + "prod": "dfinity/guestos-base@sha256:3af45c09deadf23bf09bf07d806597b842478ea5fed3c5b06bb766a8a9cf5e84", +} + +# Declare the dependencies that we will have for the built filesystem images. +# This needs to be done separately from the build rules because we want to +# compute the hash over all inputs going into the image and derive the +# "version.txt" file from it. +def _image_deps(mode, name, malicious = False): + extra_rootfs_deps = { + "dev": {":rootfs/allow_console_root": "/etc/allow_console_root:0644"}, + "prod": {}, + } + deps = { + "bootfs": { + # base layer + ":{}_rootfs-tree.tar".format(name): "/", + + # We will install extra_boot_args onto the system, after substuting + # the hash of the root filesystem into it. Add the template (before + # substitution) as a dependency nevertheless such that changes + # to the template file are reflected in the overall version hash + # (the root_hash must include the version hash, it cannot be the + # other way around). + ":bootloader/extra_boot_args.template": "/boot/extra_boot_args.template:0644", + }, + "rootfs": { + # base layer + ":{}_rootfs-tree.tar".format(name): "/", + + # additional files to install + "//:canister_sandbox": "/opt/ic/bin/canister_sandbox:0755", + "//:ic-btc-adapter": "/opt/ic/bin/ic-btc-adapter:0755", + "//:ic-consensus-pool-util": "/opt/ic/bin/ic-consensus-pool-util:0755", + "//:ic-canister-http-adapter": "/opt/ic/bin/ic-canister-http-adapter:0755", + "//:ic-crypto-csp": "/opt/ic/bin/ic-crypto-csp:0755", + "//:ic-regedit": "/opt/ic/bin/ic-regedit:0755", + "//:ic-recovery": "/opt/ic/bin/ic-recovery:0755", + "//:orchestrator": "/opt/ic/bin/orchestrator:0755", + ("//:malicious_replica" if malicious else "//:replica"): "/opt/ic/bin/replica:0755", + "//:sandbox_launcher": "/opt/ic/bin/sandbox_launcher:0755", + "//:state-tool": "/opt/ic/bin/state-tool:0755", + "//:vsock_agent": "/opt/ic/bin/vsock_agent:0755", + "//ic-os/guestos/src:infogetty": "/opt/ic/bin/infogetty:0755", + "//ic-os/guestos/src:prestorecon": "/opt/ic/bin/prestorecon:0755", + }, + } + deps["rootfs"].update(extra_rootfs_deps[mode]) + return deps + +def icos_build(name, mode = None, malicious = False, visibility = None): + """ + An ICOS build parameterized by mode. + + Args: + name: Name for the generated filegroup. You can access individual artifacts via + //ic-os/guestos:{name}_{artifact}, i.e. //ic-os/guestos:dev_disk-img.tar.gz + mode: dev or prod. If not specified, will use the value of `name` + malicious: if True, bundle the `malicious_replica` + visibility: See Bazel documentation + """ + if mode == None: + mode = name + + nm = lambda s: "{}_{}".format(name, s) + + lbl = lambda s: ":" + nm(s) + + image_deps = _image_deps(mode, name, malicious) + + dev_rootfs_args = [] + + if mode == "dev": + dev_rootfs_args = ["--extra-dockerfile", "ic-os/guestos/rootfs/Dockerfile.dev", "--dev-root-ca", "ic-os/guestos/dev-root-ca.crt"] + + docker_tar( + visibility = visibility, + name = nm("rootfs-tree.tar"), + src = ":rootfs", + dep = native.glob(["rootfs/**"] + ["dev-root-ca.crt"] if mode == "dev" else []), + extra_args_before = dev_rootfs_args, + extra_args_after = [ + "--build-arg", + "ROOT_PASSWORD=root", + "--build-arg", + "BASE_IMAGE=" + img_bases[mode], + ], + # The image is pretty big, therefore it is usually much faster to just rebuild it instead of fetching from the cache. + # TODO(IDX-2221): remove this when CI jobs and bazel infrastructure will run in the same clusters. + tags = ["no-remote-cache"], + target_compatible_with = [ + "@platforms//os:linux", + ], + ) + + ext4_image( + name = nm("partition-config.tar"), + partition_size = "100M", + target_compatible_with = [ + "@platforms//os:linux", + ], + ) + + tar_extract( + visibility = visibility, + name = nm("file_contexts"), + src = lbl("rootfs-tree.tar"), + path = "etc/selinux/default/contexts/files/file_contexts", + target_compatible_with = [ + "@platforms//os:linux", + ], + ) + + summary_sha256sum( + name = nm("version.txt"), + inputs = image_deps, + suffix = "-dev" if mode == "dev" else "", + ) + + ext4_image( + name = nm("partition-boot.tar"), + src = lbl("rootfs-tree.tar"), + # Take the dependency list declared above, and add in the "version.txt" + # as well as the generated extra_boot_args file in the correct place. + extra_files = { + k: v + for k, v in ( + image_deps["bootfs"].items() + [ + (lbl("version.txt"), "/boot/version.txt:0644"), + (lbl("extra_boot_args"), "/boot/extra_boot_args:0644"), + ] + ) + if k != ":bootloader/extra_boot_args.template" + # additional files to install + if v != "/" + }, + file_contexts = lbl("file_contexts"), + partition_size = "1G", + subdir = "boot/", + target_compatible_with = [ + "@platforms//os:linux", + ], + ) + + ext4_image( + name = nm("partition-root-unsigned.tar"), + src = lbl("rootfs-tree.tar"), + # Take the dependency list declared above, and add in the "version.txt" + # at the correct place. + extra_files = { + k: v + for k, v in (image_deps["rootfs"].items() + [(lbl("version.txt"), "/opt/ic/share/version.txt:0644")]) + if v != "/" + }, + file_contexts = lbl("file_contexts"), + partition_size = "3G", + strip_paths = [ + "/run", + "/boot", + ], + # The image is pretty big, therefore it is usually much faster to just rebuild it instead of fetching from the cache. + # TODO(IDX-2221): remove this when CI jobs and bazel infrastructure will run in the same clusters. + tags = ["no-remote-cache"], + target_compatible_with = [ + "@platforms//os:linux", + ], + ) + + native.genrule( + name = nm("partition-root-sign"), + srcs = [lbl("partition-root-unsigned.tar")], + outs = [nm("partition-root.tar"), nm("partition-root-hash")], + cmd = "$(location //toolchains/sysimage:verity_sign.py) -i $< -o $(location {}) -r $(location {})".format(nm("partition-root.tar"), nm("partition-root-hash")), + executable = False, + tools = ["//toolchains/sysimage:verity_sign.py"], + ) + + native.genrule( + name = nm("extra_boot_args_root_hash"), + srcs = [ + ":bootloader/extra_boot_args.template", + lbl("partition-root-hash"), + ], + outs = [lbl("extra_boot_args")], + cmd = "sed -e s/ROOT_HASH/$$(cat $(location {}))/ < $(location :bootloader/extra_boot_args.template) > $@".format(lbl("partition-root-hash")), + ) + + disk_image( + name = nm("disk-img.tar"), + layout = ":partitions.csv", + partitions = [ + "//ic-os/bootloader:partition-esp.tar", + "//ic-os/bootloader:partition-grub.tar", + lbl("partition-config.tar"), + lbl("partition-boot.tar"), + lbl("partition-root.tar"), + ], + # The image is pretty big, therefore it is usually much faster to just rebuild it instead of fetching from the cache. + # TODO(IDX-2221): remove this when CI jobs and bazel infrastructure will run in the same clusters. + tags = ["no-remote-cache"], + target_compatible_with = [ + "@platforms//os:linux", + ], + ) + + upgrade_image( + name = nm("upgrade.tar"), + boot_partition = lbl("partition-boot.tar"), + root_partition = lbl("partition-root.tar"), + # The image is pretty big, therefore it is usually much faster to just rebuild it instead of fetching from the cache. + # TODO(IDX-2221): remove this when CI jobs and bazel infrastructure will run in the same clusters. + tags = ["no-remote-cache"], + target_compatible_with = [ + "@platforms//os:linux", + ], + version_file = lbl("version.txt"), + ) + + native.genrule( + name = nm("disk-img.tar_zst"), + srcs = [lbl("disk-img.tar")], + outs = [lbl("disk-img.tar.zst")], + cmd = "zstd --threads=0 -10 -f -z $< -o $@", + # The image is pretty big, therefore it is usually much faster to just rebuild it instead of fetching from the cache. + # TODO(IDX-2221): remove this when CI jobs and bazel infrastructure will run in the same clusters. + tags = ["no-remote-cache"], + ) + + sha256sum( + name = nm("disk-img.tar.zst.sha256"), + srcs = [lbl("disk-img.tar.zst")], + ) + + native.genrule( + name = nm("disk-img.tar_gz"), + srcs = [lbl("disk-img.tar")], + outs = [lbl("disk-img.tar.gz")], + cmd = "gzip -9 < $< > $@", + # The image is pretty big, therefore it is usually much faster to just rebuild it instead of fetching from the cache. + # TODO(IDX-2221): remove this when CI jobs and bazel infrastructure will run in the same clusters. + tags = ["no-remote-cache"], + ) + + native.genrule( + name = nm("upgrade.tar_zst"), + srcs = [lbl("upgrade.tar")], + outs = [lbl("upgrade.tar.zst")], + cmd = "zstd --threads=0 -10 -f -z $< -o $@", + # The image is pretty big, therefore it is usually much faster to just rebuild it instead of fetching from the cache. + # TODO(IDX-2221): remove this when CI jobs and bazel infrastructure will run in the same clusters. + tags = ["no-remote-cache"], + ) + + upload_artifacts( + name = nm("upload_guestos"), + inputs = [ + lbl("disk-img.tar.zst"), + lbl("upgrade.tar.zst"), + ], + remote_subdir = "ic-os/guestos", + ) + + urls_test( + name = nm("upload_guestos_test"), + inputs = [lbl("upload_guestos")], + ) + + native.py_binary( + name = nm("launch_single_vm"), + main = "launch_single_vm.py", + srcs = [ + ":launch_single_vm.py", + "//ic-os/guestos/tests:ictools.py", + ], + env = { + "DATA_PREFIX": nm(""), + }, + data = [ + lbl("disk-img.tar.zst.sha256"), + lbl("upload_guestos"), + lbl("version.txt"), + "//rs/prep:ic-prep", + ], + tags = ["manual"], + ) + + native.filegroup( + name = name, + srcs = [nm("disk-img.tar.zst"), nm("disk-img.tar.gz"), nm("upgrade.tar.zst")], + visibility = visibility, + ) diff --git a/ic-os/guestos/BUILD.bazel b/ic-os/guestos/BUILD.bazel index c9ce334bee2..146f748d183 100644 --- a/ic-os/guestos/BUILD.bazel +++ b/ic-os/guestos/BUILD.bazel @@ -1,272 +1,78 @@ -load("//gitlab-ci/src/artifacts:upload.bzl", "upload_artifacts", "urls_test") -load("//toolchains/sysimage:toolchain.bzl", "disk_image", "docker_tar", "ext4_image", "sha256sum", "summary_sha256sum", "tar_extract", "upgrade_image") -load("@aspect_bazel_lib//lib:output_files.bzl", "output_files") +load("//ic-os:defs.bzl", "icos_build") -docker_tar( - name = "rootfs-tree.tar", - src = ":rootfs", - dep = glob(["rootfs/**"]), - extra_args = [ - "--build-arg", - "ROOT_PASSWORD=root", - "--build-arg", - "BASE_IMAGE=dfinity/guestos-base-dev@sha256:9a3c0621cc26e5b94fa3517c55c8f57eee901cc0e2d209e0bf9588bfff561709", - ], - # The image is pretty big, therefore it is usually much faster to just rebuild it instead of fetching from the cache. - # TODO(IDX-2221): remove this when CI jobs and bazel infrastructure will run in the same clusters. - tags = ["no-remote-cache"], - target_compatible_with = [ - "@platforms//os:linux", - ], -) - -ext4_image( - name = "partition-config.tar", - partition_size = "100M", - target_compatible_with = [ - "@platforms//os:linux", - ], -) +icos_build(name = "dev") -tar_extract( - name = "file_contexts", - src = ":rootfs-tree.tar", - path = "etc/selinux/default/contexts/files/file_contexts", - target_compatible_with = [ - "@platforms//os:linux", - ], +icos_build( + name = "dev-malicious", + malicious = True, + mode = "dev", ) -# Declare the dependencies that we will have for the built filesystem images. -# This needs to be done separately from the build rules because we want to -# compute the hash over all inputs going into the image and derive the -# "version.txt" file from it. -IC_OS_IMAGE_DEPS = { - "bootfs": { - # base layer - ":rootfs-tree.tar": "/", - - # We will install extra_boot_args onto the system, after substuting - # the hash of the root filesystem into it. Add the template (before - # substitution) as a dependency nevertheless such that changes - # to the template file are reflected in the overall version hash - # (the root_hash must include the version hash, it cannot be the - # other way around). - ":bootloader/extra_boot_args.template": "/boot/extra_boot_args.template:0644", - }, - "rootfs": { - # base layer - ":rootfs-tree.tar": "/", - - # additional files to install - "//:canister_sandbox": "/opt/ic/bin/canister_sandbox:0755", - "//:ic-btc-adapter": "/opt/ic/bin/ic-btc-adapter:0755", - "//:ic-consensus-pool-util": "/opt/ic/bin/ic-consensus-pool-util:0755", - "//:ic-canister-http-adapter": "/opt/ic/bin/ic-canister-http-adapter:0755", - "//:ic-crypto-csp": "/opt/ic/bin/ic-crypto-csp:0755", - "//:ic-regedit": "/opt/ic/bin/ic-regedit:0755", - "//:ic-recovery": "/opt/ic/bin/ic-recovery:0755", - "//:orchestrator": "/opt/ic/bin/orchestrator:0755", - "//:replica": "/opt/ic/bin/replica:0755", - "//:sandbox_launcher": "/opt/ic/bin/sandbox_launcher:0755", - "//:state-tool": "/opt/ic/bin/state-tool:0755", - "//:vsock_agent": "/opt/ic/bin/vsock_agent:0755", - "//ic-os/guestos/src:infogetty": "/opt/ic/bin/infogetty:0755", - "//ic-os/guestos/src:prestorecon": "/opt/ic/bin/prestorecon:0755", - # NB: this should be there for dev images only - ":rootfs/allow_console_root": "/etc/allow_console_root:0644", - }, -} - -# This macro expands to compute the hash sum of all inputs individually, and -# then aggregating the hash into a summary hash. -summary_sha256sum( - name = "version.txt", - inputs = IC_OS_IMAGE_DEPS, - suffix = "-dev", -) - -ext4_image( - name = "partition-boot.tar", - src = ":rootfs-tree.tar", - # Take the dependency list declared above, and add in the "version.txt" - # as well as the generated extra_boot_args file in the correct place. - extra_files = { - k: v - for k, v in ( - IC_OS_IMAGE_DEPS["bootfs"].items() + [ - (":version.txt", "/boot/version.txt:0644"), - (":extra_boot_args", "/boot/extra_boot_args:0644"), - ] - ) - if k != ":bootloader/extra_boot_args.template" - # additional files to install - if v != "/" - }, - file_contexts = ":file_contexts", - partition_size = "1G", - subdir = "boot/", - target_compatible_with = [ - "@platforms//os:linux", - ], -) - -ext4_image( - name = "partition-root-unsigned.tar", - src = ":rootfs-tree.tar", - # Take the dependency list declared above, and add in the "version.txt" - # at the correct place. - extra_files = { - k: v - for k, v in (IC_OS_IMAGE_DEPS["rootfs"].items() + [(":version.txt", "/opt/ic/share/version.txt:0644")]) - if v != "/" - }, - file_contexts = ":file_contexts", - partition_size = "3G", - strip_paths = [ - "/run", - "/boot", - ], - # The image is pretty big, therefore it is usually much faster to just rebuild it instead of fetching from the cache. - # TODO(IDX-2221): remove this when CI jobs and bazel infrastructure will run in the same clusters. - tags = ["no-remote-cache"], - target_compatible_with = [ - "@platforms//os:linux", - ], -) - -genrule( - name = "partition-root-sign", - srcs = [":partition-root-unsigned.tar"], - outs = [ - "partition-root.tar", - "partition-root-hash", - ], - cmd = "$(location //toolchains/sysimage:verity_sign.py) -i $(location :partition-root-unsigned.tar) -o $(location partition-root.tar) -r $(location partition-root-hash)", - tools = ["//toolchains/sysimage:verity_sign.py"], -) +icos_build(name = "prod") -genrule( - name = "extra_boot_args_root_hash", - srcs = [ - ":bootloader/extra_boot_args.template", - ":partition-root-hash", - ], - outs = [":extra_boot_args"], - cmd = "sed -e s/ROOT_HASH/$$(cat $(location :partition-root-hash))/ < $(location :bootloader/extra_boot_args.template) > $(location extra_boot_args)", -) +exports_files([ + "rootfs/opt/ic/share/ic.json5.template", +]) -disk_image( - name = "disk.img.tar", - layout = ":partitions.csv", - partitions = [ - "//ic-os/bootloader:partition-esp.tar", - "//ic-os/bootloader:partition-grub.tar", - ":partition-config.tar", - ":partition-boot.tar", - ":partition-root.tar", - ], - # The image is pretty big, therefore it is usually much faster to just rebuild it instead of fetching from the cache. - # TODO(IDX-2221): remove this when CI jobs and bazel infrastructure will run in the same clusters. - tags = ["no-remote-cache"], - target_compatible_with = [ - "@platforms//os:linux", - ], +alias( + name = "disk_image_dev", + actual = "dev_disk-img.tar.zst", ) -upgrade_image( - name = "upgrade.tar", - boot_partition = ":partition-boot.tar", - root_partition = ":partition-root.tar", - # The image is pretty big, therefore it is usually much faster to just rebuild it instead of fetching from the cache. - # TODO(IDX-2221): remove this when CI jobs and bazel infrastructure will run in the same clusters. - tags = ["no-remote-cache"], - target_compatible_with = [ - "@platforms//os:linux", - ], - version_file = ":version.txt", +alias( + name = "upgrade_image_dev", + actual = "dev_upgrade.tar.zst", ) -genrule( - name = "disk.img.tar_zstd", - srcs = [":disk.img.tar"], - outs = [":disk.img.tar.zstd"], - cmd = "zstd --threads=0 -10 -f -z $(location disk.img.tar) -o \"$@\"", - # The image is pretty big, therefore it is usually much faster to just rebuild it instead of fetching from the cache. - # TODO(IDX-2221): remove this when CI jobs and bazel infrastructure will run in the same clusters. - tags = ["no-remote-cache"], -) +# alias( +# name = "urls_test_dev", +# actual = "dev_upload_guestos_test", +# ) -sha256sum( - name = "disk.img.tar.zstd.sha256", - srcs = [":disk.img.tar.zstd"], +alias( + name = "launch_single_vm_dev", + actual = "dev_launch_single_vm", + tags = ["manual"], ) -genrule( - name = "upgrade.tar_zstd", - srcs = [":upgrade.tar"], - outs = [":upgrade.tar.zstd"], - cmd = "zstd --threads=0 -10 -f -z $(location upgrade.tar) -o \"$@\"", - # The image is pretty big, therefore it is usually much faster to just rebuild it instead of fetching from the cache. - # TODO(IDX-2221): remove this when CI jobs and bazel infrastructure will run in the same clusters. - tags = ["no-remote-cache"], +alias( + name = "disk_image_dev-malicious", + actual = "dev-malicious_disk-img.tar.zst", ) -upload_artifacts( - name = "upload_guestos", - inputs = [ - ":disk.img.tar.zstd", - ":upgrade.tar.zstd", - ], - remote_subdir = "ic-os/guestos", +alias( + name = "upgrade_image_dev-malicious", + actual = "dev-malicious_upgrade.tar.zst", ) -exports_files([ - "rootfs/opt/ic/share/ic.json5.template", -]) +# alias( +# name = "urls_test_dev-malicious", +# actual = "dev-malicious_upload_guestos_test", +# ) -urls_test( - name = "upload_guestos_test", - inputs = [":upload_guestos"], +alias( + name = "launch_single_vm_dev-malicious", + actual = "dev-malicious_launch_single_vm", + tags = ["manual"], ) -# I have no clue why I cannot use these labels directly in the data section of the -# python binary below. When I try to, the files simply do not get added to the -# runfiles of the executable. -genrule( - name = "version_wrap", - srcs = [":version.txt"], - outs = [":version.txt.wrapped"], - cmd = "cat $(location version.txt) > \"$@\"", +alias( + name = "disk_image_prod", + actual = "prod_disk-img.tar.zst", ) -genrule( - name = "hash_wrap", - srcs = [":disk.img.tar.zstd.sha256"], - outs = [":disk.img.tar.zstd.sha256.wrapped"], - cmd = "cat $(location disk.img.tar.zstd.sha256) > \"$@\"", +alias( + name = "upgrade_image_prod", + actual = "prod_upgrade.tar.zst", ) -# Filter output from the upload rule to depend on required file only. -# This allows bazel to only build and upload what is required. -output_files( - name = "disk.img.tar.zstd.url", - paths = ["ic-os/guestos/upload_guestos_disk.img.tar.zstd.url"], - tags = ["manual"], - target = ":upload_guestos", -) +# alias( +# name = "urls_test_prod", +# actual = "prod_upload_guestos_test", +# ) -py_binary( - name = "launch_single_vm", - srcs = [ - ":launch_single_vm.py", - "//ic-os/guestos/tests:ictools.py", - ], - data = [ - ":disk.img.tar.zstd.sha256.wrapped", - ":disk.img.tar.zstd.url", - ":version.txt.wrapped", - "//rs/prep:ic-prep", - ], +alias( + name = "launch_single_vm_prod", + actual = "prod_launch_single_vm", tags = ["manual"], ) diff --git a/ic-os/guestos/launch_single_vm.py b/ic-os/guestos/launch_single_vm.py index 0d23778e9de..0ac3b02b30e 100644 --- a/ic-os/guestos/launch_single_vm.py +++ b/ic-os/guestos/launch_single_vm.py @@ -14,11 +14,13 @@ def main(argv): argv = FLAGS(argv) - version = open("ic-os/guestos/version.txt.wrapped", "r").read().strip() + prefix = os.getenv("DATA_PREFIX", "") - img_hash = open("ic-os/guestos/disk.img.tar.zstd.sha256.wrapped", "r").read().strip() + version = open(f"ic-os/guestos/{prefix}version.txt", "r").read().strip() - url = open("ic-os/guestos/upload_guestos_disk.img.tar.zstd.url", "r").read().split()[0] + img_hash = open(f"ic-os/guestos/{prefix}disk-img.tar.zst.sha256", "r").read().strip() + + url = open(f"ic-os/guestos/{prefix}upload_guestos_{prefix}disk-img.tar.zst.url", "r").read().split()[0] FLAGS.ic_prep_bin = "rs/prep/ic-prep" diff --git a/rs/tests/run-system-tests.py b/rs/tests/run-system-tests.py index e2efc17d4ed..69a830fa59f 100755 --- a/rs/tests/run-system-tests.py +++ b/rs/tests/run-system-tests.py @@ -251,6 +251,7 @@ def main( SHELL_WRAPPER = os.getenv("SHELL_WRAPPER", default=SHELL_WRAPPER_DEFAULT) SSH_KEY_DIR = os.getenv("SSH_KEY_DIR", default=None) IC_VERSION_ID = os.getenv("IC_VERSION_ID", default="") + GUESTOS_VERSION_OVERRIDE = os.getenv("GUESTOS_VERSION_OVERRIDE", default=IC_VERSION_ID) JOB_ID = os.getenv("CI_JOB_ID", default=None) CI_PARENT_PIPELINE_SOURCE = os.getenv("CI_PARENT_PIPELINE_SOURCE", default="") CI_PIPELINE_SOURCE = os.getenv("CI_PIPELINE_SOURCE", default="") @@ -312,8 +313,9 @@ def main( ) TEST_ES_HOSTNAMES = replace_symbols(text=TEST_ES_HOSTNAMES, symbols_to_replace=["`", "'", " "], replace_with="") + GUESTOS_BASE_URL = f"http://download.proxy-global.dfinity.network:8080/ic/{GUESTOS_VERSION_OVERRIDE}" IMG_BASE_URL = f"http://download.proxy-global.dfinity.network:8080/ic/{IC_VERSION_ID}" - IC_OS_DEV_IMG_SHA256, IC_OS_DEV_IMG_URL = get_ic_os_image_sha(f"{IMG_BASE_URL}/guest-os/disk-img-dev/") + IC_OS_DEV_IMG_SHA256, IC_OS_DEV_IMG_URL = get_ic_os_image_sha(f"{GUESTOS_BASE_URL}/guest-os/disk-img-dev/") BOUNDARY_NODE_IMG_SHA256, BOUNDARY_NODE_IMG_URL = get_ic_os_image_sha(f"{IMG_BASE_URL}/boundary-os/disk-img-dev/") BOUNDARY_NODE_SNP_IMG_SHA256, BOUNDARY_NODE_SNP_IMG_URL = get_ic_os_image_sha( f"{IMG_BASE_URL}/boundary-os/disk-img-snp-dev/" diff --git a/toolchains/sysimage/build_disk_image.py b/toolchains/sysimage/build_disk_image.py index 11439bab4cb..a384f16fbbe 100755 --- a/toolchains/sysimage/build_disk_image.py +++ b/toolchains/sysimage/build_disk_image.py @@ -145,6 +145,9 @@ def main(): for index in range(len(partition_files)): write_partition_image_from_tar(gpt_entries[index], disk_image, tarfile.open(partition_files[index], mode="r:")) + # Provide additional space for vda10, the final partition, for immediate QEMU use + subprocess.run(["truncate", "--size", "50G", disk_image], check=True) + subprocess.run( [ "tar", diff --git a/toolchains/sysimage/docker_tar.py b/toolchains/sysimage/docker_tar.py index 118ff330be7..858925cbd66 100755 --- a/toolchains/sysimage/docker_tar.py +++ b/toolchains/sysimage/docker_tar.py @@ -324,6 +324,7 @@ def make_argparser(): default=[], help="Extra vars to pass to docker", ) + parser.add_argument("--dev-root-ca", type=str, default="", help="Root CA for the dev image") parser.add_argument( "build_args", metavar="build_args", @@ -363,6 +364,13 @@ def main(): for var in extra_vars: extra_args.append("--build-arg") extra_args.append(var) + if args.dev_root_ca != "": + try: + ca_contents = open(args.dev_root_ca, "r").read() + extra_args.append("--build-arg") + extra_args.append("DEV_ROOT_CA=" + ca_contents) + except FileNotFoundError: + print(f"WARNING: Skipping --dev-ca-root file {args.dev_root_ca}, which does not exist") image_hash = docker_build(extra_args, extra_dockerfile) # Extract and flatten all layers, build an in-memory pseudo filesystem diff --git a/toolchains/sysimage/toolchain.bzl b/toolchains/sysimage/toolchain.bzl index 4fc95b7ce84..a631f55c3f8 100644 --- a/toolchains/sysimage/toolchain.bzl +++ b/toolchains/sysimage/toolchain.bzl @@ -9,7 +9,7 @@ def _docker_tar_impl(ctx): ctx.actions.run( executable = tool.path, - arguments = ["-o", out.path, "--", in_dir.path] + ctx.attr.extra_args, + arguments = ["-o", out.path] + ctx.attr.extra_args_before + ["--", in_dir.path] + ctx.attr.extra_args_after, inputs = [in_dir] + ctx.files.dep, outputs = [out], tools = [tool], @@ -27,7 +27,8 @@ docker_tar = rule( "dep": attr.label_list( allow_files = True, ), - "extra_args": attr.string_list(), + "extra_args_before": attr.string_list(), + "extra_args_after": attr.string_list(), "_build_docker_save_tool": attr.label( allow_files = True, default = ":docker_tar.py", @@ -296,7 +297,7 @@ def _sha256sum_impl(ctx): command = "cat {} | sha256sum | sed -e 's/ \\+-/{}/' > {}".format(input_paths, ctx.attr.suffix, out.path), ) - return [DefaultInfo(files = depset([out]))] + return [DefaultInfo(files = depset([out]), runfiles = ctx.runfiles([out]))] sha256sum = rule( implementation = _sha256sum_impl, @@ -328,7 +329,7 @@ def summary_sha256sum(name, inputs, suffix = ""): all_deps.update(deps) labels = [] for dep in all_deps.keys(): - label = dep.split(":")[1] + ".sha256" + label = name + "@" + dep.split(":")[1] + ".sha256" sha256sum( name = label, srcs = [dep],