Build guestos in pipeline using bazel (re-merge)

yral-dapp · Oct 12, 2022 · e89cd15 · e89cd15
1 parent 6165596
commit e89cd15
Show file tree

Hide file tree

Showing 14 changed files with 460 additions and 365 deletions.
diff --git a/.gitignore b/.gitignore
@@ -68,12 +68,13 @@ dist-newstyle
 # JUnit files
 test_report.xml
 
-# IC-OS disk images
+# IC-OS disk image files
 *-os.iso
 *os.img
 disk.img
 *-img.tar.gz
 *-img.tar.zst
+dev-root-ca.crt
 
 # IC-OS binaries
 infogetty

diff --git a/gitlab-ci/config/46--guest-os-build--guest-os-diskimg.yml b/gitlab-ci/config/46--guest-os-build--guest-os-diskimg.yml
@@ -1,33 +1,29 @@
 guest-os-diskimg:
+  needs: []
   extends:
     - .ubuntu-docker-k8s-protected
     - .rules-parent-pipeline-autorun-on-trigger
+  tags:
+    - dfinity
+    - ubuntu
+    - zh
   stage: guest-os-build
-  needs:
-    - job: cargo-build-release-linux-native
-      artifacts: false
   variables:
       BUILD_EXTRA_ARGS: ""
       BUILD_EXTRA_SUFFIX: ""
+  artifacts:
+    paths:
+      - version.txt
   script:
     - |
-      "$CI_PROJECT_DIR"/gitlab-ci/src/artifacts/rclone_download.py \
-          --git-rev="$CI_COMMIT_SHA" --remote-path="release" \
-          --out="artifacts/release"
-
-      if [[ "$CI_JOB_NAME" == *"-malicious" ]]; then
-          "$CI_PROJECT_DIR"/gitlab-ci/src/artifacts/rclone_download.py \
-              --git-rev="$CI_COMMIT_SHA" --remote-path="release-malicious" \
-              --out="artifacts/release-malicious"
-      fi
-
       "${CI_PROJECT_DIR}/gitlab-ci/tools/job-driver.py" guest-os-diskimg
 
 # Build dev image.
 #
 # This is the same as the prod image with the following differences:
 # - it has password "root" set for root to allow console login
 guest-os-diskimg-dev:
+  needs: []
   extends:
     - guest-os-diskimg
   variables:
@@ -41,11 +37,6 @@ guest-os-diskimg-dev:
 guest-os-diskimg-dev-malicious:
   extends:
     - guest-os-diskimg
-  needs:
-    - job: cargo-build-release-linux-native
-      artifacts: false
-    - job: cargo-build-release-linux-native-malicious
-      artifacts: false
   variables:
       BUILD_EXTRA_ARGS: "-t dev -p root"
       BUILD_EXTRA_SUFFIX: "-dev-malicious"

diff --git a/gitlab-ci/config/47--guest-os-test--e2e-scalability.yml b/gitlab-ci/config/47--guest-os-test--e2e-scalability.yml
@@ -2,15 +2,15 @@ e2e-scalability-suite-test:
   extends:
     - .e2e-test-base-k8s
   needs:
-    - job: guest-os-diskimg
-      artifacts: false
+    - job: guest-os-diskimg-dev
     - job: guest-os-updateimg-build
       artifacts: false
   artifacts:
     paths:
     - scalability/
   script:
     - |
+      ICOS_VER=$(cat version.txt)
       # release
       "$CI_PROJECT_DIR"/gitlab-ci/src/artifacts/rclone_download.py \
           --git-rev="$CI_COMMIT_SHA" --remote-path="release" \
@@ -20,9 +20,6 @@ e2e-scalability-suite-test:
           --git-rev="$CI_COMMIT_SHA" --remote-path="canisters" \
           --out="artifacts/canisters"
 
-      GIT_REV=$(git rev-parse HEAD)
-      GIT_LATEST_WITH_IC_OS=$("${CI_PROJECT_DIR}/gitlab-ci/src/artifacts/newest_sha_with_disk_image.sh" "$GIT_REV")
-
       mkdir -p gitlab-runner-tmp; cd gitlab-runner-tmp
 
       ls "${CI_PROJECT_DIR}/artifacts/canisters"
@@ -52,7 +49,7 @@ e2e-scalability-suite-test:
 
       pipenv run python3 "common/tests/e2e-scalability-tests.py" \
           --timeout "$GUEST_OS_TEST_TIMEOUT" \
-          --ic_os_version "$GIT_LATEST_WITH_IC_OS" \
+          --ic_os_version "$ICOS_VER" \
           --ic_admin_bin "${CI_PROJECT_DIR}/artifacts/release/ic-admin" \
           --nns_canisters "${CI_PROJECT_DIR}/artifacts/canisters/" \
           --ic_prep_bin "${CI_PROJECT_DIR}/artifacts/release/ic-prep" \

diff --git a/gitlab-ci/config/47--guest-os-test--guest-os-e2e-test.yml b/gitlab-ci/config/47--guest-os-test--guest-os-e2e-test.yml
@@ -33,6 +33,8 @@
 .run-farm-based-test: &run-farm-based-test |
   IC_VERSION_ID=$CI_COMMIT_SHA
   export IC_VERSION_ID
+  GUESTOS_VERSION_OVERRIDE=$(cat version.txt)
+  export GUESTOS_VERSION_OVERRIDE
   #XXX(marko): we need to change this
   # binaries are needed for run-system-tests.py
   "$CI_PROJECT_DIR"/gitlab-ci/src/artifacts/rclone_download.py \
@@ -71,7 +73,6 @@
     - job: guest-os-diskimg
       artifacts: false
     - job: guest-os-diskimg-dev
-      artifacts: false
     - job: guest-os-diskimg-dev-malicious
       artifacts: false
     - job: guest-os-updateimg-build
@@ -139,34 +140,36 @@ tecdsa-pre-master:
     SUITE_NAME: "tecdsa_pre_master"
 
 test-prod-test-driver-e2e:
- extends:
-  - .system-tests
-  - .rules-test-prod-test-driver-e2e
- script:
-   - |
-     # IC_VERSION_ID is needed for run-system-tests.py.
-     IC_VERSION_ID="$CI_COMMIT_SHA"
-     export IC_VERSION_ID
-     # required binary artifacts
-     mkdir -p artifacts/release
-     pushd artifacts/release
-     curl -sfS --retry 5 --retry-delay 10 \
-        "http://download.proxy.dfinity.systems:8080/ic/$CI_COMMIT_SHA/release/prod-test-driver.gz" -O
-     popd
-      # required canisters
-     "$CI_PROJECT_DIR"/gitlab-ci/src/artifacts/rclone_download.py \
-         --git-rev="${IC_VERSION_ID}" --remote-path="canisters" \
-         --out="artifacts/canisters"
-     WORKING_DIR="${CI_PROJECT_DIR}/working_dir/"
-     mkdir -p "$WORKING_DIR"
-     # Logs produced during execution of the system tests are stored away in a structured way, i.e. one file per test.
-     # Unstructured logs, i.e. those which bypassed the logger and were outputed directly to stdout/stderr,
-     # are forwarded to a separate file.
-     # All lines produced by the logger share the same structure: e.g. "Nov 10 13:20:30.931 INFO ...".
-     # Hence, the fourth column can be used to distinguish structured from unstructured logs.
-     $SHELL_WRAPPER "${CI_PROJECT_DIR}/rs/tests/tests_e2e/test_prod_test_driver_e2e.py" \
-           | tee "${WORKING_DIR}/unstructured-logs.log" \
-           | awk '$4 ~ /CRIT|ERRO|WARN|INFO|DEBG|TRCE/'
+  extends:
+   - .system-tests
+   - .rules-test-prod-test-driver-e2e
+  script:
+    - |
+      # IC_VERSION_ID is needed for run-system-tests.py.
+      IC_VERSION_ID="$CI_COMMIT_SHA"
+      export IC_VERSION_ID
+      GUESTOS_VERSION_OVERRIDE=$(cat version.txt)
+      export GUESTOS_VERSION_OVERRIDE
+      # required binary artifacts
+      mkdir -p artifacts/release
+      pushd artifacts/release
+      curl -sfS --retry 5 --retry-delay 10 \
+         "http://download.proxy.dfinity.systems:8080/ic/$CI_COMMIT_SHA/release/prod-test-driver.gz" -O
+      popd
+       # required canisters
+      "$CI_PROJECT_DIR"/gitlab-ci/src/artifacts/rclone_download.py \
+          --git-rev="${IC_VERSION_ID}" --remote-path="canisters" \
+          --out="artifacts/canisters"
+      WORKING_DIR="${CI_PROJECT_DIR}/working_dir/"
+      mkdir -p "$WORKING_DIR"
+      # Logs produced during execution of the system tests are stored away in a structured way, i.e. one file per test.
+      # Unstructured logs, i.e. those which bypassed the logger and were outputed directly to stdout/stderr,
+      # are forwarded to a separate file.
+      # All lines produced by the logger share the same structure: e.g. "Nov 10 13:20:30.931 INFO ...".
+      # Hence, the fourth column can be used to distinguish structured from unstructured logs.
+      $SHELL_WRAPPER "${CI_PROJECT_DIR}/rs/tests/tests_e2e/test_prod_test_driver_e2e.py" \
+            | tee "${WORKING_DIR}/unstructured-logs.log" \
+            | awk '$4 ~ /CRIT|ERRO|WARN|INFO|DEBG|TRCE/'
 
 rosetta-pre-master:
   extends: .system-tests
@@ -192,8 +195,7 @@ upgrade-compatibility-pre-master:
   needs:
     - job: cargo-build-release-linux-native
       artifacts: false
-    - job: guest-os-diskimg
-      artifacts: false
+    - job: guest-os-diskimg-dev
     - job: guest-os-updateimg-build
       artifacts: false
     - job: guest-os-updateimg-build-dev
@@ -238,8 +240,10 @@ spec-compliance-pre-master:
           --out="artifacts/canisters"
 
       # requirement of run-system-tests.py
-      IC_VERSION_ID="$CI_COMMIT_SHA"
+      IC_VERSION_ID=$CI_COMMIT_SHA
       export IC_VERSION_ID
+      GUESTOS_VERSION_OVERRIDE=$(cat version.txt)
+      export GUESTOS_VERSION_OVERRIDE
 
       "$SHELL_WRAPPER" nix-shell -p "(import ./nix {}).ic-ref" --run "
         ${CI_PROJECT_DIR}/rs/tests/run-system-tests.py --suite=spec_compliance

diff --git a/gitlab-ci/config/53--host-os-build--build-setupos.yml b/gitlab-ci/config/53--host-os-build--build-setupos.yml
@@ -1,7 +1,6 @@
 setup-os-diskimg:
   needs:
     - job: guest-os-diskimg
-      artifacts: false
     - job: host-os-diskimg
       artifacts: false
   extends:
@@ -15,10 +14,11 @@ setup-os-diskimg:
       VERSION=$(git rev-parse HEAD)
       export VERSION
       echo "Build ID: ${VERSION}"
+      ICOS_VERSION=$(cat version.txt)
 
       cd "${CI_PROJECT_DIR}"/ic-os/setupos
 
-      "$CI_PROJECT_DIR"/gitlab-ci/src/artifacts/rclone_download.py --remote-path=guest-os --out=guestos --git-rev="${VERSION}"
+      "$CI_PROJECT_DIR"/gitlab-ci/src/artifacts/rclone_download.py --remote-path=guest-os --out=guestos --git-rev="${ICOS_VERSION}"
       "$CI_PROJECT_DIR"/gitlab-ci/src/artifacts/rclone_download.py --remote-path=host-os --out=hostos --git-rev="${VERSION}"
 
       BUILD_OUT="build-out/disk-img"

diff --git a/gitlab-ci/src/job_scripts/guest_os_diskimg.py b/gitlab-ci/src/job_scripts/guest_os_diskimg.py
@@ -1,6 +1,6 @@
 import logging
-from os import environ
 from os import getenv
+from pathlib import Path
 
 from ci import buildevent
 from ci import cwd
@@ -11,26 +11,20 @@
 def run():
     build_extra_suffix = getenv("BUILD_EXTRA_SUFFIX", "")
     build_out = f"build-out/disk-img{build_extra_suffix}"
-    build_tmp = f"build-tmp{build_extra_suffix}"
     upload_target = f"guest-os/disk-img{build_extra_suffix}"
     version = ENV.build_id
     logging.info(f"Build ID: {version}")
 
+    build_mode = "dev" if "-dev" in build_extra_suffix else "prod"
+    malicious_mode = "-malicious" if "-malicious" in build_extra_suffix else ""
+
     with cwd("ic-os/guestos"):
-        # lib/guest-os-diskimg.sh fails if these are not set, which they aren't when running locally
-        offline_defaults = {"BUILD_EXTRA_ARGS": "", "BUILD_EXTRA_SUFFIX": "", "CI_JOB_ID": ""}
-        # override those variables with the already-set ones if there are any
-        script_env = {**offline_defaults, **environ.copy()}
-
-        sh(
-            f"{ENV.top}/gitlab-ci/src/job_scripts/lib/guest-os-diskimg.sh",
-            build_out,
-            build_tmp,
-            upload_target,
-            version,
-            environ.get("CDPRNET", ""),
-            env=script_env,
-        )
+        sh(f"{ENV.top}/gitlab-ci/src/job_scripts/lib/guest-os-diskimg.sh", build_out, build_mode, malicious_mode)
+
+        version_txt = Path(f"{ENV.top}/version.txt")
+        if version_txt.is_file():
+            legacy_version = version
+            version = version_txt.read_text().strip()
 
         if ENV.is_gitlab:
             with buildevent("rclone"):
@@ -41,5 +35,14 @@ def run():
                     build_out,
                     upload_target,
                 )
+            if legacy_version:
+                with buildevent("rclone"):
+                    sh(
+                        f"{ENV.top}/gitlab-ci/src/artifacts/rclone_upload.py",
+                        f"--version={legacy_version}",
+                        "--verbose",
+                        build_out,
+                        upload_target,
+                    )
 
     logging.info(f"Build ID (real or fake git revision): {version}")
diff --git a/gitlab-ci/src/job_scripts/lib/guest-os-diskimg.sh b/gitlab-ci/src/job_scripts/lib/guest-os-diskimg.sh
@@ -8,50 +8,28 @@
 set -euo pipefail
 
 BUILD_OUT=${1:-"build-out/disk-img"}
-BUILD_TMP=${2:-"build-tmp"}
-UPLOAD_TARGET=${3:-"guest-os/disk-img"}
-VERSION=${4:-$(git rev-parse --verify HEAD)}
-CDPRNET=${5:-"cdpr05"}
+BUILD_MODE=${2:-"dev"}
+MALICIOUS_MODE=${3:-}
 
 ROOT_DIR=$(git rev-parse --show-toplevel)
 groups
 
-cd "$ROOT_DIR" || exit 1
-for f in replica orchestrator canister_sandbox sandbox_launcher vsock_agent state-tool ic-consensus-pool-util ic-crypto-csp ic-regedit ic-recovery ic-btc-adapter ic-canister-http-adapter; do
-    gunzip -c -d artifacts/release/$f.gz >artifacts/release/$f
-done
+cd "$ROOT_DIR"/ic-os/guestos || exit 1
+mkdir -p "$BUILD_OUT"
 
-# if we are building the malicious image, use malicious replica version
-if [[ "${BUILD_EXTRA_SUFFIX}" =~ "malicious" ]]; then
-    gunzip -c -d artifacts/release-malicious/replica.gz >artifacts/release/replica
-    chmod +x artifacts/release/replica
+if [ "$BUILD_MODE" = "dev" ]; then
+    ln -sfv "$DEV_ROOT_CA" "$PWD/dev-root-ca.crt"
 fi
 
-cd "$ROOT_DIR"/ic-os/guestos || exit 1
-mkdir -p "$BUILD_OUT" "$BUILD_TMP"
-echo "$VERSION" >"${BUILD_TMP}/version.txt"
-
-if [ -z "$CI_JOB_ID" ]; then
-    # shellcheck disable=SC2086  # Expanding BUILD_EXTRA_ARGS into multiple parameters
-    ./scripts/build-disk-image.sh -o "${BUILD_TMP}/disk.img" -v "$VERSION" -x ../../artifacts/release/ $BUILD_EXTRA_ARGS
-    tar --sort=name --owner=root:0 --group=root:0 --mtime='UTC 2020-01-01' --sparse \
-        -cvzf "${BUILD_OUT}/disk-img.tar.gz" -C "$BUILD_TMP" disk.img version.txt
-    tar --sort=name --owner=root:0 --group=root:0 --mtime='UTC 2020-01-01' --sparse \
-        -cvf "${BUILD_OUT}/disk-img.tar.zst" --use-compress-program="zstd --threads=0 -10" \
-        -C "$BUILD_TMP" disk.img version.txt
-    ls -lah "$BUILD_TMP"
-else
-    # shellcheck disable=SC2086  # Expanding BUILD_EXTRA_ARGS into multiple parameters
-    buildevents cmd "${ROOT_PIPELINE_ID}" "${CI_JOB_ID}" build-disk-img -- \
-        ./scripts/build-disk-image.sh -o "${BUILD_TMP}/disk.img" -v "$VERSION" -x ../../artifacts/release/ $BUILD_EXTRA_ARGS
-    buildevents cmd "$ROOT_PIPELINE_ID" "$CI_JOB_ID" tar-build-out -- \
-        tar --sort=name --owner=root:0 --group=root:0 --mtime='UTC 2020-01-01' --sparse \
-        -cvzf "${BUILD_OUT}/disk-img.tar.gz" -C "$BUILD_TMP" disk.img version.txt
-    buildevents cmd "$ROOT_PIPELINE_ID" "$CI_JOB_ID" tar-build-out -- \
-        tar --sort=name --owner=root:0 --group=root:0 --mtime='UTC 2020-01-01' --sparse \
-        -cvf "${BUILD_OUT}/disk-img.tar.zst" --use-compress-program="zstd --threads=0 -10" \
-        -C "$BUILD_TMP" disk.img version.txt
-    ls -lah "$BUILD_TMP"
+BUILD_MODE="${BUILD_MODE}${MALICIOUS_MODE}"
+
+bazel build --config=ci //ic-os/guestos:"$BUILD_MODE"_disk-img.tar_gz //ic-os/guestos:"$BUILD_MODE"_disk-img.tar_zst
 
+cp -fv "$ROOT_DIR"/$(bazel cquery --output=files //ic-os/guestos:"$BUILD_MODE"_disk-img.tar_gz) "$BUILD_OUT"/disk-img.tar.gz
+cp -fv "$ROOT_DIR"/$(bazel cquery --output=files //ic-os/guestos:"$BUILD_MODE"_disk-img.tar_zst) "$BUILD_OUT"/disk-img.tar.zst
+
+if [ -n "${CI_JOB_ID:-}" ]; then
     "$ROOT_DIR"/gitlab-ci/src/artifacts/openssl-sign.sh "$BUILD_OUT"
 fi
+
+cat "$ROOT_DIR"/$(bazel cquery --output=files //ic-os/guestos:"$BUILD_MODE"_version.txt) >"$ROOT_DIR"/version.txt