From b4fbac2a66f519237287c4f304e1ef20c17265be Mon Sep 17 00:00:00 2001 From: Gerald Pinder Date: Fri, 29 Nov 2024 00:16:10 -0500 Subject: [PATCH] feat(rechunk): Add the ability to rechunk an image --- .github/workflows/build-pr.yml | 122 ++++++--- .github/workflows/build.yml | 138 ++++++---- Cargo.lock | 1 + Cargo.toml | 5 + Earthfile | 16 +- .../test-repo/recipes/recipe-rechunk.yml | 58 ++++ justfile | 14 + process/Cargo.toml | 3 +- process/drivers.rs | 89 +++++- process/drivers/docker_driver.rs | 28 +- process/drivers/opts.rs | 4 + process/drivers/opts/build.rs | 3 + process/drivers/opts/rechunk.rs | 47 ++++ process/drivers/opts/run.rs | 5 +- process/drivers/podman_driver.rs | 159 +++++++++-- process/drivers/skopeo_driver.rs | 24 ++ process/drivers/traits.rs | 254 +++++++++++++++++- process/drivers/types.rs | 71 +++++ process/signal_handler.rs | 11 +- src/commands/build.rs | 115 ++++++-- src/commands/generate_iso.rs | 10 +- template/templates/Containerfile.j2 | 3 +- 22 files changed, 996 insertions(+), 184 deletions(-) create mode 100644 integration-tests/test-repo/recipes/recipe-rechunk.yml create mode 100644 process/drivers/opts/rechunk.rs diff --git a/.github/workflows/build-pr.yml b/.github/workflows/build-pr.yml index 5ee029eb..99695b05 100644 --- a/.github/workflows/build-pr.yml +++ b/.github/workflows/build-pr.yml @@ -82,7 +82,7 @@ jobs: earthly --ci --push -P +prebuild build-images: - timeout-minutes: 30 + timeout-minutes: 60 runs-on: ubuntu-latest if: github.repository == 'blue-build/cli' needs: @@ -253,9 +253,9 @@ jobs: BB_BUILDKIT_CACHE_GHA: true run: just test-docker-build - arm64-build: - timeout-minutes: 40 - runs-on: ubuntu-latest + rechunk-build: + timeout-minutes: 20 + runs-on: ubuntu-24.04 permissions: contents: read packages: write @@ -266,11 +266,9 @@ jobs: uses: ublue-os/remove-unwanted-software@v6 - uses: sigstore/cosign-installer@v3.3.0 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 with: - install: true + install-dir: /usr/bin + use-sudo: true - uses: actions-rust-lang/setup-rust-toolchain@v1 @@ -290,11 +288,14 @@ jobs: GH_TOKEN: ${{ github.token }} GH_PR_EVENT_NUMBER: ${{ github.event.number }} COSIGN_PRIVATE_KEY: ${{ secrets.TEST_SIGNING_SECRET }} - BB_BUILDKIT_CACHE_GHA: true - run: just test-arm64-build + run: | + just install-debug-all-features + cd integration-tests/test-repo + export CARGO_HOME=$HOME/.cargo + sudo -E $CARGO_HOME/bin/bluebuild build --push -vv --rechunk recipes/recipe-rechunk.yml - docker-build-external-login: - timeout-minutes: 20 + arm64-build: + timeout-minutes: 40 runs-on: ubuntu-latest permissions: contents: read @@ -312,13 +313,6 @@ jobs: with: install: true - - name: Docker Login - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ github.token }} - - uses: actions-rust-lang/setup-rust-toolchain@v1 - uses: actions/checkout@v4 @@ -334,12 +328,13 @@ jobs: - name: Run Build env: + GH_TOKEN: ${{ github.token }} GH_PR_EVENT_NUMBER: ${{ github.event.number }} COSIGN_PRIVATE_KEY: ${{ secrets.TEST_SIGNING_SECRET }} BB_BUILDKIT_CACHE_GHA: true - run: just test-docker-build-external-login + run: just test-arm64-build - docker-build-oauth-login: + docker-build-external-login: timeout-minutes: 20 runs-on: ubuntu-latest permissions: @@ -348,16 +343,6 @@ jobs: id-token: write steps: - - name: Google Auth - id: auth - uses: "google-github-actions/auth@v2" - with: - token_format: "access_token" - service_account: ${{ secrets.SERVICE_ACCOUNT }} - project_id: bluebuild-oidc - create_credentials_file: false - workload_identity_provider: ${{ secrets.WORKLOAD_IDENTITY }} - - name: Maximize build space uses: ublue-os/remove-unwanted-software@v6 @@ -368,15 +353,14 @@ jobs: with: install: true - - uses: actions-rust-lang/setup-rust-toolchain@v1 - - - name: Docker Auth - id: docker-auth - uses: "docker/login-action@v3" + - name: Docker Login + uses: docker/login-action@v3 with: - username: "oauth2accesstoken" - password: "${{ steps.auth.outputs.access_token }}" - registry: us-east1-docker.pkg.dev + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} + + - uses: actions-rust-lang/setup-rust-toolchain@v1 - uses: actions/checkout@v4 with: @@ -394,7 +378,65 @@ jobs: GH_PR_EVENT_NUMBER: ${{ github.event.number }} COSIGN_PRIVATE_KEY: ${{ secrets.TEST_SIGNING_SECRET }} BB_BUILDKIT_CACHE_GHA: true - run: just test-docker-build-oauth-login + run: just test-docker-build-external-login + + # Free trial is over + # docker-build-oauth-login: + # timeout-minutes: 20 + # runs-on: ubuntu-latest + # permissions: + # contents: read + # packages: write + # id-token: write + + # steps: + # - name: Google Auth + # id: auth + # uses: "google-github-actions/auth@v2" + # with: + # token_format: "access_token" + # service_account: ${{ secrets.SERVICE_ACCOUNT }} + # project_id: bluebuild-oidc + # create_credentials_file: false + # workload_identity_provider: ${{ secrets.WORKLOAD_IDENTITY }} + + # - name: Maximize build space + # uses: ublue-os/remove-unwanted-software@v6 + + # - uses: sigstore/cosign-installer@v3.3.0 + + # - name: Set up Docker Buildx + # uses: docker/setup-buildx-action@v3 + # with: + # install: true + + # - uses: actions-rust-lang/setup-rust-toolchain@v1 + + # - name: Docker Auth + # id: docker-auth + # uses: "docker/login-action@v3" + # with: + # username: "oauth2accesstoken" + # password: "${{ steps.auth.outputs.access_token }}" + # registry: us-east1-docker.pkg.dev + + # - uses: actions/checkout@v4 + # with: + # fetch-depth: 0 + # ref: ${{github.event.pull_request.head.ref}} + # repository: ${{github.event.pull_request.head.repo.full_name}} + + # - name: Expose GitHub Runtime + # uses: crazy-max/ghaction-github-runtime@v3 + + # - uses: extractions/setup-just@v1 + + # - name: Run Build + # env: + # GH_PR_EVENT_NUMBER: ${{ github.event.number }} + # COSIGN_PRIVATE_KEY: ${{ secrets.TEST_SIGNING_SECRET }} + # BB_BUILDKIT_CACHE_GHA: true + # run: just test-docker-build-oauth-login podman-build: timeout-minutes: 20 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index cd330e6b..a192c66d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -290,35 +290,28 @@ jobs: BB_BUILDKIT_CACHE_GHA: true run: just test-docker-build - arm64-build: - timeout-minutes: 40 - runs-on: ubuntu-latest + rechunk-build: + timeout-minutes: 20 + runs-on: ubuntu-24.04 permissions: contents: read packages: write id-token: write - if: github.repository == 'blue-build/cli' - needs: - - build-scripts steps: - name: Maximize build space uses: ublue-os/remove-unwanted-software@v6 - uses: sigstore/cosign-installer@v3.3.0 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 with: - install: true + install-dir: /usr/bin + use-sudo: true - uses: actions-rust-lang/setup-rust-toolchain@v1 - uses: actions/checkout@v4 with: - fetch-depth: 0 - ref: ${{github.event.pull_request.head.ref}} - repository: ${{github.event.pull_request.head.repo.full_name}} + ref: main - name: Expose GitHub Runtime uses: crazy-max/ghaction-github-runtime@v3 @@ -330,11 +323,14 @@ jobs: GH_TOKEN: ${{ github.token }} GH_PR_EVENT_NUMBER: ${{ github.event.number }} COSIGN_PRIVATE_KEY: ${{ secrets.TEST_SIGNING_SECRET }} - BB_BUILDKIT_CACHE_GHA: true - run: just test-arm64-build + run: | + just install-debug-all-features + cd integration-tests/test-repo + export CARGO_HOME=$HOME/.cargo + sudo -E $CARGO_HOME/bin/bluebuild build --push -vv --rechunk recipes/recipe-rechunk.yml - docker-build-external-login: - timeout-minutes: 60 + arm64-build: + timeout-minutes: 40 runs-on: ubuntu-latest permissions: contents: read @@ -355,20 +351,13 @@ jobs: with: install: true - - name: Docker Login - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ github.token }} - - uses: actions-rust-lang/setup-rust-toolchain@v1 - # Setup repo and add caching - uses: actions/checkout@v4 with: - ref: main - + fetch-depth: 0 + ref: ${{github.event.pull_request.head.ref}} + repository: ${{github.event.pull_request.head.repo.full_name}} - name: Expose GitHub Runtime uses: crazy-max/ghaction-github-runtime@v3 @@ -377,33 +366,24 @@ jobs: - name: Run Build env: + GH_TOKEN: ${{ github.token }} GH_PR_EVENT_NUMBER: ${{ github.event.number }} COSIGN_PRIVATE_KEY: ${{ secrets.TEST_SIGNING_SECRET }} BB_BUILDKIT_CACHE_GHA: true - run: just test-docker-build-external-login + run: just test-arm64-build - docker-build-oauth-login: + docker-build-external-login: timeout-minutes: 60 runs-on: ubuntu-latest permissions: contents: read packages: write id-token: write + if: github.repository == 'blue-build/cli' needs: - build-scripts - if: github.repository == 'blue-build/cli' steps: - - name: Google Auth - id: auth - uses: "google-github-actions/auth@v2" - with: - token_format: "access_token" - service_account: ${{ secrets.SERVICE_ACCOUNT }} - project_id: bluebuild-oidc - create_credentials_file: false - workload_identity_provider: ${{ secrets.WORKLOAD_IDENTITY }} - - name: Maximize build space uses: ublue-os/remove-unwanted-software@v6 @@ -414,20 +394,21 @@ jobs: with: install: true - - uses: actions-rust-lang/setup-rust-toolchain@v1 - - - name: Docker Auth - id: docker-auth - uses: "docker/login-action@v3" + - name: Docker Login + uses: docker/login-action@v3 with: - username: "oauth2accesstoken" - password: "${{ steps.auth.outputs.access_token }}" - registry: us-east1-docker.pkg.dev + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} + + - uses: actions-rust-lang/setup-rust-toolchain@v1 + # Setup repo and add caching - uses: actions/checkout@v4 with: ref: main + - name: Expose GitHub Runtime uses: crazy-max/ghaction-github-runtime@v3 @@ -438,7 +419,66 @@ jobs: GH_PR_EVENT_NUMBER: ${{ github.event.number }} COSIGN_PRIVATE_KEY: ${{ secrets.TEST_SIGNING_SECRET }} BB_BUILDKIT_CACHE_GHA: true - run: just test-docker-build-oauth-login + run: just test-docker-build-external-login + + # Free trial is over + # docker-build-oauth-login: + # timeout-minutes: 60 + # runs-on: ubuntu-latest + # permissions: + # contents: read + # packages: write + # id-token: write + # needs: + # - build-scripts + # if: github.repository == 'blue-build/cli' + + # steps: + # - name: Google Auth + # id: auth + # uses: "google-github-actions/auth@v2" + # with: + # token_format: "access_token" + # service_account: ${{ secrets.SERVICE_ACCOUNT }} + # project_id: bluebuild-oidc + # create_credentials_file: false + # workload_identity_provider: ${{ secrets.WORKLOAD_IDENTITY }} + + # - name: Maximize build space + # uses: ublue-os/remove-unwanted-software@v6 + + # - uses: sigstore/cosign-installer@v3.3.0 + + # - name: Set up Docker Buildx + # uses: docker/setup-buildx-action@v3 + # with: + # install: true + + # - uses: actions-rust-lang/setup-rust-toolchain@v1 + + # - name: Docker Auth + # id: docker-auth + # uses: "docker/login-action@v3" + # with: + # username: "oauth2accesstoken" + # password: "${{ steps.auth.outputs.access_token }}" + # registry: us-east1-docker.pkg.dev + + # - uses: actions/checkout@v4 + # with: + # ref: main + + # - name: Expose GitHub Runtime + # uses: crazy-max/ghaction-github-runtime@v3 + + # - uses: extractions/setup-just@v1 + + # - name: Run Build + # env: + # GH_PR_EVENT_NUMBER: ${{ github.event.number }} + # COSIGN_PRIVATE_KEY: ${{ secrets.TEST_SIGNING_SECRET }} + # BB_BUILDKIT_CACHE_GHA: true + # run: just test-docker-build-oauth-login podman-build: timeout-minutes: 60 diff --git a/Cargo.lock b/Cargo.lock index bb7b2995..73d6f3bb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -369,6 +369,7 @@ dependencies = [ "jsonschema", "log", "miette", + "nix", "oci-distribution", "open", "os_info", diff --git a/Cargo.toml b/Cargo.toml index 3c0a4848..a7560fe9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,6 +18,7 @@ colored = "2" indexmap = { version = "2", features = ["serde"] } indicatif = { version = "0.17", features = ["improved_unicode"] } log = "0.4" +nix = { version = "0.29" } oci-distribution = { version = "0.11", default-features = false } reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"] } miette = "7" @@ -86,6 +87,7 @@ indexmap.workspace = true indicatif.workspace = true log.workspace = true miette = { workspace = true, features = ["fancy", "syntect-highlighter"] } +nix = { workspace = true, features = ["user"] } oci-distribution.workspace = true reqwest.workspace = true semver.workspace = true @@ -122,6 +124,9 @@ validate = [ prune = [ "blue-build-process-management/prune" ] +rechunk = [ + "blue-build-process-management/rechunk" +] [dev-dependencies] rusty-hook = "0.11" diff --git a/Earthfile b/Earthfile index 583846c4..44f9dd86 100644 --- a/Earthfile +++ b/Earthfile @@ -165,9 +165,9 @@ blue-build-cli: END IF [ "$TARGETARCH" = "arm64" ] - DO --pass-args +INSTALL --OUT_DIR="/usr/bin/" --BUILD_TARGET="aarch64-unknown-linux-gnu" + DO +INSTALL --OUT_DIR="/usr/bin/" --BUILD_TARGET="aarch64-unknown-linux-gnu" --RELEASE=$RELEASE ELSE - DO --pass-args +INSTALL --OUT_DIR="/usr/bin/" --BUILD_TARGET="x86_64-unknown-linux-gnu" + DO +INSTALL --OUT_DIR="/usr/bin/" --BUILD_TARGET="x86_64-unknown-linux-gnu" --RELEASE=$RELEASE END RUN mkdir -p /bluebuild @@ -211,9 +211,9 @@ blue-build-cli-distrobox: FROM "$IMAGE:$EARTHLY_GIT_HASH-distrobox-prebuild-$TARGETARCH" IF [ "$TARGETARCH" = "arm64" ] - DO --pass-args +INSTALL --OUT_DIR="/usr/bin/" --BUILD_TARGET="aarch64-unknown-linux-musl" + DO +INSTALL --OUT_DIR="/usr/bin/" --BUILD_TARGET="aarch64-unknown-linux-musl" ELSE - DO --pass-args +INSTALL --OUT_DIR="/usr/bin/" --BUILD_TARGET="x86_64-unknown-linux-musl" + DO +INSTALL --OUT_DIR="/usr/bin/" --BUILD_TARGET="x86_64-unknown-linux-musl" END DO --pass-args +SAVE_IMAGE --SUFFIX="-distrobox" @@ -228,9 +228,9 @@ installer: ARG TARGETARCH IF [ "$TARGETARCH" = "arm64" ] - DO --pass-args +INSTALL --OUT_DIR="/out/" --BUILD_TARGET="aarch64-unknown-linux-musl" + DO +INSTALL --OUT_DIR="/out/" --BUILD_TARGET="aarch64-unknown-linux-musl" ELSE - DO --pass-args +INSTALL --OUT_DIR="/out/" --BUILD_TARGET="x86_64-unknown-linux-musl" + DO +INSTALL --OUT_DIR="/out/" --BUILD_TARGET="x86_64-unknown-linux-musl" END COPY install.sh /install.sh @@ -274,9 +274,9 @@ INSTALL: ARG RELEASE="true" IF [ "$TAGGED" = "true" ] - COPY --platform=native --pass-args +install/bluebuild $OUT_DIR + COPY --platform=native (+install/bluebuild --BUILD_TARGET=$BUILD_TARGET --RELEASE=$RELEASE) $OUT_DIR ELSE - COPY --platform=native --pass-args +install-all-features/bluebuild $OUT_DIR + COPY --platform=native (+install-all-features/bluebuild --BUILD_TARGET=$BUILD_TARGET --RELEASE=$RELEASE) $OUT_DIR END SAVE_IMAGE: diff --git a/integration-tests/test-repo/recipes/recipe-rechunk.yml b/integration-tests/test-repo/recipes/recipe-rechunk.yml new file mode 100644 index 00000000..363988ec --- /dev/null +++ b/integration-tests/test-repo/recipes/recipe-rechunk.yml @@ -0,0 +1,58 @@ +--- +# yaml-language-server: $schema=https://schema.blue-build.org/recipe-v1.json +name: cli/test-rechunk +description: This is my personal OS image. +base-image: ghcr.io/ublue-os/silverblue-main +image-version: latest +stages: + - from-file: stages.yml +modules: + - from-file: akmods.yml + - from-file: flatpaks.yml + + - type: files + files: + - source: usr + destination: /usr + + - type: script + scripts: + - example.sh + + - type: rpm-ostree + repos: + - https://copr.fedorainfracloud.org/coprs/atim/starship/repo/fedora-%OS_VERSION%/atim-starship-fedora-%OS_VERSION%.repo + install: + - micro + - starship + remove: + - firefox + - firefox-langpacks + + - type: signing + + - type: test-module + source: local + + - type: containerfile + containerfiles: + - labels + snippets: + - RUN echo "This is a snippet" && ostree container commit + + - type: copy + from: alpine-test + src: /test.txt + dest: / + - type: copy + from: ubuntu-test + src: /test.txt + dest: / + - type: copy + from: debian-test + src: /test.txt + dest: / + - type: copy + from: fedora-test + src: /test.txt + dest: / diff --git a/justfile b/justfile index 7c972b59..238731dc 100644 --- a/justfile +++ b/justfile @@ -126,6 +126,12 @@ should_push := if env('GITHUB_ACTIONS', '') != '' { '' } +cargo_bin := if env('CARGO_HOME', '') != '' { + x"${CARGO_HOME:-}/bin" +} else { + x"$HOME/.cargo/bin" +} + # Run all integration tests integration-tests: test-docker-build test-arm64-build test-podman-build test-buildah-build test-generate-iso-image test-generate-iso-recipe @@ -141,6 +147,14 @@ test-docker-build: install-debug-all-features -vv \ recipes/recipe.yml recipes/recipe-gts.yml +test-rechunk-build: install-debug-all-features + cd integration-tests/test-repo \ + && sudo {{ cargo_bin }}/bluebuild build \ + {{ should_push }} \ + -vv \ + --rechunk \ + recipes/recipe-rechunk.yml + # Run arm integration test test-arm64-build: install-debug-all-features cd integration-tests/test-repo \ diff --git a/process/Cargo.toml b/process/Cargo.toml index 1ca22bc8..e2d5ee57 100644 --- a/process/Cargo.toml +++ b/process/Cargo.toml @@ -17,7 +17,6 @@ indicatif-log-bridge = "0.2" lenient_semver = "0.4" log4rs = { version = "1", features = ["background_rotation"] } nu-ansi-term = { version = "0.50", features = ["gnu_legacy"] } -nix = { version = "0.29", features = ["signal"] } once_cell = "1" os_pipe = { version = "1", features = ["io_safety"] } rand = "0.8" @@ -33,6 +32,7 @@ indicatif.workspace = true indexmap.workspace = true log.workspace = true miette.workspace = true +nix = { workspace = true, features = ["signal", "user"] } oci-distribution.workspace = true reqwest.workspace = true semver = { workspace = true, features = ["serde"] } @@ -55,3 +55,4 @@ workspace = true sigstore = ["dep:tokio", "dep:sigstore"] validate = ["dep:tokio"] prune = [] +rechunk = [] diff --git a/process/drivers.rs b/process/drivers.rs index eb66f792..0efa904d 100644 --- a/process/drivers.rs +++ b/process/drivers.rs @@ -233,23 +233,23 @@ impl Driver { Ok(os_version) } - fn get_build_driver() -> BuildDriverType { + pub fn get_build_driver() -> BuildDriverType { impl_driver_type!(SELECTED_BUILD_DRIVER) } - fn get_inspect_driver() -> InspectDriverType { + pub fn get_inspect_driver() -> InspectDriverType { impl_driver_type!(SELECTED_INSPECT_DRIVER) } - fn get_signing_driver() -> SigningDriverType { + pub fn get_signing_driver() -> SigningDriverType { impl_driver_type!(SELECTED_SIGNING_DRIVER) } - fn get_run_driver() -> RunDriverType { + pub fn get_run_driver() -> RunDriverType { impl_driver_type!(SELECTED_RUN_DRIVER) } - fn get_ci_driver() -> CiDriverType { + pub fn get_ci_driver() -> CiDriverType { impl_driver_type!(SELECTED_CI_DRIVER) } } @@ -287,8 +287,7 @@ fn get_version_run_image(oci_ref: &Reference) -> Result { .pull(true) .remove(true) .build(), - ) - .into_diagnostic()?; + )?; progress.finish_and_clear(); Logger::multi_progress().remove(&progress); @@ -395,11 +394,11 @@ macro_rules! impl_run_driver { } impl RunDriver for Driver { - fn run(opts: &RunOpts) -> std::io::Result { + fn run(opts: &RunOpts) -> Result { impl_run_driver!(run(opts)) } - fn run_output(opts: &RunOpts) -> std::io::Result { + fn run_output(opts: &RunOpts) -> Result { impl_run_driver!(run_output(opts)) } } @@ -450,3 +449,75 @@ impl CiDriver for Driver { impl_ci_driver!(default_ci_file_path()) } } + +#[cfg(feature = "rechunk")] +impl ContainerMountDriver for Driver { + fn create_container(image: &Reference) -> Result { + PodmanDriver::create_container(image) + } + + fn remove_container(container_id: &types::ContainerId) -> Result<()> { + PodmanDriver::remove_container(container_id) + } + + fn remove_image(image: &Reference) -> Result<()> { + PodmanDriver::remove_image(image) + } + + fn mount_container(container_id: &types::ContainerId) -> Result { + PodmanDriver::mount_container(container_id) + } + + fn unmount_container(container_id: &types::ContainerId) -> Result<()> { + PodmanDriver::unmount_container(container_id) + } + + fn remove_volume(volume_id: &str) -> Result<()> { + PodmanDriver::remove_volume(volume_id) + } +} + +#[cfg(feature = "rechunk")] +impl OciCopy for Driver { + fn copy_oci_dir( + oci_dir: &self::types::OciDir, + registry: &oci_distribution::Reference, + ) -> Result<()> { + SkopeoDriver::copy_oci_dir(oci_dir, registry) + } +} + +#[cfg(feature = "rechunk")] +impl RechunkDriver for Driver { + fn rechunk(opts: &opts::RechunkOpts) -> Result> { + PodmanDriver::rechunk(opts) + } + + fn prune_image( + _mount: &types::MountId, + _container: &types::ContainerId, + _raw_image: &Reference, + _opts: &opts::RechunkOpts<'_>, + ) -> Result<(), miette::Error> { + unimplemented!("Use the `rechunk` function instead"); + } + + fn create_ostree_commit( + _mount: &types::MountId, + _ostree_cache_id: &str, + _container: &types::ContainerId, + _raw_image: &Reference, + _opts: &opts::RechunkOpts<'_>, + ) -> Result<()> { + unimplemented!("Use the `rechunk` function instead"); + } + + fn rechunk_image( + _ostree_cache_id: &str, + _temp_dir_str: &str, + _current_dir: &str, + _opts: &opts::RechunkOpts<'_>, + ) -> Result<()> { + unimplemented!("Use the `rechunk` function instead"); + } +} diff --git a/process/drivers/docker_driver.rs b/process/drivers/docker_driver.rs index 069538a1..d257989c 100644 --- a/process/drivers/docker_driver.rs +++ b/process/drivers/docker_driver.rs @@ -33,7 +33,7 @@ use crate::{ types::Platform, }, logging::CommandLogging, - signal_handler::{add_cid, remove_cid, ContainerId, ContainerRuntime}, + signal_handler::{add_cid, remove_cid, ContainerRuntime, ContainerSignalId}, }; #[derive(Debug, Deserialize)] @@ -427,32 +427,34 @@ fn get_metadata_cache(opts: &GetMetadataOpts) -> Result { } impl RunDriver for DockerDriver { - fn run(opts: &RunOpts) -> std::io::Result { + fn run(opts: &RunOpts) -> Result { trace!("DockerDriver::run({opts:#?})"); - let cid_path = TempDir::new()?; + let cid_path = TempDir::new().into_diagnostic()?; let cid_file = cid_path.path().join("cid"); - let cid = ContainerId::new(&cid_file, ContainerRuntime::Docker, false); + let cid = ContainerSignalId::new(&cid_file, ContainerRuntime::Docker, false); add_cid(&cid); - let status = docker_run(opts, &cid_file).build_status(&*opts.image, "Running container")?; + let status = docker_run(opts, &cid_file) + .build_status(&*opts.image, "Running container") + .into_diagnostic()?; remove_cid(&cid); Ok(status) } - fn run_output(opts: &RunOpts) -> std::io::Result { + fn run_output(opts: &RunOpts) -> Result { trace!("DockerDriver::run({opts:#?})"); - let cid_path = TempDir::new()?; + let cid_path = TempDir::new().into_diagnostic()?; let cid_file = cid_path.path().join("cid"); - let cid = ContainerId::new(&cid_file, ContainerRuntime::Docker, false); + let cid = ContainerSignalId::new(&cid_file, ContainerRuntime::Docker, false); add_cid(&cid); - let output = docker_run(opts, &cid_file).output()?; + let output = docker_run(opts, &cid_file).output().into_diagnostic()?; remove_cid(&cid); @@ -469,6 +471,7 @@ fn docker_run(opts: &RunOpts, cid_file: &Path) -> Command { if opts.privileged => "--privileged", if opts.remove => "--rm", if opts.pull => "--pull=always", + if let Some(user) = opts.user.as_ref() => format!("--user={user}"), for RunOptsVolume { path_or_vol_name, container_path } in opts.volumes.iter() => [ "--volume", format!("{path_or_vol_name}:{container_path}"), @@ -477,13 +480,6 @@ fn docker_run(opts: &RunOpts, cid_file: &Path) -> Command { "--env", format!("{key}={value}"), ], - |command| { - match (opts.uid, opts.gid) { - (Some(uid), None) => cmd!(command, "-u", format!("{uid}")), - (Some(uid), Some(gid)) => cmd!(command, "-u", format!("{}:{}", uid, gid)), - _ => {} - } - }, &*opts.image, for arg in opts.args.iter() => &**arg, ); diff --git a/process/drivers/opts.rs b/process/drivers/opts.rs index 3cbd025b..cae73821 100644 --- a/process/drivers/opts.rs +++ b/process/drivers/opts.rs @@ -3,12 +3,16 @@ use clap::ValueEnum; pub use build::*; pub use ci::*; pub use inspect::*; +#[cfg(feature = "rechunk")] +pub use rechunk::*; pub use run::*; pub use signing::*; mod build; mod ci; mod inspect; +#[cfg(feature = "rechunk")] +mod rechunk; mod run; mod signing; diff --git a/process/drivers/opts/build.rs b/process/drivers/opts/build.rs index ed2ae238..c1142eed 100644 --- a/process/drivers/opts/build.rs +++ b/process/drivers/opts/build.rs @@ -20,6 +20,9 @@ pub struct BuildOpts<'scope> { #[builder(default)] pub platform: Platform, + + #[builder(default)] + pub host_network: bool, } #[derive(Debug, Clone, Builder)] diff --git a/process/drivers/opts/rechunk.rs b/process/drivers/opts/rechunk.rs new file mode 100644 index 00000000..ebb6653d --- /dev/null +++ b/process/drivers/opts/rechunk.rs @@ -0,0 +1,47 @@ +use std::{borrow::Cow, path::Path}; + +use bon::Builder; + +use crate::drivers::types::Platform; + +use super::CompressionType; + +#[derive(Debug, Clone, Builder)] +#[builder(on(Cow<'_, str>, into))] +pub struct RechunkOpts<'scope> { + pub image: Cow<'scope, str>, + + #[builder(into)] + pub containerfile: Cow<'scope, Path>, + + #[builder(default)] + pub platform: Platform, + pub version: Cow<'scope, str>, + pub name: Cow<'scope, str>, + pub description: Cow<'scope, str>, + pub base_digest: Cow<'scope, str>, + pub base_image: Cow<'scope, str>, + pub repo: Cow<'scope, str>, + + /// The list of tags for the image being built. + #[builder(default, into)] + pub tags: Vec>, + + /// Enable pushing the image. + #[builder(default)] + pub push: bool, + + /// Enable retry logic for pushing. + #[builder(default)] + pub retry_push: bool, + + /// Number of times to retry pushing. + /// + /// Defaults to 1. + #[builder(default = 1)] + pub retry_count: u8, + + /// The compression type to use when pushing. + #[builder(default)] + pub compression: CompressionType, +} diff --git a/process/drivers/opts/run.rs b/process/drivers/opts/run.rs index 7b3a23f0..cd86d04d 100644 --- a/process/drivers/opts/run.rs +++ b/process/drivers/opts/run.rs @@ -15,8 +15,9 @@ pub struct RunOpts<'scope> { #[builder(default, into)] pub volumes: Vec>, - pub uid: Option, - pub gid: Option, + + #[builder(into)] + pub user: Option>, #[builder(default)] pub privileged: bool, diff --git a/process/drivers/podman_driver.rs b/process/drivers/podman_driver.rs index 30c0d1f6..7317a904 100644 --- a/process/drivers/podman_driver.rs +++ b/process/drivers/podman_driver.rs @@ -10,7 +10,7 @@ use blue_build_utils::{cmd, credentials::Credentials}; use cached::proc_macro::cached; use colored::Colorize; use indicatif::{ProgressBar, ProgressStyle}; -use log::{debug, error, info, trace, warn}; +use log::{debug, error, info, trace}; use miette::{bail, miette, IntoDiagnostic, Report, Result}; use oci_distribution::Reference; use semver::Version; @@ -24,7 +24,13 @@ use crate::{ BuildDriver, DriverVersion, InspectDriver, RunDriver, }, logging::{CommandLogging, Logger}, - signal_handler::{add_cid, remove_cid, ContainerId, ContainerRuntime}, + signal_handler::{add_cid, remove_cid, ContainerRuntime, ContainerSignalId}, +}; + +#[cfg(feature = "rechunk")] +use super::{ + types::{ContainerId, MountId}, + ContainerMountDriver, RechunkDriver, }; #[derive(Deserialize, Debug, Clone)] @@ -136,6 +142,7 @@ impl BuildDriver for PodmanDriver { opts.platform.to_string(), ], "--pull=true", + if opts.host_network => "--net=host", format!("--layers={}", !opts.squash), "-f", &*opts.containerfile, @@ -334,39 +341,151 @@ fn get_metadata_cache(opts: &GetMetadataOpts) -> Result { .inspect(|metadata| trace!("{metadata:#?}")) } +#[cfg(feature = "rechunk")] +impl ContainerMountDriver for PodmanDriver { + fn create_container(image: &Reference) -> Result { + let output = { + let c = cmd!("podman", "create", image.to_string(), "bash"); + trace!("{c:?}"); + c + } + .output() + .into_diagnostic()?; + + if !output.status.success() { + bail!("Failed to create a container from image {image}"); + } + + Ok(ContainerId( + String::from_utf8(output.stdout.trim_ascii().to_vec()).into_diagnostic()?, + )) + } + + fn remove_container(container_id: &super::types::ContainerId) -> Result<()> { + let output = { + let c = cmd!("podman", "rm", container_id); + trace!("{c:?}"); + c + } + .output() + .into_diagnostic()?; + + if !output.status.success() { + bail!("Failed to remove container {container_id}"); + } + + Ok(()) + } + + fn remove_image(image: &Reference) -> Result<()> { + let output = { + let c = cmd!("podman", "rmi", image.to_string()); + trace!("{c:?}"); + c + } + .output() + .into_diagnostic()?; + + if !output.status.success() { + bail!("Failed to remove the image {image}"); + } + + Ok(()) + } + + fn mount_container(container_id: &super::types::ContainerId) -> Result { + let output = { + let c = cmd!("podman", "mount", container_id); + trace!("{c:?}"); + c + } + .output() + .into_diagnostic()?; + + if !output.status.success() { + bail!("Failed to mount container {container_id}"); + } + + Ok(MountId( + String::from_utf8(output.stdout.trim_ascii().to_vec()).into_diagnostic()?, + )) + } + + fn unmount_container(container_id: &super::types::ContainerId) -> Result<()> { + let output = { + let c = cmd!("podman", "unmount", container_id); + trace!("{c:?}"); + c + } + .output() + .into_diagnostic()?; + + if !output.status.success() { + bail!("Failed to unmount container {container_id}"); + } + + Ok(()) + } + + fn remove_volume(volume_id: &str) -> Result<()> { + let output = { + let c = cmd!("podman", "volume", "rm", volume_id); + trace!("{c:?}"); + c + } + .output() + .into_diagnostic()?; + + if !output.status.success() { + bail!("Failed to remove volume {volume_id}"); + } + + Ok(()) + } +} + +#[cfg(feature = "rechunk")] +impl RechunkDriver for PodmanDriver {} + impl RunDriver for PodmanDriver { - fn run(opts: &RunOpts) -> std::io::Result { + fn run(opts: &RunOpts) -> Result { trace!("PodmanDriver::run({opts:#?})"); - let cid_path = TempDir::new()?; + if !nix::unistd::Uid::effective().is_root() { + bail!("You must be root to run privileged podman!"); + } + + let cid_path = TempDir::new().into_diagnostic()?; let cid_file = cid_path.path().join("cid"); - let cid = ContainerId::new(&cid_file, ContainerRuntime::Podman, opts.privileged); + let cid = ContainerSignalId::new(&cid_file, ContainerRuntime::Podman, opts.privileged); add_cid(&cid); - let status = if opts.privileged { - podman_run(opts, &cid_file).status()? - } else { - podman_run(opts, &cid_file).build_status(&*opts.image, "Running container")? - }; + let status = podman_run(opts, &cid_file) + .build_status(&*opts.image, "Running container") + .into_diagnostic()?; remove_cid(&cid); Ok(status) } - fn run_output(opts: &RunOpts) -> std::io::Result { + fn run_output(opts: &RunOpts) -> Result { trace!("PodmanDriver::run_output({opts:#?})"); - let cid_path = TempDir::new()?; + if !nix::unistd::Uid::effective().is_root() { + bail!("You must be root to run privileged podman!"); + } + + let cid_path = TempDir::new().into_diagnostic()?; let cid_file = cid_path.path().join("cid"); - let cid = ContainerId::new(&cid_file, ContainerRuntime::Podman, opts.privileged); + let cid = ContainerSignalId::new(&cid_file, ContainerRuntime::Podman, opts.privileged); add_cid(&cid); - let output = podman_run(opts, &cid_file).output()?; + let output = podman_run(opts, &cid_file).output().into_diagnostic()?; remove_cid(&cid); @@ -376,16 +495,7 @@ impl RunDriver for PodmanDriver { fn podman_run(opts: &RunOpts, cid_file: &Path) -> Command { let command = cmd!( - if opts.privileged { - warn!( - "Running 'podman' in privileged mode requires '{}'", - "sudo".bold().red() - ); - "sudo" - } else { - "podman" - }, - if opts.privileged => "podman", + "podman", "run", format!("--cidfile={}", cid_file.display()), if opts.privileged => [ @@ -394,6 +504,7 @@ fn podman_run(opts: &RunOpts, cid_file: &Path) -> Command { ], if opts.remove => "--rm", if opts.pull => "--pull=always", + if let Some(user) = opts.user.as_ref() => format!("--user={user}"), for RunOptsVolume { path_or_vol_name, container_path } in opts.volumes.iter() => [ "--volume", format!("{path_or_vol_name}:{container_path}"), diff --git a/process/drivers/skopeo_driver.rs b/process/drivers/skopeo_driver.rs index b9e169ad..747b2b90 100644 --- a/process/drivers/skopeo_driver.rs +++ b/process/drivers/skopeo_driver.rs @@ -65,3 +65,27 @@ fn get_metadata_cache(opts: &GetMetadataOpts) -> Result { } serde_json::from_slice(&output.stdout).into_diagnostic() } + +#[cfg(feature = "rechunk")] +impl super::OciCopy for SkopeoDriver { + fn copy_oci_dir( + oci_dir: &super::types::OciDir, + registry: &oci_distribution::Reference, + ) -> Result<()> { + use crate::logging::CommandLogging; + + let status = { + let c = cmd!("skopeo", "copy", oci_dir, format!("docker://{registry}"),); + trace!("{c:?}"); + c + } + .build_status(registry.to_string(), format!("Copying {oci_dir} to")) + .into_diagnostic()?; + + if !status.success() { + bail!("Failed to copy {oci_dir} to {registry}"); + } + + Ok(()) + } +} diff --git a/process/drivers/traits.rs b/process/drivers/traits.rs index f5c58e16..25bcd093 100644 --- a/process/drivers/traits.rs +++ b/process/drivers/traits.rs @@ -30,6 +30,11 @@ use super::{ skopeo_driver::SkopeoDriver, types::ImageMetadata, }; +#[cfg(feature = "rechunk")] +use super::{ + opts::RechunkOpts, + types::{ContainerId, MountId}, +}; trait PrivateDriver {} @@ -209,13 +214,258 @@ pub trait RunDriver: PrivateDriver { /// /// # Errors /// Will error if there is an issue running the container. - fn run(opts: &RunOpts) -> std::io::Result; + fn run(opts: &RunOpts) -> Result; /// Run a container to perform an action and capturing output. /// /// # Errors /// Will error if there is an issue running the container. - fn run_output(opts: &RunOpts) -> std::io::Result; + fn run_output(opts: &RunOpts) -> Result; +} + +#[allow(private_bounds)] +#[cfg(feature = "rechunk")] +pub(super) trait ContainerMountDriver: PrivateDriver { + /// Creates container + /// + /// # Errors + /// Will error if the container create command fails. + fn create_container(image: &Reference) -> Result; + + /// Removes a container + /// + /// # Errors + /// Will error if the container remove command fails. + fn remove_container(container_id: &ContainerId) -> Result<()>; + + /// Removes an image + /// + /// # Errors + /// Will error if the image remove command fails. + fn remove_image(image: &Reference) -> Result<()>; + + /// Mounts the container + /// + /// # Errors + /// Will error if the container mount command fails. + fn mount_container(container_id: &ContainerId) -> Result; + + /// Unmount the container + /// + /// # Errors + /// Will error if the container unmount command fails. + fn unmount_container(container_id: &ContainerId) -> Result<()>; + + /// Remove a volume + /// + /// # Errors + /// Will error if the volume remove command fails. + fn remove_volume(volume_id: &str) -> Result<()>; +} + +#[cfg(feature = "rechunk")] +pub(super) trait OciCopy { + fn copy_oci_dir( + oci_dir: &super::types::OciDir, + registry: &oci_distribution::Reference, + ) -> Result<()>; +} + +#[allow(private_bounds)] +#[cfg(feature = "rechunk")] +pub trait RechunkDriver: RunDriver + BuildDriver + ContainerMountDriver { + const RECHUNK_IMAGE: &str = "ghcr.io/hhd-dev/rechunk:v1.0.1"; + + /// Perform a rechunk build of a recipe. + /// + /// # Errors + /// Will error if the rechunk process fails. + fn rechunk(opts: &RechunkOpts) -> Result> { + let ostree_cache_id = &uuid::Uuid::new_v4().to_string(); + let raw_image = + &Reference::try_from(format!("localhost/{ostree_cache_id}/raw-rechunk")).unwrap(); + let current_dir = &std::env::current_dir().into_diagnostic()?; + let current_dir = &*current_dir.to_string_lossy(); + let full_image = Reference::try_from(opts.tags.first().map_or_else( + || opts.image.to_string(), + |tag| format!("{}:{tag}", opts.image), + )) + .into_diagnostic()?; + + Self::build( + &BuildOpts::builder() + .image(raw_image.to_string()) + .containerfile(&*opts.containerfile) + .platform(opts.platform) + .squash(true) + .host_network(true) + .build(), + )?; + + let container = &Self::create_container(raw_image)?; + let mount = &Self::mount_container(container)?; + + Self::prune_image(mount, container, raw_image, opts)?; + Self::create_ostree_commit(mount, ostree_cache_id, container, raw_image, opts)?; + + let temp_dir = tempfile::TempDir::new().into_diagnostic()?; + let temp_dir_str = &*temp_dir.path().to_string_lossy(); + + Self::rechunk_image(ostree_cache_id, temp_dir_str, current_dir, opts)?; + + let mut image_list = Vec::with_capacity(opts.tags.len()); + + if opts.push { + let oci_dir = &super::types::OciDir::try_from(temp_dir.path().join(ostree_cache_id))?; + + for tag in &opts.tags { + let tagged_image = Reference::with_tag( + full_image.registry().to_string(), + full_image.repository().to_string(), + tag.to_string(), + ); + + blue_build_utils::retry(opts.retry_count, 5, || { + debug!("Pushing image {tagged_image}"); + + Driver::copy_oci_dir(oci_dir, &tagged_image) + })?; + image_list.push(tagged_image.into()); + } + } + + Ok(image_list) + } + + /// Step 1 of the rechunk process that prunes excess files. + /// + /// # Errors + /// Will error if the prune process fails. + fn prune_image( + mount: &MountId, + container: &ContainerId, + raw_image: &Reference, + opts: &RechunkOpts<'_>, + ) -> Result<(), miette::Error> { + let status = Self::run( + &RunOpts::builder() + .image(Self::RECHUNK_IMAGE) + .remove(true) + .user("0:0") + .privileged(true) + .volumes(crate::run_volumes! { + mount => "/var/tree", + }) + .env_vars(crate::run_envs! { + "TREE" => "/var/tree", + }) + .args(bon::vec!["/sources/rechunk/1_prune.sh"]) + .build(), + )?; + + if !status.success() { + Self::unmount_container(container)?; + Self::remove_container(container)?; + Self::remove_image(raw_image)?; + bail!("Failed to run prune step for {}", &opts.image); + } + + Ok(()) + } + + /// Step 2 of the rechunk process that creates the ostree commit. + /// + /// # Errors + /// Will error if the ostree commit process fails. + fn create_ostree_commit( + mount: &MountId, + ostree_cache_id: &str, + container: &ContainerId, + raw_image: &Reference, + opts: &RechunkOpts<'_>, + ) -> Result<()> { + let status = Self::run( + &RunOpts::builder() + .image(Self::RECHUNK_IMAGE) + .remove(true) + .user("0:0") + .privileged(true) + .volumes(crate::run_volumes! { + mount => "/var/tree", + ostree_cache_id => "/var/ostree", + }) + .env_vars(crate::run_envs! { + "TREE" => "/var/tree", + "REPO" => "/var/ostree/repo", + "RESET_TIMESTAMP" => "1", + }) + .args(bon::vec!["/sources/rechunk/2_create.sh"]) + .build(), + )?; + Self::unmount_container(container)?; + Self::remove_container(container)?; + Self::remove_image(raw_image)?; + + if !status.success() { + bail!("Failed to run Ostree create step for {}", &opts.image); + } + + Ok(()) + } + + /// Step 3 of the rechunk process that generates the final chunked image. + /// + /// # Errors + /// Will error if the chunk process fails. + fn rechunk_image( + ostree_cache_id: &str, + temp_dir_str: &str, + current_dir: &str, + opts: &RechunkOpts<'_>, + ) -> Result<()> { + let status = Self::run( + &RunOpts::builder() + .image(Self::RECHUNK_IMAGE) + .remove(true) + .user("0:0") + .privileged(true) + .volumes(crate::run_volumes! { + ostree_cache_id => "/var/ostree", + temp_dir_str => "/workspace", + current_dir => "/var/git" + }) + .env_vars(crate::run_envs! { + "REPO" => "/var/ostree/repo", + "PREV_REF" => &*opts.image, + "OUT_NAME" => ostree_cache_id, + // "PREV_REF_FAIL" => "true", + "VERSION" => format!("{}", opts.version), + "OUT_REF" => format!("oci:{ostree_cache_id}"), + "GIT_DIR" => "/var/git", + "LABELS" => format!( + "{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}", + format_args!("{}={}", blue_build_utils::constants::BUILD_ID_LABEL, Driver::get_build_id()), + format_args!("org.opencontainers.image.title={}", &opts.name), + format_args!("org.opencontainers.image.description={}", &opts.description), + format_args!("org.opencontainers.image.source={}", &opts.repo), + format_args!("org.opencontainers.image.base.digest={}", &opts.base_digest), + format_args!("org.opencontainers.image.base.name={}", &opts.base_image), + "org.opencontainers.image.created=", + "io.artifacthub.package.readme-url=https://raw.githubusercontent.com/blue-build/cli/main/README.md", + ) + }) + .args(bon::vec!["/sources/rechunk/3_chunk.sh"]) + .build(), + )?; + + Self::remove_volume(ostree_cache_id)?; + + if !status.success() { + bail!("Failed to run rechunking for {}", &opts.image); + } + + Ok(()) + } } /// Allows agnostic management of signature keys. diff --git a/process/drivers/types.rs b/process/drivers/types.rs index ebc291df..e1cc6aaa 100644 --- a/process/drivers/types.rs +++ b/process/drivers/types.rs @@ -235,3 +235,74 @@ impl ImageMetadata { ) } } + +#[cfg(feature = "rechunk")] +pub struct ContainerId(pub(super) String); + +#[cfg(feature = "rechunk")] +impl std::fmt::Display for ContainerId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", &self.0) + } +} + +#[cfg(feature = "rechunk")] +impl AsRef for ContainerId { + fn as_ref(&self) -> &std::ffi::OsStr { + self.0.as_ref() + } +} + +#[cfg(feature = "rechunk")] +pub struct MountId(pub(super) String); + +#[cfg(feature = "rechunk")] +impl std::fmt::Display for MountId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", &self.0) + } +} + +#[cfg(feature = "rechunk")] +impl AsRef for MountId { + fn as_ref(&self) -> &std::ffi::OsStr { + self.0.as_ref() + } +} + +#[cfg(feature = "rechunk")] +impl<'a> From<&'a MountId> for std::borrow::Cow<'a, str> { + fn from(value: &'a MountId) -> Self { + Self::Borrowed(&value.0) + } +} + +#[cfg(feature = "rechunk")] +pub struct OciDir(String); + +#[cfg(feature = "rechunk")] +impl std::fmt::Display for OciDir { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", &self.0) + } +} + +#[cfg(feature = "rechunk")] +impl AsRef for OciDir { + fn as_ref(&self) -> &std::ffi::OsStr { + self.0.as_ref() + } +} + +#[cfg(feature = "rechunk")] +impl TryFrom for OciDir { + type Error = miette::Report; + + fn try_from(value: std::path::PathBuf) -> Result { + if !value.is_dir() { + miette::bail!("OCI directory doesn't exist at {}", value.display()); + } + + Ok(Self(format!("oci:{}", value.display()))) + } +} diff --git a/process/signal_handler.rs b/process/signal_handler.rs index 4225e0d5..a2fd78c7 100644 --- a/process/signal_handler.rs +++ b/process/signal_handler.rs @@ -24,7 +24,7 @@ use signal_hook::{ use crate::logging::Logger; #[derive(Debug, Clone, PartialEq, Eq)] -pub struct ContainerId { +pub struct ContainerSignalId { cid_path: PathBuf, requires_sudo: bool, container_runtime: ContainerRuntime, @@ -45,7 +45,7 @@ impl std::fmt::Display for ContainerRuntime { } } -impl ContainerId { +impl ContainerSignalId { pub fn new

(cid_path: P, container_runtime: ContainerRuntime, requires_sudo: bool) -> Self where P: Into, @@ -60,7 +60,8 @@ impl ContainerId { } static PID_LIST: Lazy>>> = Lazy::new(|| Arc::new(Mutex::new(vec![]))); -static CID_LIST: Lazy>>> = Lazy::new(|| Arc::new(Mutex::new(vec![]))); +static CID_LIST: Lazy>>> = + Lazy::new(|| Arc::new(Mutex::new(vec![]))); /// Initialize Ctrl-C handler. This should be done at the start /// of a binary. @@ -225,7 +226,7 @@ where /// /// # Panics /// Will panic if the mutex cannot be locked. -pub fn add_cid(cid: &ContainerId) { +pub fn add_cid(cid: &ContainerSignalId) { let mut cid_list = CID_LIST.lock().expect("Should lock cid_list"); if !cid_list.contains(cid) { @@ -237,7 +238,7 @@ pub fn add_cid(cid: &ContainerId) { /// /// # Panics /// Will panic if the mutex cannot be locked. -pub fn remove_cid(cid: &ContainerId) { +pub fn remove_cid(cid: &ContainerSignalId) { let mut cid_list = CID_LIST.lock().expect("Should lock cid_list"); if let Some(index) = cid_list.iter().position(|val| *val == *cid) { diff --git a/src/commands/build.rs b/src/commands/build.rs index c7380196..0662ba26 100644 --- a/src/commands/build.rs +++ b/src/commands/build.rs @@ -52,7 +52,7 @@ pub struct BuildCommand { /// Requires `--registry`, /// `--username`, and `--password` if not /// building in CI. - #[arg(short, long)] + #[arg(short, long, group = "archive_push")] #[builder(default)] push: bool, @@ -84,7 +84,7 @@ pub struct BuildCommand { /// Archives the built image into a tarfile /// in the specified directory. - #[arg(short, long)] + #[arg(short, long, group = "archive_rechunk", group = "archive_push")] #[builder(into)] archive: Option, @@ -110,6 +110,16 @@ pub struct BuildCommand { #[builder(default)] squash: bool, + /// Performs rechunking on the image to allow for smaller images + /// and smaller updates. This will increase the build-time + /// and take up more space during build-time. + /// + /// NOTE: This must be run as root! + #[arg(long, group = "archive_rechunk")] + #[builder(default)] + #[cfg(feature = "rechunk")] + rechunk: bool, + #[clap(flatten)] #[builder(default)] credentials: CredentialsArgs, @@ -127,6 +137,11 @@ impl BlueBuildCommand for BuildCommand { fn try_run(&mut self) -> Result<()> { trace!("BuildCommand::try_run()"); + #[cfg(feature = "rechunk")] + if !nix::unistd::Uid::effective().is_root() && self.rechunk { + bail!("You must be root to use the rechunk feature!"); + } + Driver::init(self.drivers); Credentials::init(self.credentials.clone()); @@ -272,32 +287,82 @@ impl BuildCommand { )?; let image_name = self.image_name(&recipe)?; - let opts = if let Some(archive_dir) = self.archive.as_ref() { - BuildTagPushOpts::builder() - .containerfile(containerfile) - .platform(self.platform) - .archive_path(format!( - "{}/{}.{ARCHIVE_SUFFIX}", - archive_dir.to_string_lossy().trim_end_matches('/'), - recipe.name.to_lowercase().replace('/', "_"), - )) - .squash(self.squash) - .build() + let build_fn = || -> Result> { + Driver::build_tag_push(&self.archive.as_ref().map_or_else( + || { + BuildTagPushOpts::builder() + .image(&image_name) + .containerfile(containerfile) + .platform(self.platform) + .tags(tags.collect_cow_vec()) + .push(self.push) + .retry_push(self.retry_push) + .retry_count(self.retry_count) + .compression(self.compression_format) + .squash(self.squash) + .build() + }, + |archive_dir| { + BuildTagPushOpts::builder() + .containerfile(containerfile) + .platform(self.platform) + .archive_path(format!( + "{}/{}.{ARCHIVE_SUFFIX}", + archive_dir.to_string_lossy().trim_end_matches('/'), + recipe.name.to_lowercase().replace('/', "_"), + )) + .squash(self.squash) + .build() + }, + )) + }; + + #[cfg(feature = "rechunk")] + let images = if self.rechunk { + use blue_build_process_management::drivers::{ + opts::{GetMetadataOpts, RechunkOpts}, + InspectDriver, RechunkDriver, + }; + + Driver::rechunk( + &RechunkOpts::builder() + .image(&image_name) + .containerfile(containerfile) + .platform(self.platform) + .tags(tags.collect_cow_vec()) + .push(self.push) + .version(format!( + "{version}.", + version = Driver::get_os_version() + .oci_ref(&recipe.base_image_ref()?) + .platform(self.platform) + .call()?, + )) + .retry_push(self.retry_push) + .retry_count(self.retry_count) + .compression(self.compression_format) + .base_digest( + Driver::get_metadata( + &GetMetadataOpts::builder() + .image(&*recipe.base_image) + .tag(&*recipe.image_version) + .platform(self.platform) + .build(), + )? + .digest, + ) + .repo(Driver::get_repo_url()?) + .name(&*recipe.name) + .description(&*recipe.description) + .base_image(format!("{}:{}", &recipe.base_image, &recipe.image_version)) + .build(), + )? } else { - BuildTagPushOpts::builder() - .image(&image_name) - .containerfile(containerfile) - .platform(self.platform) - .tags(tags.collect_cow_vec()) - .push(self.push) - .retry_push(self.retry_push) - .retry_count(self.retry_count) - .compression(self.compression_format) - .squash(self.squash) - .build() + build_fn()? }; - let images = Driver::build_tag_push(&opts)?; + #[cfg(not(feature = "rechunk"))] + let images = build_fn()?; if self.push && !self.no_sign { Driver::sign_and_verify( diff --git a/src/commands/generate_iso.rs b/src/commands/generate_iso.rs index b1816fae..e80c3bc3 100644 --- a/src/commands/generate_iso.rs +++ b/src/commands/generate_iso.rs @@ -12,7 +12,7 @@ use oci_distribution::Reference; use tempfile::TempDir; use blue_build_process_management::{ - drivers::{opts::RunOpts, Driver, DriverArgs, RunDriver}, + drivers::{opts::RunOpts, types::RunDriverType, Driver, DriverArgs, RunDriver}, run_volumes, }; @@ -122,6 +122,12 @@ impl BlueBuildCommand for GenerateIsoCommand { fn try_run(&mut self) -> Result<()> { Driver::init(self.drivers); + if !nix::unistd::Uid::effective().is_root() + && matches!(Driver::get_run_driver(), RunDriverType::Podman) + { + bail!("You must be root to build an ISO!"); + } + let image_out_dir = TempDir::new().into_diagnostic()?; let output_dir = if let Some(output_dir) = self.output_dir.clone() { @@ -239,7 +245,7 @@ impl GenerateIsoCommand { .volumes(vols) .build(); - let status = Driver::run(&opts).into_diagnostic()?; + let status = Driver::run(&opts)?; if !status.success() { bail!("Failed to create ISO"); diff --git a/template/templates/Containerfile.j2 b/template/templates/Containerfile.j2 index 1880e8f3..5d07b706 100644 --- a/template/templates/Containerfile.j2 +++ b/template/templates/Containerfile.j2 @@ -1,8 +1,9 @@ {%- import "modules/modules.j2" as modules -%} {%- include "stages.j2" %} +{%- set main_stage = recipe.name|replace('/', "-") %} # Main image -FROM {{ recipe.base_image }}@{{ base_digest }} AS {{ recipe.name|replace('/', "-") }} +FROM {{ recipe.base_image }}@{{ base_digest }} AS {{ main_stage }} ARG RECIPE={{ recipe_path.display() }} ARG IMAGE_REGISTRY={{ registry }}