diff --git a/Cargo.lock b/Cargo.lock index c4257171..140b80c7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -563,6 +563,16 @@ version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" +[[package]] +name = "env_logger" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a19187fea3ac7e84da7dacf48de0c45d63c6a76f9490dae389aead16c243fce3" +dependencies = [ + "log", + "regex", +] + [[package]] name = "equivalent" version = "1.0.1" @@ -1354,14 +1364,13 @@ dependencies = [ [[package]] name = "ostree-ext" version = "0.15.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3749ad9f089cd49975c43d4e510888f7063a7b2339265f15f165a0b8b2268a5c" dependencies = [ "anyhow", "camino", "cap-std-ext", "chrono", "clap", + "clap_mangen", "containers-image-proxy", "flate2", "fn-error-context", @@ -1370,6 +1379,7 @@ dependencies = [ "hex", "indexmap", "indicatif", + "indoc", "io-lifetimes", "libc", "libsystemd", @@ -1378,11 +1388,14 @@ dependencies = [ "once_cell", "openssl", "ostree", + "ostree-ext", "pin-project", + "quickcheck", "regex", "rustix", "serde", "serde_json", + "similar-asserts", "tar", "tempfile", "terminal_size", @@ -1390,6 +1403,7 @@ dependencies = [ "tokio-stream", "tokio-util", "tracing", + "xshell", "zstd", ] @@ -1523,6 +1537,17 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "007d8adb5ddab6f8e3f491ac63566a7d5002cc7ed73901f72057943fa71ae1ae" +[[package]] +name = "quickcheck" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "588f6378e4dd99458b60ec275b4477add41ce4fa9f64dcba6f15adccb19b50d6" +dependencies = [ + "env_logger", + "log", + "rand", +] + [[package]] name = "quote" version = "1.0.35" diff --git a/Cargo.toml b/Cargo.toml index 0f775d4a..3eeefb32 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = ["cli", "lib", "xtask", "tests-integration"] +members = ["cli", "lib", "ostree-ext", "xtask", "tests-integration"] resolver = "2" [profile.dev] @@ -23,9 +23,13 @@ camino = "1.1.6" cap-std-ext = "4.0.2" chrono = { version = "0.4.38", default-features = false } clap = "4.5.4" +clap_mangen = { version = "0.2.20" } +hex = "0.4.3" indoc = "2.0.5" +indicatif = "0.17.0" fn-error-context = "0.2.1" libc = "0.2.154" +openssl = "0.10.33" rustix = { "version" = "0.38.34", features = ["thread", "fs", "system", "process", "mount"] } serde = "1.0.199" serde_json = "1.0.116" @@ -34,6 +38,7 @@ static_assertions = "1.1.0" tempfile = "3.10.1" tracing = "0.1.40" tokio = ">= 1.37.0" +tokio-util = { features = ["io-util"], version = "0.7.10" } # See https://github.com/coreos/cargo-vendor-filterer [workspace.metadata.vendor-filter] diff --git a/lib/Cargo.toml b/lib/Cargo.toml index 015b489c..a43aaca2 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -19,18 +19,18 @@ anstyle = "1.0.6" anyhow = { workspace = true } bootc-utils = { path = "../utils" } camino = { workspace = true, features = ["serde1"] } -ostree-ext = { version = "0.15.0" } +ostree-ext = { path = "../ostree-ext" } chrono = { workspace = true, features = ["serde"] } clap = { workspace = true, features = ["derive","cargo"] } -clap_mangen = { version = "0.2.20", optional = true } +clap_mangen = { workspace = true, optional = true } cap-std-ext = { workspace = true, features = ["fs_utf8"] } -hex = "^0.4.3" +hex = { workspace = true } fn-error-context = { workspace = true } -indicatif = "0.17.8" +indicatif = { workspace = true } libc = { workspace = true } liboverdrop = "0.1.0" libsystemd = "0.7" -openssl = "^0.10.64" +openssl = { workspace = true } regex = "1.10.4" rustix = { workspace = true } schemars = { version = "0.8.17", features = ["chrono"] } @@ -39,7 +39,7 @@ serde_ignored = "0.1.10" serde_json = { workspace = true } serde_yaml = "0.9.34" tokio = { workspace = true, features = ["io-std", "time", "process", "rt", "net"] } -tokio-util = { features = ["io-util"], version = "0.7.10" } +tokio-util = { workspace = true } tracing = { workspace = true } tempfile = { workspace = true } toml = "0.8.12" diff --git a/ostree-ext/.github/workflows/bootc.yml b/ostree-ext/.github/workflows/bootc.yml new file mode 100644 index 00000000..65548679 --- /dev/null +++ b/ostree-ext/.github/workflows/bootc.yml @@ -0,0 +1,65 @@ +name: bootc + +permissions: + actions: read + +on: + push: + branches: [main] + pull_request: + branches: [main] + workflow_dispatch: {} + +jobs: + build-c9s: + runs-on: ubuntu-latest + container: quay.io/centos/centos:stream9 + steps: + - run: dnf -y install git-core + - uses: actions/checkout@v3 + with: + repository: containers/bootc + path: bootc + - uses: actions/checkout@v3 + with: + path: ostree-rs-ext + - name: Patch bootc to use ostree-rs-ext + run: | + set -xeuo pipefail + cd bootc + cat >> Cargo.toml << 'EOF' + [patch.crates-io] + ostree-ext = { path = "../ostree-rs-ext/lib" } + EOF + - name: Install deps + run: ./bootc/ci/installdeps.sh + - name: Cache Dependencies + uses: Swatinem/rust-cache@v2 + with: + key: "build-bootc-c9s" + workspaces: bootc + - name: Build + run: cd bootc && make test-bin-archive + - name: Upload binary + uses: actions/upload-artifact@v4 + with: + name: bootc-c9s.tar.zst + path: bootc/target/bootc.tar.zst + privtest-alongside: + name: "Test install-alongside" + needs: build-c9s + runs-on: ubuntu-latest + steps: + - name: Download + uses: actions/download-artifact@v4.1.7 + with: + name: bootc-c9s.tar.zst + - name: Install + run: tar -xvf bootc.tar.zst + - name: Integration tests + run: | + set -xeuo pipefail + sudo podman run --rm -ti --privileged -v /:/target -v /var/lib/containers:/var/lib/containers -v ./usr/bin/bootc:/usr/bin/bootc --pid=host --security-opt label=disable \ + quay.io/centos-bootc/centos-bootc-dev:stream9 bootc install to-filesystem \ + --karg=foo=bar --disable-selinux --replace=alongside /target + diff --git a/ostree-ext/.github/workflows/rust.yml b/ostree-ext/.github/workflows/rust.yml new file mode 100644 index 00000000..c4dbd46c --- /dev/null +++ b/ostree-ext/.github/workflows/rust.yml @@ -0,0 +1,184 @@ +# Inspired by https://github.com/rust-analyzer/rust-analyzer/blob/master/.github/workflows/ci.yaml +# but tweaked in several ways. If you make changes here, consider doing so across other +# repositories in e.g. ostreedev etc. +name: Rust + +permissions: + actions: read + +on: + push: + branches: [main] + pull_request: + branches: [main] + workflow_dispatch: {} + +env: + CARGO_TERM_COLOR: always + +jobs: + tests: + runs-on: ubuntu-latest + container: quay.io/coreos-assembler/fcos-buildroot:testing-devel + steps: + - uses: actions/checkout@v3 + - name: Code lints + run: ./ci/lints.sh + - name: Install deps + run: ./ci/installdeps.sh + # xref containers/containers-image-proxy-rs + - name: Cache Dependencies + uses: Swatinem/rust-cache@v2 + with: + key: "tests" + - name: cargo fmt (check) + run: cargo fmt -- --check -l + - name: Build + run: cargo test --no-run + - name: Individual checks + run: (cd cli && cargo check) && (cd lib && cargo check) + - name: Run tests + run: cargo test -- --nocapture --quiet + - name: Manpage generation + run: mkdir -p target/man && cargo run --features=docgen -- man --directory target/man + - name: cargo clippy + run: cargo clippy + build: + runs-on: ubuntu-latest + container: quay.io/coreos-assembler/fcos-buildroot:testing-devel + steps: + - uses: actions/checkout@v3 + - name: Install deps + run: ./ci/installdeps.sh + - name: Cache Dependencies + uses: Swatinem/rust-cache@v2 + with: + key: "build" + - name: Build + run: cargo build --release --features=internal-testing-api + - name: Upload binary + uses: actions/upload-artifact@v4 + with: + name: ostree-ext-cli + path: target/release/ostree-ext-cli + build-minimum-toolchain: + name: "Build using MSRV" + runs-on: ubuntu-latest + container: quay.io/coreos-assembler/fcos-buildroot:testing-devel + steps: + - name: Checkout repository + uses: actions/checkout@v3 + - name: Install deps + run: ./ci/installdeps.sh + - name: Detect crate MSRV + shell: bash + run: | + msrv=$(cargo metadata --format-version 1 --no-deps | \ + jq -r '.packages[1].rust_version') + echo "Crate MSRV: $msrv" + echo "ACTION_MSRV_TOOLCHAIN=$msrv" >> $GITHUB_ENV + - name: Remove system Rust toolchain + run: dnf remove -y rust cargo + - uses: dtolnay/rust-toolchain@master + with: + toolchain: ${{ env['ACTION_MSRV_TOOLCHAIN'] }} + - name: Cache Dependencies + uses: Swatinem/rust-cache@v2 + with: + key: "min" + - name: cargo check + run: cargo check + cargo-deny: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: EmbarkStudios/cargo-deny-action@v1 + with: + log-level: warn + command: check bans sources licenses + integration: + name: "Integration" + needs: build + runs-on: ubuntu-latest + container: quay.io/fedora/fedora-coreos:testing-devel + steps: + - name: Checkout repository + uses: actions/checkout@v3 + - name: Download ostree-ext-cli + uses: actions/download-artifact@v4.1.7 + with: + name: ostree-ext-cli + - name: Install + run: install ostree-ext-cli /usr/bin && rm -v ostree-ext-cli + - name: Integration tests + run: ./ci/integration.sh + ima: + name: "Integration (IMA)" + needs: build + runs-on: ubuntu-latest + container: quay.io/fedora/fedora-coreos:testing-devel + steps: + - name: Checkout repository + uses: actions/checkout@v3 + - name: Download ostree-ext-cli + uses: actions/download-artifact@v4.1.7 + with: + name: ostree-ext-cli + - name: Install + run: install ostree-ext-cli /usr/bin && rm -v ostree-ext-cli + - name: Integration tests + run: ./ci/ima.sh + privtest: + name: "Privileged testing" + needs: build + runs-on: ubuntu-latest + container: + image: quay.io/fedora/fedora-coreos:testing-devel + options: "--privileged --pid=host -v /var/tmp:/var/tmp -v /run/dbus:/run/dbus -v /run/systemd:/run/systemd -v /:/run/host" + steps: + - name: Checkout repository + uses: actions/checkout@v3 + - name: Download + uses: actions/download-artifact@v4.1.7 + with: + name: ostree-ext-cli + - name: Install + run: install ostree-ext-cli /usr/bin && rm -v ostree-ext-cli + - name: Integration tests + run: ./ci/priv-integration.sh + privtest-cockpit: + name: "Privileged testing (cockpit)" + needs: build + runs-on: ubuntu-latest + container: + image: quay.io/fedora/fedora-bootc:41 + options: "--privileged --pid=host -v /var/tmp:/var/tmp -v /run/dbus:/run/dbus -v /run/systemd:/run/systemd -v /:/run/host" + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Download + uses: actions/download-artifact@v4.1.7 + with: + name: ostree-ext-cli + - name: Install + run: install ostree-ext-cli /usr/bin && rm -v ostree-ext-cli + - name: Integration tests + run: ./ci/priv-test-cockpit-selinux.sh + container-build: + name: "Container build" + needs: build + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v3 + - name: Checkout coreos-layering-examples + uses: actions/checkout@v3 + with: + repository: coreos/coreos-layering-examples + path: coreos-layering-examples + - name: Download + uses: actions/download-artifact@v4.1.7 + with: + name: ostree-ext-cli + - name: Integration tests + run: ./ci/container-build-integration.sh diff --git a/ostree-ext/.gitignore b/ostree-ext/.gitignore new file mode 100644 index 00000000..b59902fd --- /dev/null +++ b/ostree-ext/.gitignore @@ -0,0 +1,7 @@ +example + + +# Added by cargo + +/target +Cargo.lock diff --git a/ostree-ext/Cargo.toml b/ostree-ext/Cargo.toml new file mode 100644 index 00000000..2d1d400d --- /dev/null +++ b/ostree-ext/Cargo.toml @@ -0,0 +1,71 @@ +[package] +authors = ["Colin Walters "] +description = "Extension APIs for OSTree" +edition = "2021" +license = "MIT OR Apache-2.0" +name = "ostree-ext" +readme = "../README.md" +repository = "https://github.com/ostreedev/ostree-rs-ext" +version = "0.15.3" +rust-version = "1.74.0" + +[dependencies] +# Note that we re-export the oci-spec types +# that are exported by this crate, so when bumping +# semver here you must also bump our semver. +containers-image-proxy = "0.7.0" +# We re-export this library too. +ostree = { features = ["v2022_6"], version = "0.19.0" } + +# Private dependencies +anyhow = { workspace = true } +camino = { workspace = true, features = ["serde1"] } +chrono = { workspace = true } +olpc-cjson = "0.1.1" +clap = { workspace = true, features = ["derive","cargo"] } +clap_mangen = { workspace = true, optional = true } +cap-std-ext = { workspace = true, features = ["fs_utf8"] } +flate2 = { features = ["zlib"], default-features = false, version = "1.0.20" } +fn-error-context = { workspace = true } +futures-util = "0.3.13" +gvariant = "0.5.0" +hex = { workspace = true } +io-lifetimes = "2" +indicatif = { workspace = true } +once_cell = "1.9" +libc = { workspace = true } +libsystemd = "0.7.0" +openssl = { workspace = true } +ocidir = "0.3.0" +pin-project = "1.0" +regex = "1.5.4" +rustix = { workspace = true, features = ["fs", "process"] } +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } +tar = "0.4.43" +tempfile = { workspace = true } +terminal_size = "0.3" +tokio = { workspace = true, features = ["io-std", "time", "process", "rt", "net"] } +tokio-util = { workspace = true } +tokio-stream = { features = ["sync"], version = "0.1.8" } +tracing = "0.1" +zstd = { version = "0.13.1", features = ["pkg-config"] } +indexmap = { version = "2.2.2", features = ["serde"] } + +indoc = { version = "2", optional = true } +xshell = { version = "0.2", optional = true } +similar-asserts = { version = "1.5.0", optional = true } + +[dev-dependencies] +quickcheck = "1" +# https://github.com/rust-lang/cargo/issues/2911 +# https://github.com/rust-lang/rfcs/pull/1956 +ostree-ext = { path = ".", features = ["internal-testing-api"] } + +[package.metadata.docs.rs] +features = ["dox"] + +[features] +docgen = ["clap_mangen"] +dox = ["ostree/dox"] +internal-testing-api = ["xshell", "indoc", "similar-asserts"] diff --git a/ostree-ext/LICENSE-APACHE b/ostree-ext/LICENSE-APACHE new file mode 100644 index 00000000..8f71f43f --- /dev/null +++ b/ostree-ext/LICENSE-APACHE @@ -0,0 +1,202 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/ostree-ext/LICENSE-MIT b/ostree-ext/LICENSE-MIT new file mode 100644 index 00000000..dbd7f657 --- /dev/null +++ b/ostree-ext/LICENSE-MIT @@ -0,0 +1,19 @@ +Copyright (c) 2016 The openat Developers + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/ostree-ext/README.md b/ostree-ext/README.md new file mode 100644 index 00000000..b1e0ab5a --- /dev/null +++ b/ostree-ext/README.md @@ -0,0 +1,187 @@ +# ostree-ext + +Extension APIs for [ostree](https://github.com/ostreedev/ostree/) that are written in Rust, using the [Rust ostree bindings](https://crates.io/crates/ostree). + +If you are writing tooling that uses ostree and Rust, this crate is intended for you. +However, while the ostree core is very stable, the APIs and data models and this crate +should be considered "slushy". An effort will be made to preserve backwards compatibility +for data written by prior versions (e.g. of tar and container serialization), but +if you choose to use this crate, please [file an issue](https://github.com/ostreedev/ostree-rs-ext/issues) +to let us know. + +At the moment, the following projects are known to use this crate: + +- https://github.com/containers/bootc +- https://github.com/coreos/rpm-ostree + +The intention of this crate is to be where new high level ostree-related features +land. However, at this time it is kept separate from the core C library, which +is in turn separate from the [ostree-rs bindings](https://github.com/ostreedev/ostree-rs). + +High level features (more on this below): + +- ostree and [opencontainers/image](https://github.com/opencontainers/image-spec) bridging/integration +- Generalized tar import/export +- APIs to diff ostree commits + +```mermaid +flowchart TD + ostree-rs-ext --- ostree-rs --- ostree + ostree-rs-ext --- containers-image-proxy-rs --- skopeo --- containers/image +``` + +For more information on the container stack, see below. + +## module "tar": tar export/import + +ostree's support for exporting to a tarball is lossy because it doesn't have e.g. commit +metadata. This adds a new export format that is effectively a new custom repository mode +combined with a hardlinked checkout. + +This new export stream can be losslessly imported back into a different repository. + +### Filesystem layout + +``` +. +├── etc # content is at traditional /etc, not /usr/etc +│   └── passwd +├── sysroot +│   └── ostree # ostree object store with hardlinks to destinations +│   ├── repo +│   │   └── objects +│   │   ├── 00 +│   │   └── 8b +│   │   └── 7df143d91c716ecfa5fc1730022f6b421b05cedee8fd52b1fc65a96030ad52.file.xattrs +│   │   └── 7df143d91c716ecfa5fc1730022f6b421b05cedee8fd52b1fc65a96030ad52.file +│   └── xattrs # A new directory with extended attributes, hardlinked with .xattr files +│   └── 58d523efd29244331392770befa2f8bd55b3ef594532d3b8dbf94b70dc72e674 +└── usr + ├── bin + │   └── bash + └── lib64 + └── libc.so +``` + +Think of this like a new ostree repository mode `tar-stream` or so, although right now it only holds a single commit. + +A major distinction is the addition of special `.xattr` files; tar variants and support library differ too much for us to rely on this making it through round trips. And further, to support the webserver-in-container we need e.g. `security.selinux` to not be changed/overwritten by the container runtime. + +## module "diff": Compute the difference between two ostree commits + +```rust + let subdir: Option<&str> = None; + let refname = "fedora/coreos/x86_64/stable"; + let diff = ostree_ext::diff::diff(repo, &format!("{}^", refname), refname, subdir)?; +``` + +This is used by `rpm-ostree ex apply-live`. + +## module "container": Bridging between ostree and OCI/Docker images + + +This module contains APIs to bidirectionally map between OSTree commits and the [opencontainers](https://github.com/opencontainers) +ecosystem. + +Because container images are just layers of tarballs, this builds on the [`crate::tar`] module. + +This module builds on [containers-image-proxy-rs](https://github.com/containers/containers-image-proxy-rs) +and [skopeo](https://github.com/containers/skopeo), which in turn is ultimately a frontend +around the [containers/image](https://github.com/containers/image) ecosystem. + +In particular, the `containers/image` library is used to fetch content from remote registries, +which allows building on top of functionality in that library, including signatures, mirroring +and in general a battle tested codebase for interacting with both OCI and Docker registries. + +### Encapsulation + +For existing organizations which use ostree, APIs (and a CLI) are provided to "encapsulate" +and "unencapsulate" an OSTree commit as as an OCI image. + +``` +$ ostree-ext-cli container encapsulate --repo=/path/to/repo exampleos/x86_64/stable docker://quay.io/exampleos/exampleos:stable +``` +You can then e.g. + +``` +$ podman run --rm -ti --entrypoint bash quay.io/exampleos/exampleos:stable +``` + +Running the container directly for e.g. CI testing is one use case. But more importantly, this container image +can be pushed to any registry, and used as part of ostree-based operating system release engineering. + +However, this is a very simplistic model - it currently generates a container image with a single layer, which means +every change requires redownloading that entire layer. As of recently, the underlying APIs for generating +container images support "chunked" images. But this requires coding for a specific package/build system. + +A good reference code base for generating "chunked" images is [rpm-ostree compose container-encapsulate](https://coreos.github.io/rpm-ostree/container/#converting-ostree-commits-to-new-base-images). This is used to generate the current [Fedora CoreOS](https://quay.io/repository/fedora/fedora-coreos?tab=tags&tag=latest) +images. + +### Unencapsulate an ostree-container directly + +A primary goal of this effort is to make it fully native to an ostree-based operating system to pull a container image directly too. + +The CLI offers a method to "unencapsulate" - fetch a container image in a streaming fashion and +import the embedded OSTree commit. Here, you must use a prefix scheme which defines signature verification. + +- `ostree-remote-image:$remote:$imagereference`: This declares that the OSTree commit embedded in the image reference should be verified using the ostree remote config `$remote`. +- `ostree-image-signed:$imagereference`: Fetch via the containers/image stack, but require *some* signature verification (not via ostree). +- `ostree-unverified-image:$imagereference`: Don't do any signature verification + +``` +$ ostree-ext-cli container unencapsulate --repo=/ostree/repo ostree-remote-image:someremote:docker://quay.io/exampleos/exampleos:stable +``` + +But a project like rpm-ostree could hence support: + +``` +$ rpm-ostree rebase ostree-remote-image:someremote:quay.io/exampleos/exampleos:stable +``` + +(Along with the usual `rpm-ostree upgrade` knowing to pull that container image) + + +To emphasize this, the current high level model is that this is a one-to-one mapping - an ostree commit +can be exported (wrapped) into a container image, which will have exactly one layer. Upon import +back into an ostree repository, all container metadata except for its digested checksum will be discarded. + +#### Signatures + +OSTree supports GPG and ed25519 signatures natively, and it's expected by default that +when booting from a fetched container image, one verifies ostree-level signatures. +For ostree, a signing configuration is specified via an ostree remote. In order to +pair this configuration together, this library defines a "URL-like" string schema: +`ostree-remote-registry::` +A concrete instantiation might be e.g.: `ostree-remote-registry:fedora:quay.io/coreos/fedora-coreos:stable` +To parse and generate these strings, see [`OstreeImageReference`]. + +### Layering + +A key feature of container images is support for layering. This functionality is handled +via a separate [container/store](https://docs.rs/ostree_ext/latest/ostree_ext/container/store/) module. + +These APIs are also exposed via the CLI: + +``` +$ ostree-ext-cli container image --help +ostree-ext-cli-container-image 0.4.0-alpha.0 +Commands for working with (possibly layered, non-encapsulated) container images + +USAGE: + ostree-ext-cli container image + +FLAGS: + -h, --help Prints help information + -V, --version Prints version information + +SUBCOMMANDS: + copy Copy a pulled container image from one repo to another + deploy Perform initial deployment for a container image + help Prints this message or the help of the given subcommand(s) + list List container images + pull Pull (or update) a container image +``` + +## More details about ostree and containers + +See [ostree-and-containers.md](ostree-and-containers.md). diff --git a/ostree-ext/ci/container-build-integration.sh b/ostree-ext/ci/container-build-integration.sh new file mode 100755 index 00000000..4f10dac9 --- /dev/null +++ b/ostree-ext/ci/container-build-integration.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# Verify `ostree container commit` +set -euo pipefail + +image=quay.io/fedora/fedora-coreos:stable +example=coreos-layering-examples/tailscale +set -x + +chmod a+x ostree-ext-cli +workdir=${PWD} +cd ${example} +cp ${workdir}/ostree-ext-cli . +sed -ie 's,ostree container commit,ostree-ext-cli container commit,' Containerfile +sed -ie 's,^\(FROM .*\),\1\nADD ostree-ext-cli /usr/bin/,' Containerfile +git diff + +for runtime in podman docker; do + $runtime build -t localhost/fcos-tailscale -f Containerfile . + $runtime run --rm localhost/fcos-tailscale rpm -q tailscale +done + +cd $(mktemp -d) +cp ${workdir}/ostree-ext-cli . +cat > Containerfile << EOF +FROM $image +ADD ostree-ext-cli /usr/bin/ +RUN set -x; test \$(ostree-ext-cli internal-only-for-testing detect-env) = ostree-container +EOF +# Also verify docker buildx, which apparently doesn't have /.dockerenv +docker buildx build -t localhost/fcos-tailscale -f Containerfile . + +echo ok container image integration diff --git a/ostree-ext/ci/ima.sh b/ostree-ext/ci/ima.sh new file mode 100755 index 00000000..6be4dc61 --- /dev/null +++ b/ostree-ext/ci/ima.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# Assumes that the current environment is a mutable ostree-container +# with ostree-ext-cli installed in /usr/bin. +# Runs IMA tests. +set -xeuo pipefail + +# https://github.com/ostreedev/ostree-rs-ext/issues/417 +mkdir -p /var/tmp + +if test '!' -x /usr/bin/evmctl; then + rpm-ostree install ima-evm-utils +fi + +ostree-ext-cli internal-only-for-testing run-ima +echo ok "ima" diff --git a/ostree-ext/ci/installdeps.sh b/ostree-ext/ci/installdeps.sh new file mode 100755 index 00000000..cc79b8f9 --- /dev/null +++ b/ostree-ext/ci/installdeps.sh @@ -0,0 +1,22 @@ +#!/bin/bash +set -xeuo pipefail + +# For some reason dnf copr enable -y says there are no builds? +cat >/etc/yum.repos.d/coreos-continuous.repo << 'EOF' +[copr:copr.fedorainfracloud.org:group_CoreOS:continuous] +name=Copr repo for continuous owned by @CoreOS +baseurl=https://download.copr.fedorainfracloud.org/results/@CoreOS/continuous/fedora-$releasever-$basearch/ +type=rpm-md +skip_if_unavailable=True +gpgcheck=1 +gpgkey=https://download.copr.fedorainfracloud.org/results/@CoreOS/continuous/pubkey.gpg +repo_gpgcheck=0 +enabled=1 +enabled_metadata=1 +EOF + +# Our tests depend on this +dnf -y install skopeo + +# Always pull ostree from updates-testing to avoid the bodhi wait +dnf -y update ostree diff --git a/ostree-ext/ci/integration.sh b/ostree-ext/ci/integration.sh new file mode 100755 index 00000000..342207cd --- /dev/null +++ b/ostree-ext/ci/integration.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# Assumes that the current environment is a mutable ostree-container +# with ostree-ext-cli installed in /usr/bin. +# Runs integration tests. +set -xeuo pipefail + +# Output an ok message for TAP +n_tap_tests=0 +tap_ok() { + echo "ok" "$@" + n_tap_tests=$(($n_tap_tests+1)) +} + +tap_end() { + echo "1..${n_tap_tests}" +} + +env=$(ostree-ext-cli internal-only-for-testing detect-env) +test "${env}" = ostree-container +tap_ok environment + +ostree-ext-cli internal-only-for-testing run +tap_ok integrationtests + +tap_end diff --git a/ostree-ext/ci/lints.sh b/ostree-ext/ci/lints.sh new file mode 100755 index 00000000..4a07f669 --- /dev/null +++ b/ostree-ext/ci/lints.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -xeuo pipefail +tmpf=$(mktemp) +git grep 'dbg!' '*.rs' > ${tmpf} || true +if test -s ${tmpf}; then + echo "Found dbg!" 1>&2 + cat "${tmpf}" + exit 1 +fi \ No newline at end of file diff --git a/ostree-ext/ci/priv-integration.sh b/ostree-ext/ci/priv-integration.sh new file mode 100755 index 00000000..a226ef03 --- /dev/null +++ b/ostree-ext/ci/priv-integration.sh @@ -0,0 +1,198 @@ +#!/bin/bash +# Assumes that the current environment is a privileged container +# with the host mounted at /run/host. We can basically write +# whatever we want, however we can't actually *reboot* the host. +set -euo pipefail + +# https://github.com/ostreedev/ostree-rs-ext/issues/417 +mkdir -p /var/tmp + +sysroot=/run/host +# Current stable image fixture +image=quay.io/fedora/fedora-coreos:testing-devel +imgref=ostree-unverified-registry:${image} +stateroot=testos + +# This image was generated manually; TODO auto-generate in quay.io/coreos-assembler or better start sigstore signing our production images +FIXTURE_SIGSTORE_SIGNED_FCOS_IMAGE=quay.io/rh_ee_rsaini/coreos + +cd $(mktemp -d -p /var/tmp) + +set -x + +if test '!' -e "${sysroot}/ostree"; then + ostree admin init-fs --modern "${sysroot}" + ostree config --repo $sysroot/ostree/repo set sysroot.bootloader none +fi +if test '!' -d "${sysroot}/ostree/deploy/${stateroot}"; then + ostree admin os-init "${stateroot}" --sysroot "${sysroot}" +fi +# Should be no images pruned +ostree-ext-cli container image prune-images --sysroot "${sysroot}" +# Test the syntax which uses full imgrefs. +ostree-ext-cli container image deploy --sysroot "${sysroot}" \ + --stateroot "${stateroot}" --imgref "${imgref}" +ostree admin --sysroot="${sysroot}" status +ostree-ext-cli container image metadata --repo "${sysroot}/ostree/repo" registry:"${image}" > manifest.json +jq '.schemaVersion' < manifest.json +ostree-ext-cli container image remove --repo "${sysroot}/ostree/repo" registry:"${image}" +ostree admin --sysroot="${sysroot}" undeploy 0 +# Now test the new syntax which has a nicer --image that defaults to registry. +ostree-ext-cli container image deploy --transport registry --sysroot "${sysroot}" \ + --stateroot "${stateroot}" --image "${image}" +ostree admin --sysroot="${sysroot}" status +ostree admin --sysroot="${sysroot}" undeploy 0 +if ostree-ext-cli container image deploy --transport registry --sysroot "${sysroot}" \ + --stateroot "${stateroot}" --image "${image}" --enforce-container-sigpolicy 2>err.txt; then + echo "Deployment with enforced verification succeeded unexpectedly" 1>&2 + exit 1 +fi +if ! grep -Ee 'insecureAcceptAnything.*refusing usage' err.txt; then + echo "unexpected error" 1>&2 + cat err.txt +fi +# Now we should prune it +ostree-ext-cli container image prune-images --sysroot "${sysroot}" +ostree-ext-cli container image list --repo "${sysroot}/ostree/repo" > out.txt +test $(stat -c '%s' out.txt) = 0 + +for img in "${image}"; do + ostree-ext-cli container image deploy --sysroot "${sysroot}" \ + --stateroot "${stateroot}" --imgref ostree-unverified-registry:"${img}" + ostree admin --sysroot="${sysroot}" status + initial_refs=$(ostree --repo="${sysroot}/ostree/repo" refs | wc -l) + ostree-ext-cli container image remove --repo "${sysroot}/ostree/repo" registry:"${img}" + pruned_refs=$(ostree --repo="${sysroot}/ostree/repo" refs | wc -l) + # Removing the image should only drop the image reference, not its layers + test "$(($initial_refs - 1))" = "$pruned_refs" + ostree admin --sysroot="${sysroot}" undeploy 0 + # TODO: when we fold together ostree and ostree-ext, automatically prune layers + n_commits=$(find ${sysroot}/ostree/repo -name '*.commit' | wc -l) + test "${n_commits}" -gt 0 + # But right now this still doesn't prune *content* + ostree-ext-cli container image prune-layers --repo="${sysroot}/ostree/repo" + ostree --repo="${sysroot}/ostree/repo" refs > refs.txt + if test "$(wc -l < refs.txt)" -ne 0; then + echo "found refs" + cat refs.txt + exit 1 + fi + # And this one should GC the objects too + ostree-ext-cli container image prune-images --full --sysroot="${sysroot}" > out.txt + n_commits=$(find ${sysroot}/ostree/repo -name '*.commit' | wc -l) + test "${n_commits}" -eq 0 +done + +# Verify we have systemd journal messages +nsenter -m -t 1 journalctl _COMM=ostree-ext-cli > logs.txt +grep 'layers already present: ' logs.txt + +podman pull ${image} +ostree --repo="${sysroot}/ostree/repo" init --mode=bare-user +ostree-ext-cli container image pull ${sysroot}/ostree/repo ostree-unverified-image:containers-storage:${image} +echo "ok pulled from containers storage" + +ostree-ext-cli container compare ${imgref} ${imgref} > compare.txt +grep "Removed layers: *0 *Size: 0 bytes" compare.txt +grep "Added layers: *0 *Size: 0 bytes" compare.txt + +mkdir build +cd build +cat >Dockerfile << EOF +FROM ${image} +RUN touch /usr/share/somefile +EOF +systemd-run -dP --wait podman build -t localhost/fcos-derived . +derived_img=oci:/var/tmp/derived.oci +derived_img_dir=dir:/var/tmp/derived.dir +systemd-run -dP --wait skopeo copy containers-storage:localhost/fcos-derived "${derived_img}" +systemd-run -dP --wait skopeo copy "${derived_img}" "${derived_img_dir}" + +# Prune to reset state +ostree refs ostree/container/image --delete + +repo="${sysroot}/ostree/repo" +images=$(ostree container image list --repo "${repo}" | wc -l) +test "${images}" -eq 1 +ostree-ext-cli container image deploy --sysroot "${sysroot}" \ + --stateroot "${stateroot}" --imgref ostree-unverified-image:"${derived_img}" +imgref=$(ostree refs --repo=${repo} ostree/container/image | head -1) +img_commit=$(ostree --repo=${repo} rev-parse ostree/container/image/${imgref}) +ostree-ext-cli container image remove --repo "${repo}" "${derived_img}" + +ostree-ext-cli container image deploy --sysroot "${sysroot}" \ + --stateroot "${stateroot}" --imgref ostree-unverified-image:"${derived_img}" +img_commit2=$(ostree --repo=${repo} rev-parse ostree/container/image/${imgref}) +test "${img_commit}" = "${img_commit2}" +echo "ok deploy derived container identical revs" + +ostree-ext-cli container image deploy --sysroot "${sysroot}" \ + --stateroot "${stateroot}" --imgref ostree-unverified-image:"${derived_img_dir}" +echo "ok deploy derived container from local dir" +ostree-ext-cli container image remove --repo "${repo}" "${derived_img_dir}" +rm -rf /var/tmp/derived.dir + +# Verify policy + +mkdir -p /etc/pki/containers +#Ensure Wrong Public Key fails +cat > /etc/pki/containers/fcos.pub << EOF +-----BEGIN PUBLIC KEY----- +MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEPw/TzXY5FQ00LT2orloOuAbqoOKv +relAN0my/O8tziGvc16PtEhF6A7Eun0/9//AMRZ8BwLn2cORZiQsGd5adA== +-----END PUBLIC KEY----- +EOF + +cat > /etc/containers/registries.d/default.yaml << EOF +docker: + ${FIXTURE_SIGSTORE_SIGNED_FCOS_IMAGE}: + use-sigstore-attachments: true +EOF + +cat > /etc/containers/policy.json << EOF +{ + "default": [ + { + "type": "reject" + } + ], + "transports": { + "docker": { + "quay.io/fedora/fedora-coreos": [ + { + "type": "insecureAcceptAnything" + } + ], + "${FIXTURE_SIGSTORE_SIGNED_FCOS_IMAGE}": [ + { + "type": "sigstoreSigned", + "keyPath": "/etc/pki/containers/fcos.pub", + "signedIdentity": { + "type": "matchRepository" + } + } + ] + + } + } +} +EOF + +if ostree container image pull ${repo} ostree-image-signed:docker://${FIXTURE_SIGSTORE_SIGNED_FCOS_IMAGE} 2> error; then + echo "unexpectedly pulled image" 1>&2 + exit 1 +else + grep -q "invalid signature" error +fi + +#Ensure Correct Public Key succeeds +cat > /etc/pki/containers/fcos.pub << EOF +-----BEGIN PUBLIC KEY----- +MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEREpVb8t/Rp/78fawILAodC6EXGCG +rWNjJoPo7J99cBu5Ui4oCKD+hAHagop7GTi/G3UBP/dtduy2BVdICuBETQ== +-----END PUBLIC KEY----- +EOF +ostree container image pull ${repo} ostree-image-signed:docker://${FIXTURE_SIGSTORE_SIGNED_FCOS_IMAGE} +ostree container image history --repo ${repo} docker://${FIXTURE_SIGSTORE_SIGNED_FCOS_IMAGE} + +echo ok privileged integration diff --git a/ostree-ext/ci/priv-test-cockpit-selinux.sh b/ostree-ext/ci/priv-test-cockpit-selinux.sh new file mode 100755 index 00000000..2d71038d --- /dev/null +++ b/ostree-ext/ci/priv-test-cockpit-selinux.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# Assumes that the current environment is a privileged container +# with the host mounted at /run/host. We can basically write +# whatever we want, however we can't actually *reboot* the host. +set -euo pipefail + +sysroot=/run/host +stateroot=test-cockpit +repo=$sysroot/ostree/repo +image=registry.gitlab.com/fedora/bootc/tests/container-fixtures/cockpit +imgref=ostree-unverified-registry:${image} + +cd $(mktemp -d -p /var/tmp) + +set -x + +if test '!' -e "${sysroot}/ostree"; then + ostree admin init-fs --epoch=1 "${sysroot}" + ostree config --repo $repo set sysroot.bootloader none +fi +ostree admin stateroot-init "${stateroot}" --sysroot "${sysroot}" +ostree-ext-cli container image deploy --sysroot "${sysroot}" \ + --stateroot "${stateroot}" --imgref "${imgref}" +ref=$(ostree refs --repo $repo ostree/container/image | head -1) +commit=$(ostree rev-parse --repo $repo ostree/container/image/$ref) +ostree ls --repo $repo -X ${commit} /usr/lib/systemd/system|grep -i cockpit >out.txt +if ! grep -q :cockpit_unit_file_t:s0 out.txt; then + echo "failed to find cockpit_unit_file_t" 1>&2 + exit 1 +fi + +echo ok "derived selinux" diff --git a/ostree-ext/deny.toml b/ostree-ext/deny.toml new file mode 100644 index 00000000..24802969 --- /dev/null +++ b/ostree-ext/deny.toml @@ -0,0 +1,10 @@ +[licenses] +unlicensed = "deny" +allow = ["Apache-2.0", "Apache-2.0 WITH LLVM-exception", "MIT", "BSD-3-Clause", "BSD-2-Clause", "Unicode-DFS-2016"] + +[bans] + +[sources] +unknown-registry = "deny" +unknown-git = "deny" +allow-git = [] diff --git a/ostree-ext/docs/questions-and-answers.md b/ostree-ext/docs/questions-and-answers.md new file mode 100644 index 00000000..2c7abb32 --- /dev/null +++ b/ostree-ext/docs/questions-and-answers.md @@ -0,0 +1,40 @@ +# Questions and answers + +## module "container": Encapsulate OSTree commits in OCI/Docker images + +### How is this different from the "tarball-of-archive-repo" approach currently used in RHEL CoreOS? Aren't both encapsulating an OSTree commit in an OCI image? + +- The "tarball-of-archive-repo" approach is essentially just putting an OSTree repo in archive mode under `/srv` as an additional layer over a regular RHEL base image. In the new data format, users can do e.g. `podman run --rm -ti quay.io/fedora/fedora-coreos:stable bash`. This could be quite useful for some tests for OSTree commits (at one point we had a test that literally booted a whole VM to run `rpm -q` - it'd be much cheaper to do those kinds of "OS sanity checks" in a container). + +- The new data format is intentionally designed to be streamed; the files inside the tarball are ordered by (commit, metadata, content ...). With "tarball-of-archive-repo" as is today that's not true, so we need to pull and extract the whole thing to a temporary location, which is inefficient. See also https://github.com/ostreedev/ostree-rs-ext/issues/1. + +- We have a much clearer story for adding Docker/OCI style _derivation_ later. + +- The new data format abstracts away OSTree a bit more and avoids needing people to think about OSTree unnecessarily. + +### Why pull from a container image instead of the current (older) method of pulling from OSTree repos? + +A good example is for people who want to do offline/disconnected installations and updates. They will almost certainly have container images they want to pull too - now the OS is just another container image. Users no longer need to mirror OSTree repos. Overall, as mentioned already, we want to abstract away OSTree a bit more. + +### Can users view this as a regular container image? + +Yes, and it also provides some extras. In addition to being able to be run as a container, if the host is OSTree-based, the host itself can be deployed/updated into this image, too. There is also GPG signing and per-file integrity validation that comes with OSTree. + +### So then would this OSTree commit in container image also work as a bootimage (bootable from a USB drive)? + +No. Though there could certainly be kernels and initramfses in the (OSTree commit in the) container image, that doesn't make it bootable. OSTree _understands_ bootloaders and can update kernels/initramfs images, but it doesn't update bootloaders, that is [bootupd](https://github.com/coreos/bootupd)'s job. Furthermore, this is still a container image, made of tarballs and manifests; it is not formatted to be a disk image (e.g. it doesn't have a FAT32 formatted ESP). Related to this topic is https://github.com/iximiuz/docker-to-linux, which illustrates the difference between a docker image and a bootable image. +TL;DR, OSTree commit in container image is meant only to deliver OS updates (OSTree commits), not bootable disk images. + +### How much deduplication do we still get with this new approach? + +Unfortunately, today, we do indeed need to download more than actually needed, but the files will still be deduplicated on disk, just like before. So we still won't be storing extra files, but we will be downloading extra files. +But for users doing offline mirroring, this shouldn't matter that much. In OpenShift, the entire image is downloaded today, as well. +Nevertheless, see https://github.com/ostreedev/ostree-rs-ext/#integrating-with-future-container-deltas. + +### Will there be support for "layers" in the OSTree commit in container image? + +Not yet, but, as mentioned above, this opens up the possibility of doing OCI style derivation, so this could certainly be added later. It would be useful to make this image as familiar to admins as possible. Right now, the ostree-rs-ext client is only parsing one layer of the container image. + +### How will mirroring image registries work? + +since ostree-rs-ext uses skopeo (which uses `containers/image`), mirroring is transparently supported, i.e. admins can configure their mirroring in `containers-registries.conf` and it'll just work. diff --git a/ostree-ext/man/ostree-container-auth.md b/ostree-ext/man/ostree-container-auth.md new file mode 100644 index 00000000..234f4995 --- /dev/null +++ b/ostree-ext/man/ostree-container-auth.md @@ -0,0 +1,29 @@ +% ostree-container-auth 5 + +# NAME +ostree-container-auth description of the registry authentication file + +# DESCRIPTION + +The OSTree container stack uses the same file formats as **containers-auth(5)** but +not the same locations. + +When running as uid 0 (root), the tooling uses `/etc/ostree/auth.json` first, then looks +in `/run/ostree/auth.json`, and finally checks `/usr/lib/ostree/auth.json`. +For any other uid, the file paths used are in `${XDG_RUNTIME_DIR}/ostree/auth.json`. + +In the future, it is likely that a path that is supported for both "system podman" +usage and ostree will be added. + +## FORMAT + +The auth.json file stores, or references, credentials that allow the user to authenticate +to container image registries. +It is primarily managed by a `login` command from a container tool such as `podman login`, +`buildah login`, or `skopeo login`. + +For more information, see **containers-auth(5)**. + +# SEE ALSO + +**containers-auth(5)**, **skopeo-login(1)**, **skopeo-logout(1)** diff --git a/ostree-ext/ostree-and-containers.md b/ostree-ext/ostree-and-containers.md new file mode 100644 index 00000000..8e4b9c6e --- /dev/null +++ b/ostree-ext/ostree-and-containers.md @@ -0,0 +1,65 @@ +# ostree vs OCI/Docker + +Be sure to see the main [README.md](README.md) which describes the current architecture intersecting ostree and OCI. + +Looking at this project, one might ask: why even have ostree? Why not just have the operating system directly use something like the [containers/image](https://github.com/containers/image/) storage? + +The first answer to this is that it's a goal of this project to "hide" ostree usage; it should feel "native" to ship and manage the operating system "as if" it was just running a container. + +But, ostree has a *lot* of stuff built up around it and we can't just throw that away. + +## Understanding kernels + +ostree was designed from the start to manage bootable operating system trees - hence the name of the project. For example, ostree understands bootloaders and kernels/initramfs images. Container tools don't. + +## Signing + +ostree also quite early on gained an opinionated mechanism to sign images (commits) via GPG. As of this time there are multiple competing mechanisms for container signing, and it is not widely deployed. +For running random containers from `docker.io`, it can be OK to just trust TLS or pin via `@sha256` - a whole idea of Docker is that containers are isolated and it should be reasonably safe to +at least try out random containers. But for the *operating system* its integrity is paramount because it's ultimately trusted. + +## Deduplication + +ostree's hardlink store is designed around de-duplication. Operating systems can get large and they are most natural as "base images" - which in the Docker container model +are duplicated on disk. Of course storage systems like containers/image could learn to de-duplicate; but it would be a use case that *mostly* applied to just the operating system. + +## Being able to remove all container images + +In Kubernetes, the kubelet will prune the image storage periodically, removing images not backed by containers. If we store the operating system itself as an image...well, we'd need to do something like teach the container storage to have the concept of an image that is "pinned" because it's actually the booted filesystem. Or create a "fake" container representing the running operating system. + +Other projects in this space ended up having an "early docker" distinct from the "main docker" which brings its own large set of challenges. + +## SELinux + +OSTree has *first class* support for SELinux. It was baked into the design from the very start. Handling SELinux is very tricky because it's a part of the operating system that can influence *everything else*. And specifically file labels. + +In this approach we aren't trying to inject xattrs into the tar stream; they're stored out of band for reliability. + +## Independence of complexity of container storage + +This stuff could be done - but the container storage and tooling is already quite complex, and introducing a special case like this would be treading into new ground. + +Today for example, cri-o ships a `crio-wipe.service` which removes all container storage across major version upgrades. + +ostree is a fairly simple format and has been 100% stable throughout its life so far. + +## ostree format has per-file integrity + +More on this here: https://ostreedev.github.io/ostree/related-projects/#docker + +## Allow hiding ostree while not reinventing everything + +So, again the goal here is: make it feel "native" to ship and manage the operating system "as if" it was just running a container without throwing away everything in ostree today. + + +### Future: Running an ostree-container as a webserver + +It also should work to run the ostree-container as a webserver, which will expose a webserver that responds to `GET /repo`. + +The effect will be as if it was built from a `Dockerfile` that contains `EXPOSE 8080`; it will work to e.g. +`kubectl run nginx --image=quay.io/exampleos/exampleos:latest --replicas=1` +and then also create a service for it. + +### Integrating with future container deltas + +See https://blogs.gnome.org/alexl/2020/05/13/putting-container-updates-on-a-diet/ diff --git a/ostree-ext/src/bootabletree.rs b/ostree-ext/src/bootabletree.rs new file mode 100644 index 00000000..23ab3b72 --- /dev/null +++ b/ostree-ext/src/bootabletree.rs @@ -0,0 +1,124 @@ +//! Helper functions for bootable OSTrees. + +use std::path::Path; + +use anyhow::Result; +use camino::Utf8Path; +use camino::Utf8PathBuf; +use cap_std::fs::Dir; +use cap_std_ext::cap_std; +use ostree::gio; +use ostree::prelude::*; + +const MODULES: &str = "usr/lib/modules"; +const VMLINUZ: &str = "vmlinuz"; + +/// Find the kernel modules directory in a bootable OSTree commit. +/// The target directory will have a `vmlinuz` file representing the kernel binary. +pub fn find_kernel_dir( + root: &gio::File, + cancellable: Option<&gio::Cancellable>, +) -> Result> { + let moddir = root.resolve_relative_path(MODULES); + let e = moddir.enumerate_children( + "standard::name", + gio::FileQueryInfoFlags::NOFOLLOW_SYMLINKS, + cancellable, + )?; + let mut r = None; + for child in e.clone() { + let child = &child?; + if child.file_type() != gio::FileType::Directory { + continue; + } + let childpath = e.child(child); + let vmlinuz = childpath.child(VMLINUZ); + if !vmlinuz.query_exists(cancellable) { + continue; + } + if r.replace(childpath).is_some() { + anyhow::bail!("Found multiple subdirectories in {}", MODULES); + } + } + Ok(r) +} + +fn read_dir_optional( + d: &Dir, + p: impl AsRef, +) -> std::io::Result> { + match d.read_dir(p.as_ref()) { + Ok(r) => Ok(Some(r)), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(e) => Err(e), + } +} + +/// Find the kernel modules directory in checked out directory tree. +/// The target directory will have a `vmlinuz` file representing the kernel binary. +pub fn find_kernel_dir_fs(root: &Dir) -> Result> { + let mut r = None; + let entries = if let Some(entries) = read_dir_optional(root, MODULES)? { + entries + } else { + return Ok(None); + }; + for child in entries { + let child = &child?; + if !child.file_type()?.is_dir() { + continue; + } + let name = child.file_name(); + let name = if let Some(n) = name.to_str() { + n + } else { + continue; + }; + let mut pbuf = Utf8Path::new(MODULES).to_owned(); + pbuf.push(name); + pbuf.push(VMLINUZ); + if !root.try_exists(&pbuf)? { + continue; + } + pbuf.pop(); + if r.replace(pbuf).is_some() { + anyhow::bail!("Found multiple subdirectories in {}", MODULES); + } + } + Ok(r) +} + +#[cfg(test)] +mod test { + use super::*; + use cap_std_ext::{cap_std, cap_tempfile}; + + #[test] + fn test_find_kernel_dir_fs() -> Result<()> { + let td = cap_tempfile::tempdir(cap_std::ambient_authority())?; + + // Verify the empty case + assert!(find_kernel_dir_fs(&td).unwrap().is_none()); + let moddir = Utf8Path::new("usr/lib/modules"); + td.create_dir_all(moddir)?; + assert!(find_kernel_dir_fs(&td).unwrap().is_none()); + + let kpath = moddir.join("5.12.8-32.aarch64"); + td.create_dir_all(&kpath)?; + td.write(kpath.join("vmlinuz"), "some kernel")?; + let kpath2 = moddir.join("5.13.7-44.aarch64"); + td.create_dir_all(&kpath2)?; + td.write(kpath2.join("foo.ko"), "some kmod")?; + + assert_eq!( + find_kernel_dir_fs(&td) + .unwrap() + .unwrap() + .file_name() + .unwrap(), + kpath.file_name().unwrap() + ); + + Ok(()) + } +} diff --git a/ostree-ext/src/chunking.rs b/ostree-ext/src/chunking.rs new file mode 100644 index 00000000..da11a09e --- /dev/null +++ b/ostree-ext/src/chunking.rs @@ -0,0 +1,1007 @@ +//! Split an OSTree commit into separate chunks + +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use std::borrow::{Borrow, Cow}; +use std::collections::{BTreeMap, BTreeSet}; +use std::fmt::Write; +use std::hash::{Hash, Hasher}; +use std::num::NonZeroU32; +use std::rc::Rc; +use std::time::Instant; + +use crate::container::{COMPONENT_SEPARATOR, CONTENT_ANNOTATION}; +use crate::objectsource::{ContentID, ObjectMeta, ObjectMetaMap, ObjectSourceMeta}; +use crate::objgv::*; +use crate::statistics; +use anyhow::{anyhow, Result}; +use camino::Utf8PathBuf; +use containers_image_proxy::oci_spec; +use gvariant::aligned_bytes::TryAsAligned; +use gvariant::{Marker, Structure}; +use indexmap::IndexMap; +use ostree::{gio, glib}; +use serde::{Deserialize, Serialize}; + +/// Maximum number of layers (chunks) we will use. +// We take half the limit of 128. +// https://github.com/ostreedev/ostree-rs-ext/issues/69 +pub(crate) const MAX_CHUNKS: u32 = 64; +/// Minimum number of layers we can create in a "chunked" flow; otherwise +/// we will just drop down to one. +const MIN_CHUNKED_LAYERS: u32 = 4; + +/// A convenient alias for a reference-counted, immutable string. +pub(crate) type RcStr = Rc; +/// Maps from a checksum to its size and file names (multiple in the case of +/// hard links). +pub(crate) type ChunkMapping = BTreeMap)>; +// TODO type PackageSet = HashSet; + +const LOW_PARTITION: &str = "2ls"; +const HIGH_PARTITION: &str = "1hs"; + +#[derive(Debug, Default)] +pub(crate) struct Chunk { + pub(crate) name: String, + pub(crate) content: ChunkMapping, + pub(crate) size: u64, + pub(crate) packages: Vec, +} + +#[derive(Debug, Deserialize, Serialize)] +/// Object metadata, but with additional size data +pub struct ObjectSourceMetaSized { + /// The original metadata + #[serde(flatten)] + pub meta: ObjectSourceMeta, + /// Total size of associated objects + pub size: u64, +} + +impl Hash for ObjectSourceMetaSized { + fn hash(&self, state: &mut H) { + self.meta.identifier.hash(state); + } +} + +impl Eq for ObjectSourceMetaSized {} + +impl PartialEq for ObjectSourceMetaSized { + fn eq(&self, other: &Self) -> bool { + self.meta.identifier == other.meta.identifier + } +} + +/// Extend content source metadata with sizes. +#[derive(Debug)] +pub struct ObjectMetaSized { + /// Mapping from content object to source. + pub map: ObjectMetaMap, + /// Computed sizes of each content source + pub sizes: Vec, +} + +impl ObjectMetaSized { + /// Given object metadata and a repo, compute the size of each content source. + pub fn compute_sizes(repo: &ostree::Repo, meta: ObjectMeta) -> Result { + let cancellable = gio::Cancellable::NONE; + // Destructure into component parts; we'll create the version with sizes + let map = meta.map; + let mut set = meta.set; + // Maps content id -> total size of associated objects + let mut sizes = BTreeMap::<&str, u64>::new(); + // Populate two mappings above, iterating over the object -> contentid mapping + for (checksum, contentid) in map.iter() { + let finfo = repo.query_file(checksum, cancellable)?.0; + let sz = sizes.entry(contentid).or_default(); + *sz += finfo.size() as u64; + } + // Combine data from sizes and the content mapping. + let sized: Result> = sizes + .into_iter() + .map(|(id, size)| -> Result { + set.take(id) + .ok_or_else(|| anyhow!("Failed to find {} in content set", id)) + .map(|meta| ObjectSourceMetaSized { meta, size }) + }) + .collect(); + let mut sizes = sized?; + sizes.sort_by(|a, b| b.size.cmp(&a.size)); + Ok(ObjectMetaSized { map, sizes }) + } +} + +/// How to split up an ostree commit into "chunks" - designed to map to container image layers. +#[derive(Debug, Default)] +pub struct Chunking { + pub(crate) metadata_size: u64, + pub(crate) remainder: Chunk, + pub(crate) chunks: Vec, + + pub(crate) max: u32, + + processed_mapping: bool, + /// Number of components (e.g. packages) provided originally + pub(crate) n_provided_components: u32, + /// The above, but only ones with non-zero size + pub(crate) n_sized_components: u32, +} + +#[derive(Default)] +struct Generation { + path: Utf8PathBuf, + metadata_size: u64, + dirtree_found: BTreeSet, + dirmeta_found: BTreeSet, +} + +fn push_dirmeta(repo: &ostree::Repo, gen: &mut Generation, checksum: &str) -> Result<()> { + if gen.dirtree_found.contains(checksum) { + return Ok(()); + } + let checksum = RcStr::from(checksum); + gen.dirmeta_found.insert(RcStr::clone(&checksum)); + let child_v = repo.load_variant(ostree::ObjectType::DirMeta, checksum.borrow())?; + gen.metadata_size += child_v.data_as_bytes().as_ref().len() as u64; + Ok(()) +} + +fn push_dirtree( + repo: &ostree::Repo, + gen: &mut Generation, + checksum: &str, +) -> Result { + let child_v = repo.load_variant(ostree::ObjectType::DirTree, checksum)?; + if !gen.dirtree_found.contains(checksum) { + gen.metadata_size += child_v.data_as_bytes().as_ref().len() as u64; + } else { + let checksum = RcStr::from(checksum); + gen.dirtree_found.insert(checksum); + } + Ok(child_v) +} + +fn generate_chunking_recurse( + repo: &ostree::Repo, + gen: &mut Generation, + chunk: &mut Chunk, + dt: &glib::Variant, +) -> Result<()> { + let dt = dt.data_as_bytes(); + let dt = dt.try_as_aligned()?; + let dt = gv_dirtree!().cast(dt); + let (files, dirs) = dt.to_tuple(); + // A reusable buffer to avoid heap allocating these + let mut hexbuf = [0u8; 64]; + for file in files { + let (name, csum) = file.to_tuple(); + let fpath = gen.path.join(name.to_str()); + hex::encode_to_slice(csum, &mut hexbuf)?; + let checksum = std::str::from_utf8(&hexbuf)?; + let meta = repo.query_file(checksum, gio::Cancellable::NONE)?.0; + let size = meta.size() as u64; + let entry = chunk.content.entry(RcStr::from(checksum)).or_default(); + entry.0 = size; + let first = entry.1.is_empty(); + if first { + chunk.size += size; + } + entry.1.push(fpath); + } + for item in dirs { + let (name, contents_csum, meta_csum) = item.to_tuple(); + let name = name.to_str(); + // Extend our current path + gen.path.push(name); + hex::encode_to_slice(contents_csum, &mut hexbuf)?; + let checksum_s = std::str::from_utf8(&hexbuf)?; + let dirtree_v = push_dirtree(repo, gen, checksum_s)?; + generate_chunking_recurse(repo, gen, chunk, &dirtree_v)?; + drop(dirtree_v); + hex::encode_to_slice(meta_csum, &mut hexbuf)?; + let checksum_s = std::str::from_utf8(&hexbuf)?; + push_dirmeta(repo, gen, checksum_s)?; + // We did a push above, so pop must succeed. + assert!(gen.path.pop()); + } + Ok(()) +} + +impl Chunk { + fn new(name: &str) -> Self { + Chunk { + name: name.to_string(), + ..Default::default() + } + } + + pub(crate) fn move_obj(&mut self, dest: &mut Self, checksum: &str) -> bool { + // In most cases, we expect the object to exist in the source. However, it's + // conveneient here to simply ignore objects which were already moved into + // a chunk. + if let Some((name, (size, paths))) = self.content.remove_entry(checksum) { + let v = dest.content.insert(name, (size, paths)); + debug_assert!(v.is_none()); + self.size -= size; + dest.size += size; + true + } else { + false + } + } +} + +impl Chunking { + /// Generate an initial single chunk. + pub fn new(repo: &ostree::Repo, rev: &str) -> Result { + // Find the target commit + let rev = repo.require_rev(rev)?; + + // Load and parse the commit object + let (commit_v, _) = repo.load_commit(&rev)?; + let commit_v = commit_v.data_as_bytes(); + let commit_v = commit_v.try_as_aligned()?; + let commit = gv_commit!().cast(commit_v); + let commit = commit.to_tuple(); + + // Load it all into a single chunk + let mut gen = Generation { + path: Utf8PathBuf::from("/"), + ..Default::default() + }; + let mut chunk: Chunk = Default::default(); + + // Find the root directory tree + let contents_checksum = &hex::encode(commit.6); + let contents_v = repo.load_variant(ostree::ObjectType::DirTree, contents_checksum)?; + push_dirtree(repo, &mut gen, contents_checksum)?; + let meta_checksum = &hex::encode(commit.7); + push_dirmeta(repo, &mut gen, meta_checksum.as_str())?; + + generate_chunking_recurse(repo, &mut gen, &mut chunk, &contents_v)?; + + let chunking = Chunking { + metadata_size: gen.metadata_size, + remainder: chunk, + ..Default::default() + }; + Ok(chunking) + } + + /// Generate a chunking from an object mapping. + pub fn from_mapping( + repo: &ostree::Repo, + rev: &str, + meta: &ObjectMetaSized, + max_layers: &Option, + prior_build_metadata: Option<&oci_spec::image::ImageManifest>, + ) -> Result { + let mut r = Self::new(repo, rev)?; + r.process_mapping(meta, max_layers, prior_build_metadata)?; + Ok(r) + } + + fn remaining(&self) -> u32 { + self.max.saturating_sub(self.chunks.len() as u32) + } + + /// Given metadata about which objects are owned by a particular content source, + /// generate chunks that group together those objects. + #[allow(clippy::or_fun_call)] + pub fn process_mapping( + &mut self, + meta: &ObjectMetaSized, + max_layers: &Option, + prior_build_metadata: Option<&oci_spec::image::ImageManifest>, + ) -> Result<()> { + self.max = max_layers + .unwrap_or(NonZeroU32::new(MAX_CHUNKS).unwrap()) + .get(); + + let sizes = &meta.sizes; + // It doesn't make sense to handle multiple mappings + assert!(!self.processed_mapping); + self.processed_mapping = true; + let remaining = self.remaining(); + if remaining == 0 { + return Ok(()); + } + + // Reverses `contentmeta.map` i.e. contentid -> Vec + let mut rmap = IndexMap::>::new(); + for (checksum, contentid) in meta.map.iter() { + rmap.entry(Rc::clone(contentid)).or_default().push(checksum); + } + + // Safety: Let's assume no one has over 4 billion components. + self.n_provided_components = meta.sizes.len().try_into().unwrap(); + self.n_sized_components = sizes + .iter() + .filter(|v| v.size > 0) + .count() + .try_into() + .unwrap(); + + // TODO: Compute bin packing in a better way + let start = Instant::now(); + let packing = basic_packing( + sizes, + NonZeroU32::new(self.max).unwrap(), + prior_build_metadata, + )?; + let duration = start.elapsed(); + tracing::debug!("Time elapsed in packing: {:#?}", duration); + + for bin in packing.into_iter() { + let name = match bin.len() { + 0 => Cow::Borrowed("Reserved for new packages"), + 1 => { + let first = bin[0]; + let first_name = &*first.meta.identifier; + Cow::Borrowed(first_name) + } + 2..=5 => { + let first = bin[0]; + let first_name = &*first.meta.identifier; + let r = bin.iter().map(|v| &*v.meta.identifier).skip(1).fold( + String::from(first_name), + |mut acc, v| { + write!(acc, " and {}", v).unwrap(); + acc + }, + ); + Cow::Owned(r) + } + n => Cow::Owned(format!("{n} components")), + }; + let mut chunk = Chunk::new(&name); + chunk.packages = bin.iter().map(|v| String::from(&*v.meta.name)).collect(); + for szmeta in bin { + for &obj in rmap.get(&szmeta.meta.identifier).unwrap() { + self.remainder.move_obj(&mut chunk, obj.as_str()); + } + } + self.chunks.push(chunk); + } + + assert_eq!(self.remainder.content.len(), 0); + + Ok(()) + } + + pub(crate) fn take_chunks(&mut self) -> Vec { + let mut r = Vec::new(); + std::mem::swap(&mut self.chunks, &mut r); + r + } + + /// Print information about chunking to standard output. + pub fn print(&self) { + println!("Metadata: {}", glib::format_size(self.metadata_size)); + if self.n_provided_components > 0 { + println!( + "Components: provided={} sized={}", + self.n_provided_components, self.n_sized_components + ); + } + for (n, chunk) in self.chunks.iter().enumerate() { + let sz = glib::format_size(chunk.size); + println!( + "Chunk {}: \"{}\": objects:{} size:{}", + n, + chunk.name, + chunk.content.len(), + sz + ); + } + if !self.remainder.content.is_empty() { + let sz = glib::format_size(self.remainder.size); + println!( + "Remainder: \"{}\": objects:{} size:{}", + self.remainder.name, + self.remainder.content.len(), + sz + ); + } + } +} + +#[cfg(test)] +fn components_size(components: &[&ObjectSourceMetaSized]) -> u64 { + components.iter().map(|k| k.size).sum() +} + +/// Compute the total size of a packing +#[cfg(test)] +fn packing_size(packing: &[Vec<&ObjectSourceMetaSized>]) -> u64 { + packing.iter().map(|v| components_size(v)).sum() +} + +/// Given a certain threshold, divide a list of packages into all combinations +/// of (high, medium, low) size and (high,medium,low) using the following +/// outlier detection methods: +/// - Median and Median Absolute Deviation Method +/// Aggressively detects outliers in size and classifies them by +/// high, medium, low. The high size and low size are separate partitions +/// and deserve bins of their own +/// - Mean and Standard Deviation Method +/// The medium partition from the previous step is less aggressively +/// classified by using mean for both size and frequency +/// +/// Note: Assumes components is sorted by descending size +fn get_partitions_with_threshold<'a>( + components: &[&'a ObjectSourceMetaSized], + limit_hs_bins: usize, + threshold: f64, +) -> Option>> { + let mut partitions: BTreeMap> = BTreeMap::new(); + let mut med_size: Vec<&ObjectSourceMetaSized> = Vec::new(); + let mut high_size: Vec<&ObjectSourceMetaSized> = Vec::new(); + + let mut sizes: Vec = components.iter().map(|a| a.size).collect(); + let (median_size, mad_size) = statistics::median_absolute_deviation(&mut sizes)?; + + // We use abs here to ensure the lower limit stays positive + let size_low_limit = 0.5 * f64::abs(median_size - threshold * mad_size); + let size_high_limit = median_size + threshold * mad_size; + + for pkg in components { + let size = pkg.size as f64; + + // high size (hs) + if size >= size_high_limit { + high_size.push(pkg); + } + // low size (ls) + else if size <= size_low_limit { + partitions + .entry(LOW_PARTITION.to_string()) + .and_modify(|bin| bin.push(pkg)) + .or_insert_with(|| vec![pkg]); + } + // medium size (ms) + else { + med_size.push(pkg); + } + } + + // Extra high-size packages + let mut remaining_pkgs: Vec<_> = if high_size.len() <= limit_hs_bins { + Vec::new() + } else { + high_size.drain(limit_hs_bins..).collect() + }; + assert!(high_size.len() <= limit_hs_bins); + + // Concatenate extra high-size packages + med_sizes to keep it descending sorted + remaining_pkgs.append(&mut med_size); + partitions.insert(HIGH_PARTITION.to_string(), high_size); + + // Ascending sorted by frequency, so each partition within medium-size is freq sorted + remaining_pkgs.sort_by(|a, b| { + a.meta + .change_frequency + .partial_cmp(&b.meta.change_frequency) + .unwrap() + }); + let med_sizes: Vec = remaining_pkgs.iter().map(|a| a.size).collect(); + let med_frequencies: Vec = remaining_pkgs + .iter() + .map(|a| a.meta.change_frequency.into()) + .collect(); + + let med_mean_freq = statistics::mean(&med_frequencies)?; + let med_stddev_freq = statistics::std_deviation(&med_frequencies)?; + let med_mean_size = statistics::mean(&med_sizes)?; + let med_stddev_size = statistics::std_deviation(&med_sizes)?; + + // We use abs to avoid the lower limit being negative + let med_freq_low_limit = 0.5f64 * f64::abs(med_mean_freq - threshold * med_stddev_freq); + let med_freq_high_limit = med_mean_freq + threshold * med_stddev_freq; + let med_size_low_limit = 0.5f64 * f64::abs(med_mean_size - threshold * med_stddev_size); + let med_size_high_limit = med_mean_size + threshold * med_stddev_size; + + for pkg in remaining_pkgs { + let size = pkg.size as f64; + let freq = pkg.meta.change_frequency as f64; + + let size_name; + if size >= med_size_high_limit { + size_name = "hs"; + } else if size <= med_size_low_limit { + size_name = "ls"; + } else { + size_name = "ms"; + } + + // Numbered to maintain order of partitions in a BTreeMap of hf, mf, lf + let freq_name; + if freq >= med_freq_high_limit { + freq_name = "3hf"; + } else if freq <= med_freq_low_limit { + freq_name = "5lf"; + } else { + freq_name = "4mf"; + } + + let bucket = format!("{freq_name}_{size_name}"); + partitions + .entry(bucket.to_string()) + .and_modify(|bin| bin.push(pkg)) + .or_insert_with(|| vec![pkg]); + } + + for (name, pkgs) in &partitions { + tracing::debug!("{:#?}: {:#?}", name, pkgs.len()); + } + + Some(partitions) +} + +/// If the current rpm-ostree commit to be encapsulated is not the one in which packing structure changes, then +/// Flatten out prior_build_metadata to view all the packages in prior build as a single vec +/// Compare the flattened vector to components to see if pkgs added, updated, +/// removed or kept same +/// if pkgs added, then add them to the last bin of prior +/// if pkgs removed, then remove them from the prior[i] +/// iterate through prior[i] and make bins according to the name in nevra of pkgs to update +/// required packages +/// else if pkg structure to be changed || prior build not specified +/// Recompute optimal packaging strcuture (Compute partitions, place packages and optimize build) +fn basic_packing_with_prior_build<'a>( + components: &'a [ObjectSourceMetaSized], + bin_size: NonZeroU32, + prior_build: &oci_spec::image::ImageManifest, +) -> Result>> { + let before_processing_pkgs_len = components.len(); + + tracing::debug!("Keeping old package structure"); + + // The first layer is the ostree commit, which will always be different for different builds, + // so we ignore it. For the remaining layers, extract the components/packages in each one. + let curr_build: Result>> = prior_build + .layers() + .iter() + .skip(1) + .map(|layer| -> Result<_> { + let annotation_layer = layer + .annotations() + .as_ref() + .and_then(|annos| annos.get(CONTENT_ANNOTATION)) + .ok_or_else(|| anyhow!("Missing {CONTENT_ANNOTATION} on prior build"))?; + Ok(annotation_layer + .split(COMPONENT_SEPARATOR) + .map(ToOwned::to_owned) + .collect()) + }) + .collect(); + let mut curr_build = curr_build?; + + // View the packages as unordered sets for lookups and differencing + let prev_pkgs_set: BTreeSet = curr_build + .iter() + .flat_map(|v| v.iter().cloned()) + .filter(|name| !name.is_empty()) + .collect(); + let curr_pkgs_set: BTreeSet = components + .iter() + .map(|pkg| pkg.meta.name.to_string()) + .collect(); + + // Added packages are included in the last bin which was reserved space. + if let Some(last_bin) = curr_build.last_mut() { + let added = curr_pkgs_set.difference(&prev_pkgs_set); + last_bin.retain(|name| !name.is_empty()); + last_bin.extend(added.into_iter().cloned()); + } else { + panic!("No empty last bin for added packages"); + } + + // Handle removed packages + let removed: BTreeSet<&String> = prev_pkgs_set.difference(&curr_pkgs_set).collect(); + for bin in curr_build.iter_mut() { + bin.retain(|pkg| !removed.contains(pkg)); + } + + // Handle updated packages + let mut name_to_component: BTreeMap = BTreeMap::new(); + for component in components.iter() { + name_to_component + .entry(component.meta.name.to_string()) + .or_insert(component); + } + let mut modified_build: Vec> = Vec::new(); + for bin in curr_build { + let mut mod_bin = Vec::new(); + for pkg in bin { + // An empty component set can happen for the ostree commit layer; ignore that. + if pkg.is_empty() { + continue; + } + mod_bin.push(name_to_component[&pkg]); + } + modified_build.push(mod_bin); + } + + // Verify all packages are included + let after_processing_pkgs_len: usize = modified_build.iter().map(|b| b.len()).sum(); + assert_eq!(after_processing_pkgs_len, before_processing_pkgs_len); + assert!(modified_build.len() <= bin_size.get() as usize); + Ok(modified_build) +} + +/// Given a set of components with size metadata (e.g. boxes of a certain size) +/// and a number of bins (possible container layers) to use, determine which components +/// go in which bin. This algorithm is pretty simple: +/// Total available bins = n +/// +/// 1 bin for all the u32_max frequency pkgs +/// 1 bin for all newly added pkgs +/// 1 bin for all low size pkgs +/// +/// 60% of n-3 bins for high size pkgs +/// 40% of n-3 bins for medium size pkgs +/// +/// If HS bins > limit, spillover to MS to package +/// If MS bins > limit, fold by merging 2 bins from the end +/// +fn basic_packing<'a>( + components: &'a [ObjectSourceMetaSized], + bin_size: NonZeroU32, + prior_build_metadata: Option<&oci_spec::image::ImageManifest>, +) -> Result>> { + const HIGH_SIZE_CUTOFF: f32 = 0.6; + let before_processing_pkgs_len = components.len(); + + anyhow::ensure!(bin_size.get() >= MIN_CHUNKED_LAYERS); + + // If we have a prior build, then use that + if let Some(prior_build) = prior_build_metadata { + return basic_packing_with_prior_build(components, bin_size, prior_build); + } + + tracing::debug!("Creating new packing structure"); + + // If there are fewer packages/components than there are bins, then we don't need to do + // any "bin packing" at all; just assign a single component to each and we're done. + if before_processing_pkgs_len < bin_size.get() as usize { + let mut r = components.iter().map(|pkg| vec![pkg]).collect::>(); + if before_processing_pkgs_len > 0 { + let new_pkgs_bin: Vec<&ObjectSourceMetaSized> = Vec::new(); + r.push(new_pkgs_bin); + } + return Ok(r); + } + + let mut r = Vec::new(); + // Split off the components which are "max frequency". + let (components, max_freq_components) = components + .iter() + .partition::, _>(|pkg| pkg.meta.change_frequency != u32::MAX); + if !components.is_empty() { + // Given a total number of bins (layers), compute how many should be assigned to our + // partitioning based on size and frequency. + let limit_ls_bins = 1usize; + let limit_new_bins = 1usize; + let _limit_new_pkgs = 0usize; + let limit_max_frequency_pkgs = max_freq_components.len(); + let limit_max_frequency_bins = limit_max_frequency_pkgs.min(1); + let low_and_other_bin_limit = limit_ls_bins + limit_new_bins + limit_max_frequency_bins; + let limit_hs_bins = (HIGH_SIZE_CUTOFF + * (bin_size.get() - low_and_other_bin_limit as u32) as f32) + .floor() as usize; + let limit_ms_bins = + (bin_size.get() - (limit_hs_bins + low_and_other_bin_limit) as u32) as usize; + let partitions = get_partitions_with_threshold(&components, limit_hs_bins, 2f64) + .expect("Partitioning components into sets"); + + // Compute how many low-sized package/components we have. + let low_sized_component_count = partitions + .get(LOW_PARTITION) + .map(|p| p.len()) + .unwrap_or_default(); + + // Approximate number of components we should have per medium-size bin. + let pkg_per_bin_ms: usize = (components.len() - limit_hs_bins - low_sized_component_count) + .checked_div(limit_ms_bins) + .ok_or_else(|| anyhow::anyhow!("number of bins should be >= {}", MIN_CHUNKED_LAYERS))?; + + // Bins assignment + for (partition, pkgs) in partitions.iter() { + if partition == HIGH_PARTITION { + for pkg in pkgs { + r.push(vec![*pkg]); + } + } else if partition == LOW_PARTITION { + let mut bin: Vec<&ObjectSourceMetaSized> = Vec::new(); + for pkg in pkgs { + bin.push(*pkg); + } + r.push(bin); + } else { + let mut bin: Vec<&ObjectSourceMetaSized> = Vec::new(); + for (i, pkg) in pkgs.iter().enumerate() { + if bin.len() < pkg_per_bin_ms { + bin.push(*pkg); + } else { + r.push(bin.clone()); + bin.clear(); + bin.push(*pkg); + } + if i == pkgs.len() - 1 && !bin.is_empty() { + r.push(bin.clone()); + bin.clear(); + } + } + } + } + tracing::debug!("Bins before unoptimized build: {}", r.len()); + + // Despite allocation certain number of pkgs per bin in medium-size partitions, the + // hard limit of number of medium-size bins can be exceeded. This is because the pkg_per_bin_ms + // is only upper limit and there is no lower limit. Thus, if a partition in medium-size has only 1 pkg + // but pkg_per_bin_ms > 1, then the entire bin will have 1 pkg. This prevents partition + // mixing. + // + // Addressing medium-size bins limit breach by mergin internal MS partitions + // The partitions in medium-size are merged beginning from the end so to not mix high-frequency bins with low-frequency bins. The + // bins are kept in this order: high-frequency, medium-frequency, low-frequency. + while r.len() > (bin_size.get() as usize - limit_new_bins - limit_max_frequency_bins) { + for i in (limit_ls_bins + limit_hs_bins..r.len() - 1) + .step_by(2) + .rev() + { + if r.len() <= (bin_size.get() as usize - limit_new_bins - limit_max_frequency_bins) + { + break; + } + let prev = &r[i - 1]; + let curr = &r[i]; + let mut merge: Vec<&ObjectSourceMetaSized> = Vec::new(); + merge.extend(prev.iter()); + merge.extend(curr.iter()); + r.remove(i); + r.remove(i - 1); + r.insert(i, merge); + } + } + tracing::debug!("Bins after optimization: {}", r.len()); + } + + if !max_freq_components.is_empty() { + r.push(max_freq_components); + } + + // Allocate an empty bin for new packages + r.push(Vec::new()); + let after_processing_pkgs_len = r.iter().map(|b| b.len()).sum::(); + assert_eq!(after_processing_pkgs_len, before_processing_pkgs_len); + assert!(r.len() <= bin_size.get() as usize); + Ok(r) +} + +#[cfg(test)] +mod test { + use super::*; + + use oci_spec::image as oci_image; + use std::str::FromStr; + + const FCOS_CONTENTMETA: &[u8] = include_bytes!("fixtures/fedora-coreos-contentmeta.json.gz"); + const SHA256_EXAMPLE: &str = + "sha256:0000111122223333444455556666777788889999aaaabbbbccccddddeeeeffff"; + + #[test] + fn test_packing_basics() -> Result<()> { + // null cases + for v in [4, 7].map(|v| NonZeroU32::new(v).unwrap()) { + assert_eq!(basic_packing(&[], v, None).unwrap().len(), 0); + } + Ok(()) + } + + #[test] + fn test_packing_fcos() -> Result<()> { + let contentmeta: Vec = + serde_json::from_reader(flate2::read::GzDecoder::new(FCOS_CONTENTMETA))?; + let total_size = contentmeta.iter().map(|v| v.size).sum::(); + + let packing = + basic_packing(&contentmeta, NonZeroU32::new(MAX_CHUNKS).unwrap(), None).unwrap(); + assert!(!contentmeta.is_empty()); + // We should fit into the assigned chunk size + assert_eq!(packing.len() as u32, MAX_CHUNKS); + // And verify that the sizes match + let packed_total_size = packing_size(&packing); + assert_eq!(total_size, packed_total_size); + Ok(()) + } + + #[test] + fn test_packing_one_layer() -> Result<()> { + let contentmeta: Vec = + serde_json::from_reader(flate2::read::GzDecoder::new(FCOS_CONTENTMETA))?; + let r = basic_packing(&contentmeta, NonZeroU32::new(1).unwrap(), None); + assert!(r.is_err()); + Ok(()) + } + + fn create_manifest(prev_expected_structure: Vec>) -> oci_spec::image::ImageManifest { + use std::collections::HashMap; + + let mut p = prev_expected_structure + .iter() + .map(|b| { + b.iter() + .map(|p| p.split('.').collect::>()[0].to_string()) + .collect() + }) + .collect(); + let mut metadata_with_ostree_commit = vec![vec![String::from("ostree_commit")]]; + metadata_with_ostree_commit.append(&mut p); + + let config = oci_spec::image::DescriptorBuilder::default() + .media_type(oci_spec::image::MediaType::ImageConfig) + .size(7023_u64) + .digest(oci_image::Digest::from_str(SHA256_EXAMPLE).unwrap()) + .build() + .expect("build config descriptor"); + + let layers: Vec = metadata_with_ostree_commit + .iter() + .map(|l| { + let mut buf = [0; 8]; + let sep = COMPONENT_SEPARATOR.encode_utf8(&mut buf); + oci_spec::image::DescriptorBuilder::default() + .media_type(oci_spec::image::MediaType::ImageLayerGzip) + .size(100_u64) + .digest(oci_image::Digest::from_str(SHA256_EXAMPLE).unwrap()) + .annotations(HashMap::from([( + CONTENT_ANNOTATION.to_string(), + l.join(sep), + )])) + .build() + .expect("build layer") + }) + .collect(); + + let image_manifest = oci_spec::image::ImageManifestBuilder::default() + .schema_version(oci_spec::image::SCHEMA_VERSION) + .config(config) + .layers(layers) + .build() + .expect("build image manifest"); + image_manifest + } + + #[test] + fn test_advanced_packing() -> Result<()> { + // Step1 : Initial build (Packing sructure computed) + let contentmeta_v0: Vec = vec![ + vec![1, u32::MAX, 100000], + vec![2, u32::MAX, 99999], + vec![3, 30, 99998], + vec![4, 100, 99997], + vec![10, 51, 1000], + vec![8, 50, 500], + vec![9, 1, 200], + vec![11, 100000, 199], + vec![6, 30, 2], + vec![7, 30, 1], + ] + .iter() + .map(|data| ObjectSourceMetaSized { + meta: ObjectSourceMeta { + identifier: RcStr::from(format!("pkg{}.0", data[0])), + name: RcStr::from(format!("pkg{}", data[0])), + srcid: RcStr::from(format!("srcpkg{}", data[0])), + change_time_offset: 0, + change_frequency: data[1], + }, + size: data[2] as u64, + }) + .collect(); + + let packing = basic_packing( + &contentmeta_v0.as_slice(), + NonZeroU32::new(6).unwrap(), + None, + ) + .unwrap(); + let structure: Vec> = packing + .iter() + .map(|bin| bin.iter().map(|pkg| &*pkg.meta.identifier).collect()) + .collect(); + let v0_expected_structure = vec![ + vec!["pkg3.0"], + vec!["pkg4.0"], + vec!["pkg6.0", "pkg7.0", "pkg11.0"], + vec!["pkg9.0", "pkg8.0", "pkg10.0"], + vec!["pkg1.0", "pkg2.0"], + vec![], + ]; + assert_eq!(structure, v0_expected_structure); + + // Step 2: Derive packing structure from last build + + let mut contentmeta_v1: Vec = contentmeta_v0; + // Upgrade pkg1.0 to 1.1 + contentmeta_v1[0].meta.identifier = RcStr::from("pkg1.1"); + // Remove pkg7 + contentmeta_v1.remove(contentmeta_v1.len() - 1); + // Add pkg5 + contentmeta_v1.push(ObjectSourceMetaSized { + meta: ObjectSourceMeta { + identifier: RcStr::from("pkg5.0"), + name: RcStr::from("pkg5"), + srcid: RcStr::from("srcpkg5"), + change_time_offset: 0, + change_frequency: 42, + }, + size: 100000, + }); + + let image_manifest_v0 = create_manifest(v0_expected_structure); + let packing_derived = basic_packing( + &contentmeta_v1.as_slice(), + NonZeroU32::new(6).unwrap(), + Some(&image_manifest_v0), + ) + .unwrap(); + let structure_derived: Vec> = packing_derived + .iter() + .map(|bin| bin.iter().map(|pkg| &*pkg.meta.identifier).collect()) + .collect(); + let v1_expected_structure = vec![ + vec!["pkg3.0"], + vec!["pkg4.0"], + vec!["pkg6.0", "pkg11.0"], + vec!["pkg9.0", "pkg8.0", "pkg10.0"], + vec!["pkg1.1", "pkg2.0"], + vec!["pkg5.0"], + ]; + + assert_eq!(structure_derived, v1_expected_structure); + + // Step 3: Another update on derived where the pkg in the last bin updates + + let mut contentmeta_v2: Vec = contentmeta_v1; + // Upgrade pkg5.0 to 5.1 + contentmeta_v2[9].meta.identifier = RcStr::from("pkg5.1"); + // Add pkg12 + contentmeta_v2.push(ObjectSourceMetaSized { + meta: ObjectSourceMeta { + identifier: RcStr::from("pkg12.0"), + name: RcStr::from("pkg12"), + srcid: RcStr::from("srcpkg12"), + change_time_offset: 0, + change_frequency: 42, + }, + size: 100000, + }); + + let image_manifest_v1 = create_manifest(v1_expected_structure); + let packing_derived = basic_packing( + &contentmeta_v2.as_slice(), + NonZeroU32::new(6).unwrap(), + Some(&image_manifest_v1), + ) + .unwrap(); + let structure_derived: Vec> = packing_derived + .iter() + .map(|bin| bin.iter().map(|pkg| &*pkg.meta.identifier).collect()) + .collect(); + let v2_expected_structure = vec![ + vec!["pkg3.0"], + vec!["pkg4.0"], + vec!["pkg6.0", "pkg11.0"], + vec!["pkg9.0", "pkg8.0", "pkg10.0"], + vec!["pkg1.1", "pkg2.0"], + vec!["pkg5.1", "pkg12.0"], + ]; + + assert_eq!(structure_derived, v2_expected_structure); + Ok(()) + } +} diff --git a/ostree-ext/src/cli.rs b/ostree-ext/src/cli.rs new file mode 100644 index 00000000..f5e9e078 --- /dev/null +++ b/ostree-ext/src/cli.rs @@ -0,0 +1,1394 @@ +//! # Commandline parsing +//! +//! While there is a separate `ostree-ext-cli` crate that +//! can be installed and used directly, the CLI code is +//! also exported as a library too, so that projects +//! such as `rpm-ostree` can directly reuse it. + +use anyhow::{Context, Result}; +use camino::{Utf8Path, Utf8PathBuf}; +use cap_std::fs::Dir; +use cap_std_ext::cap_std; +use cap_std_ext::prelude::CapStdExtDirExt; +use clap::{Parser, Subcommand}; +use fn_error_context::context; +use indexmap::IndexMap; +use io_lifetimes::AsFd; +use ostree::{gio, glib}; +use std::borrow::Cow; +use std::collections::BTreeMap; +use std::ffi::OsString; +use std::fs::File; +use std::io::{BufReader, BufWriter, Write}; +use std::num::NonZeroU32; +use std::path::PathBuf; +use std::process::Command; +use tokio::sync::mpsc::Receiver; + +use crate::chunking::{ObjectMetaSized, ObjectSourceMetaSized}; +use crate::commit::container_commit; +use crate::container::store::{ExportToOCIOpts, ImportProgress, LayerProgress, PreparedImport}; +use crate::container::{self as ostree_container, ManifestDiff}; +use crate::container::{Config, ImageReference, OstreeImageReference}; +use crate::objectsource::ObjectSourceMeta; +use crate::sysroot::SysrootLock; +use ostree_container::store::{ImageImporter, PrepareResult}; +use serde::{Deserialize, Serialize}; + +/// Parse an [`OstreeImageReference`] from a CLI arguemnt. +pub fn parse_imgref(s: &str) -> Result { + OstreeImageReference::try_from(s) +} + +/// Parse a base [`ImageReference`] from a CLI arguemnt. +pub fn parse_base_imgref(s: &str) -> Result { + ImageReference::try_from(s) +} + +/// Parse an [`ostree::Repo`] from a CLI arguemnt. +pub fn parse_repo(s: &Utf8Path) -> Result { + let repofd = cap_std::fs::Dir::open_ambient_dir(s, cap_std::ambient_authority()) + .with_context(|| format!("Opening directory at '{s}'"))?; + ostree::Repo::open_at_dir(repofd.as_fd(), ".") + .with_context(|| format!("Opening ostree repository at '{s}'")) +} + +/// Options for importing a tar archive. +#[derive(Debug, Parser)] +pub(crate) struct ImportOpts { + /// Path to the repository + #[clap(long, value_parser)] + repo: Utf8PathBuf, + + /// Path to a tar archive; if unspecified, will be stdin. Currently the tar archive must not be compressed. + path: Option, +} + +/// Options for exporting a tar archive. +#[derive(Debug, Parser)] +pub(crate) struct ExportOpts { + /// Path to the repository + #[clap(long, value_parser)] + repo: Utf8PathBuf, + + /// The format version. Must be 1. + #[clap(long, hide(true))] + format_version: u32, + + /// The ostree ref or commit to export + rev: String, +} + +/// Options for import/export to tar archives. +#[derive(Debug, Subcommand)] +pub(crate) enum TarOpts { + /// Import a tar archive (currently, must not be compressed) + Import(ImportOpts), + + /// Write a tar archive to stdout + Export(ExportOpts), +} + +/// Options for container import/export. +#[derive(Debug, Subcommand)] +pub(crate) enum ContainerOpts { + #[clap(alias = "import")] + /// Import an ostree commit embedded in a remote container image + Unencapsulate { + /// Path to the repository + #[clap(long, value_parser)] + repo: Utf8PathBuf, + + #[clap(flatten)] + proxyopts: ContainerProxyOpts, + + /// Image reference, e.g. registry:quay.io/exampleos/exampleos:latest + #[clap(value_parser = parse_imgref)] + imgref: OstreeImageReference, + + /// Create an ostree ref pointing to the imported commit + #[clap(long)] + write_ref: Option, + + /// Don't display progress + #[clap(long)] + quiet: bool, + }, + + /// Print information about an exported ostree-container image. + Info { + /// Image reference, e.g. registry:quay.io/exampleos/exampleos:latest + #[clap(value_parser = parse_imgref)] + imgref: OstreeImageReference, + }, + + /// Wrap an ostree commit into a container image. + /// + /// The resulting container image will have a single layer, which is + /// very often not what's desired. To handle things more intelligently, + /// you will need to use (or create) a higher level tool that splits + /// content into distinct "chunks"; functionality for this is + /// exposed by the API but not CLI currently. + #[clap(alias = "export")] + Encapsulate { + /// Path to the repository + #[clap(long, value_parser)] + repo: Utf8PathBuf, + + /// The ostree ref or commit to export + rev: String, + + /// Image reference, e.g. registry:quay.io/exampleos/exampleos:latest + #[clap(value_parser = parse_base_imgref)] + imgref: ImageReference, + + /// Additional labels for the container + #[clap(name = "label", long, short)] + labels: Vec, + + #[clap(long)] + /// Path to Docker-formatted authentication file. + authfile: Option, + + /// Path to a JSON-formatted serialized container configuration; this is the + /// `config` property of https://github.com/opencontainers/image-spec/blob/main/config.md + #[clap(long)] + config: Option, + + /// Propagate an OSTree commit metadata key to container label + #[clap(name = "copymeta", long)] + copy_meta_keys: Vec, + + /// Propagate an optionally-present OSTree commit metadata key to container label + #[clap(name = "copymeta-opt", long)] + copy_meta_opt_keys: Vec, + + /// Corresponds to the Dockerfile `CMD` instruction. + #[clap(long)] + cmd: Option>, + + /// Compress at the fastest level (e.g. gzip level 1) + #[clap(long)] + compression_fast: bool, + + /// Path to a JSON-formatted content meta object. + #[clap(long)] + contentmeta: Option, + }, + + /// Perform build-time checking and canonicalization. + /// This is presently an optional command, but may become required in the future. + Commit, + + /// Commands for working with (possibly layered, non-encapsulated) container images. + #[clap(subcommand)] + Image(ContainerImageOpts), + + /// Compare the contents of two OCI compliant images. + Compare { + /// Image reference, e.g. ostree-remote-image:someremote:registry:quay.io/exampleos/exampleos:latest + #[clap(value_parser = parse_imgref)] + imgref_old: OstreeImageReference, + + /// Image reference, e.g. ostree-remote-image:someremote:registry:quay.io/exampleos/exampleos:latest + #[clap(value_parser = parse_imgref)] + imgref_new: OstreeImageReference, + }, +} + +/// Options for container image fetching. +#[derive(Debug, Parser)] +pub(crate) struct ContainerProxyOpts { + #[clap(long)] + /// Do not use default authentication files. + auth_anonymous: bool, + + #[clap(long)] + /// Path to Docker-formatted authentication file. + authfile: Option, + + #[clap(long)] + /// Directory with certificates (*.crt, *.cert, *.key) used to connect to registry + /// Equivalent to `skopeo --cert-dir` + cert_dir: Option, + + #[clap(long)] + /// Skip TLS verification. + insecure_skip_tls_verification: bool, +} + +/// Options for import/export to tar archives. +#[derive(Debug, Subcommand)] +pub(crate) enum ContainerImageOpts { + /// List container images + List { + /// Path to the repository + #[clap(long, value_parser)] + repo: Utf8PathBuf, + }, + + /// Pull (or update) a container image. + Pull { + /// Path to the repository + #[clap(value_parser)] + repo: Utf8PathBuf, + + /// Image reference, e.g. ostree-remote-image:someremote:registry:quay.io/exampleos/exampleos:latest + #[clap(value_parser = parse_imgref)] + imgref: OstreeImageReference, + + #[clap(flatten)] + proxyopts: ContainerProxyOpts, + + /// Don't display progress + #[clap(long)] + quiet: bool, + + /// Just check for an updated manifest, but do not download associated container layers. + /// If an updated manifest is found, a file at the provided path will be created and contain + /// the new manifest. + #[clap(long)] + check: Option, + }, + + /// Output metadata about an already stored container image. + History { + /// Path to the repository + #[clap(long, value_parser)] + repo: Utf8PathBuf, + + /// Container image reference, e.g. registry:quay.io/exampleos/exampleos:latest + #[clap(value_parser = parse_base_imgref)] + imgref: ImageReference, + }, + + /// Output manifest or configuration for an already stored container image. + Metadata { + /// Path to the repository + #[clap(long, value_parser)] + repo: Utf8PathBuf, + + /// Container image reference, e.g. registry:quay.io/exampleos/exampleos:latest + #[clap(value_parser = parse_base_imgref)] + imgref: ImageReference, + + /// Output the config, not the manifest + #[clap(long)] + config: bool, + }, + + /// Copy a pulled container image from one repo to another. + Copy { + /// Path to the source repository + #[clap(long, value_parser)] + src_repo: Utf8PathBuf, + + /// Path to the destination repository + #[clap(long, value_parser)] + dest_repo: Utf8PathBuf, + + /// Image reference, e.g. ostree-remote-image:someremote:registry:quay.io/exampleos/exampleos:latest + #[clap(value_parser = parse_imgref)] + imgref: OstreeImageReference, + }, + + /// Re-export a fetched image. + /// + /// Unlike `encapsulate`, this verb handles layered images, and will + /// also automatically preserve chunked structure from the fetched image. + Reexport { + /// Path to the repository + #[clap(long, value_parser)] + repo: Utf8PathBuf, + + /// Source image reference, e.g. registry:quay.io/exampleos/exampleos:latest + #[clap(value_parser = parse_base_imgref)] + src_imgref: ImageReference, + + /// Destination image reference, e.g. registry:quay.io/exampleos/exampleos:latest + #[clap(value_parser = parse_base_imgref)] + dest_imgref: ImageReference, + + #[clap(long)] + /// Path to Docker-formatted authentication file. + authfile: Option, + + /// Compress at the fastest level (e.g. gzip level 1) + #[clap(long)] + compression_fast: bool, + }, + + /// Replace the detached metadata (e.g. to add a signature) + ReplaceDetachedMetadata { + /// Path to the source repository + #[clap(long)] + #[clap(value_parser = parse_base_imgref)] + src: ImageReference, + + /// Target image + #[clap(long)] + #[clap(value_parser = parse_base_imgref)] + dest: ImageReference, + + /// Path to file containing new detached metadata; if not provided, + /// any existing detached metadata will be deleted. + contents: Option, + }, + + /// Unreference one or more pulled container images and perform a garbage collection. + Remove { + /// Path to the repository + #[clap(long, value_parser)] + repo: Utf8PathBuf, + + /// Image reference, e.g. quay.io/exampleos/exampleos:latest + #[clap(value_parser = parse_base_imgref)] + imgrefs: Vec, + + /// Do not garbage collect unused layers + #[clap(long)] + skip_gc: bool, + }, + + /// Garbage collect unreferenced image layer references. + PruneLayers { + /// Path to the repository + #[clap(long, value_parser)] + repo: Utf8PathBuf, + }, + + /// Garbage collect unreferenced image layer references. + PruneImages { + /// Path to the system root + #[clap(long)] + sysroot: Utf8PathBuf, + + #[clap(long)] + /// Also prune layers + and_layers: bool, + + #[clap(long, conflicts_with = "and_layers")] + /// Also prune layers and OSTree objects + full: bool, + }, + + /// Perform initial deployment for a container image + Deploy { + /// Path to the system root + #[clap(long)] + sysroot: Option, + + /// Name for the state directory, also known as "osname". + /// If the current system is booted via ostree, then this will default to the booted stateroot. + /// Otherwise, the default is `default`. + #[clap(long)] + stateroot: Option, + + /// Source image reference, e.g. ostree-remote-image:someremote:registry:quay.io/exampleos/exampleos@sha256:abcd... + /// This conflicts with `--image`. + /// This conflicts with `--image`. Supports `registry:`, `docker://`, `oci:`, `oci-archive:`, `containers-storage:`, and `dir:` + #[clap(long, required_unless_present = "image")] + imgref: Option, + + /// Name of the container image; for the `registry` transport this would be e.g. `quay.io/exampleos/foo:latest`. + /// This conflicts with `--imgref`. + #[clap(long, required_unless_present = "imgref")] + image: Option, + + /// The transport; e.g. registry, oci, oci-archive. The default is `registry`. + #[clap(long)] + transport: Option, + + /// This option does nothing and is now deprecated. Signature verification enforcement + /// proved to not be viable. + /// + /// If you want to still enforce it, use `--enforce-container-sigpolicy`. + #[clap(long, conflicts_with = "enforce_container_sigpolicy")] + no_signature_verification: bool, + + /// Require that the containers-storage stack + #[clap(long)] + enforce_container_sigpolicy: bool, + + /// Enable verification via an ostree remote + #[clap(long)] + ostree_remote: Option, + + #[clap(flatten)] + proxyopts: ContainerProxyOpts, + + /// Target image reference, e.g. ostree-remote-image:someremote:registry:quay.io/exampleos/exampleos:latest + /// + /// If specified, `--imgref` will be used as a source, but this reference will be emitted into the origin + /// so that later OS updates pull from it. + #[clap(long)] + #[clap(value_parser = parse_imgref)] + target_imgref: Option, + + /// If set, only write the layer refs, but not the final container image reference. This + /// allows generating a disk image that when booted uses "native ostree", but has layer + /// references "pre-cached" such that a container image fetch will avoid redownloading + /// everything. + #[clap(long)] + no_imgref: bool, + + #[clap(long)] + /// Add a kernel argument + karg: Option>, + + /// Write the deployed checksum to this file + #[clap(long)] + write_commitid_to: Option, + }, +} + +/// Options for deployment repair. +#[derive(Debug, Parser)] +pub(crate) enum ProvisionalRepairOpts { + AnalyzeInodes { + /// Path to the repository + #[clap(long, value_parser)] + repo: Utf8PathBuf, + + /// Print additional information + #[clap(long)] + verbose: bool, + + /// Serialize the repair result to this file as JSON + #[clap(long)] + write_result_to: Option, + }, + + Repair { + /// Path to the sysroot + #[clap(long, value_parser)] + sysroot: Utf8PathBuf, + + /// Do not mutate any system state + #[clap(long)] + dry_run: bool, + + /// Serialize the repair result to this file as JSON + #[clap(long)] + write_result_to: Option, + + /// Print additional information + #[clap(long)] + verbose: bool, + }, +} + +/// Options for the Integrity Measurement Architecture (IMA). +#[derive(Debug, Parser)] +pub(crate) struct ImaSignOpts { + /// Path to the repository + #[clap(long, value_parser)] + repo: Utf8PathBuf, + + /// The ostree ref or commit to use as a base + src_rev: String, + /// The ostree ref to use for writing the signed commit + target_ref: String, + + /// Digest algorithm + algorithm: String, + /// Path to IMA key + key: Utf8PathBuf, + + #[clap(long)] + /// Overwrite any existing signatures + overwrite: bool, +} + +/// Options for internal testing +#[derive(Debug, Subcommand)] +pub(crate) enum TestingOpts { + /// Detect the current environment + DetectEnv, + /// Generate a test fixture + CreateFixture, + /// Execute integration tests, assuming mutable environment + Run, + /// Execute IMA tests + RunIMA, + FilterTar, +} + +/// Options for man page generation +#[derive(Debug, Parser)] +pub(crate) struct ManOpts { + #[clap(long)] + /// Output to this directory + directory: Utf8PathBuf, +} + +/// Toplevel options for extended ostree functionality. +#[derive(Debug, Parser)] +#[clap(name = "ostree-ext")] +#[clap(rename_all = "kebab-case")] +#[allow(clippy::large_enum_variant)] +pub(crate) enum Opt { + /// Import and export to tar + #[clap(subcommand)] + Tar(TarOpts), + /// Import and export to a container image + #[clap(subcommand)] + Container(ContainerOpts), + /// IMA signatures + ImaSign(ImaSignOpts), + /// Internal integration testing helpers. + #[clap(hide(true), subcommand)] + #[cfg(feature = "internal-testing-api")] + InternalOnlyForTesting(TestingOpts), + #[clap(hide(true))] + #[cfg(feature = "docgen")] + Man(ManOpts), + #[clap(hide = true, subcommand)] + ProvisionalRepair(ProvisionalRepairOpts), +} + +#[allow(clippy::from_over_into)] +impl Into for ContainerProxyOpts { + fn into(self) -> ostree_container::store::ImageProxyConfig { + ostree_container::store::ImageProxyConfig { + auth_anonymous: self.auth_anonymous, + authfile: self.authfile, + certificate_directory: self.cert_dir, + insecure_skip_tls_verification: Some(self.insecure_skip_tls_verification), + ..Default::default() + } + } +} + +/// Import a tar archive containing an ostree commit. +async fn tar_import(opts: &ImportOpts) -> Result<()> { + let repo = parse_repo(&opts.repo)?; + let imported = if let Some(path) = opts.path.as_ref() { + let instream = tokio::fs::File::open(path).await?; + crate::tar::import_tar(&repo, instream, None).await? + } else { + let stdin = tokio::io::stdin(); + crate::tar::import_tar(&repo, stdin, None).await? + }; + println!("Imported: {}", imported); + Ok(()) +} + +/// Export a tar archive containing an ostree commit. +fn tar_export(opts: &ExportOpts) -> Result<()> { + let repo = parse_repo(&opts.repo)?; + #[allow(clippy::needless_update)] + let subopts = crate::tar::ExportOptions { + ..Default::default() + }; + crate::tar::export_commit(&repo, opts.rev.as_str(), std::io::stdout(), Some(subopts))?; + Ok(()) +} + +/// Render an import progress notification as a string. +pub fn layer_progress_format(p: &ImportProgress) -> String { + let (starting, s, layer) = match p { + ImportProgress::OstreeChunkStarted(v) => (true, "ostree chunk", v), + ImportProgress::OstreeChunkCompleted(v) => (false, "ostree chunk", v), + ImportProgress::DerivedLayerStarted(v) => (true, "layer", v), + ImportProgress::DerivedLayerCompleted(v) => (false, "layer", v), + }; + // podman outputs 12 characters of digest, let's add 7 for `sha256:`. + let short_digest = layer + .digest() + .digest() + .chars() + .take(12 + 7) + .collect::(); + if starting { + let size = glib::format_size(layer.size()); + format!("Fetching {s} {short_digest} ({size})") + } else { + format!("Fetched {s} {short_digest}") + } +} + +/// Write container fetch progress to standard output. +pub async fn handle_layer_progress_print( + mut layers: Receiver, + mut layer_bytes: tokio::sync::watch::Receiver>, +) { + let style = indicatif::ProgressStyle::default_bar(); + let pb = indicatif::ProgressBar::new(100); + pb.set_style( + style + .template("{prefix} {bytes} [{bar:20}] ({eta}) {msg}") + .unwrap(), + ); + loop { + tokio::select! { + // Always handle layer changes first. + biased; + layer = layers.recv() => { + if let Some(l) = layer { + if l.is_starting() { + pb.set_position(0); + } else { + pb.finish(); + } + pb.set_message(layer_progress_format(&l)); + } else { + // If the receiver is disconnected, then we're done + break + }; + }, + r = layer_bytes.changed() => { + if r.is_err() { + // If the receiver is disconnected, then we're done + break + } + let bytes = layer_bytes.borrow(); + if let Some(bytes) = &*bytes { + pb.set_length(bytes.total); + pb.set_position(bytes.fetched); + } + } + + } + } +} + +/// Write the status of layers to download. +pub fn print_layer_status(prep: &PreparedImport) { + if let Some(status) = prep.format_layer_status() { + println!("{status}"); + } +} + +/// Write a deprecation notice, and sleep for 3 seconds. +pub async fn print_deprecated_warning(msg: &str) { + eprintln!("warning: {msg}"); + tokio::time::sleep(std::time::Duration::from_secs(3)).await +} + +/// Import a container image with an encapsulated ostree commit. +async fn container_import( + repo: &ostree::Repo, + imgref: &OstreeImageReference, + proxyopts: ContainerProxyOpts, + write_ref: Option<&str>, + quiet: bool, +) -> Result<()> { + let target = indicatif::ProgressDrawTarget::stdout(); + let style = indicatif::ProgressStyle::default_bar(); + let pb = (!quiet).then(|| { + let pb = indicatif::ProgressBar::new_spinner(); + pb.set_draw_target(target); + pb.set_style(style.template("{spinner} {prefix} {msg}").unwrap()); + pb.enable_steady_tick(std::time::Duration::from_millis(200)); + pb.set_message("Downloading..."); + pb + }); + let importer = ImageImporter::new(repo, imgref, proxyopts.into()).await?; + let import = importer.unencapsulate().await; + // Ensure we finish the progress bar before potentially propagating an error + if let Some(pb) = pb.as_ref() { + pb.finish(); + } + let import = import?; + if let Some(warning) = import.deprecated_warning.as_deref() { + print_deprecated_warning(warning).await; + } + if let Some(write_ref) = write_ref { + repo.set_ref_immediate( + None, + write_ref, + Some(import.ostree_commit.as_str()), + gio::Cancellable::NONE, + )?; + println!( + "Imported: {} => {}", + write_ref, + import.ostree_commit.as_str() + ); + } else { + println!("Imported: {}", import.ostree_commit); + } + + Ok(()) +} + +/// Grouping of metadata about an object. +#[derive(Debug, Default, Serialize, Deserialize)] +pub struct RawMeta { + /// The metadata format version. Should be set to 1. + pub version: u32, + /// The image creation timestamp. Format is YYYY-MM-DDTHH:MM:SSZ. + /// Should be synced with the label io.container.image.created. + pub created: Option, + /// Top level labels, to be prefixed to the ones with --label + /// Applied to both the outer config annotations and the inner config labels. + pub labels: Option>, + /// The output layers ordered. Provided as an ordered mapping of a unique + /// machine readable strings to a human readable name (e.g., the layer contents). + /// The human-readable name is placed in a layer annotation. + pub layers: IndexMap, + /// The layer contents. The key is an ostree hash and the value is the + /// machine readable string of the layer the hash belongs to. + /// WARNING: needs to contain all ostree hashes in the input commit. + pub mapping: IndexMap, + /// Whether the mapping is ordered. If true, the output tar stream of the + /// layers will reflect the order of the hashes in the mapping. + /// Otherwise, a deterministic ordering will be used regardless of mapping + /// order. Potentially useful for optimizing zstd:chunked compression. + /// WARNING: not currently supported. + pub ordered: Option, +} + +/// Export a container image with an encapsulated ostree commit. +#[allow(clippy::too_many_arguments)] +async fn container_export( + repo: &ostree::Repo, + rev: &str, + imgref: &ImageReference, + labels: BTreeMap, + authfile: Option, + copy_meta_keys: Vec, + copy_meta_opt_keys: Vec, + container_config: Option, + cmd: Option>, + compression_fast: bool, + contentmeta: Option, +) -> Result<()> { + let container_config = if let Some(container_config) = container_config { + serde_json::from_reader(File::open(container_config).map(BufReader::new)?)? + } else { + None + }; + + let mut contentmeta_data = None; + let mut created = None; + let mut labels = labels.clone(); + if let Some(contentmeta) = contentmeta { + let buf = File::open(contentmeta).map(BufReader::new); + let raw: RawMeta = serde_json::from_reader(buf?)?; + + // Check future variables are set correctly + let supported_version = 1; + if raw.version != supported_version { + return Err(anyhow::anyhow!( + "Unsupported metadata version: {}. Currently supported: {}", + raw.version, + supported_version + )); + } + if let Some(ordered) = raw.ordered { + if ordered { + return Err(anyhow::anyhow!("Ordered mapping not currently supported.")); + } + } + + created = raw.created; + contentmeta_data = Some(ObjectMetaSized { + map: raw + .mapping + .into_iter() + .map(|(k, v)| (k, v.into())) + .collect(), + sizes: raw + .layers + .into_iter() + .map(|(k, v)| ObjectSourceMetaSized { + meta: ObjectSourceMeta { + identifier: k.clone().into(), + name: v.into(), + srcid: k.clone().into(), + change_frequency: if k == "unpackaged" { u32::MAX } else { 1 }, + change_time_offset: 1, + }, + size: 1, + }) + .collect(), + }); + + // Merge --label args to the labels from the metadata + labels.extend(raw.labels.into_iter().flatten()); + } + + // Use enough layers so that each package ends in its own layer + // while respecting the layer ordering. + let max_layers = if let Some(contentmeta_data) = &contentmeta_data { + NonZeroU32::new((contentmeta_data.sizes.len() + 1).try_into().unwrap()) + } else { + None + }; + + let config = Config { + labels: Some(labels), + cmd, + }; + + let opts = crate::container::ExportOpts { + copy_meta_keys, + copy_meta_opt_keys, + container_config, + authfile, + skip_compression: compression_fast, // TODO rename this in the struct at the next semver break + contentmeta: contentmeta_data.as_ref(), + max_layers, + created, + ..Default::default() + }; + let pushed = crate::container::encapsulate(repo, rev, &config, Some(opts), imgref).await?; + println!("{}", pushed); + Ok(()) +} + +/// Load metadata for a container image with an encapsulated ostree commit. +async fn container_info(imgref: &OstreeImageReference) -> Result<()> { + let (_, digest) = crate::container::fetch_manifest(imgref).await?; + println!("{} digest: {}", imgref, digest); + Ok(()) +} + +/// Write a layered container image into an OSTree commit. +async fn container_store( + repo: &ostree::Repo, + imgref: &OstreeImageReference, + proxyopts: ContainerProxyOpts, + quiet: bool, + check: Option, +) -> Result<()> { + let mut imp = ImageImporter::new(repo, imgref, proxyopts.into()).await?; + let prep = match imp.prepare().await? { + PrepareResult::AlreadyPresent(c) => { + println!("No changes in {} => {}", imgref, c.merge_commit); + return Ok(()); + } + PrepareResult::Ready(r) => r, + }; + if let Some(warning) = prep.deprecated_warning() { + print_deprecated_warning(warning).await; + } + if let Some(check) = check.as_deref() { + let rootfs = Dir::open_ambient_dir("/", cap_std::ambient_authority())?; + rootfs.atomic_replace_with(check.as_str().trim_start_matches('/'), |w| { + serde_json::to_writer(w, &prep.manifest).context("Serializing manifest") + })?; + // In check mode, we're done + return Ok(()); + } + if let Some(previous_state) = prep.previous_state.as_ref() { + let diff = ManifestDiff::new(&previous_state.manifest, &prep.manifest); + diff.print(); + } + print_layer_status(&prep); + let printer = (!quiet).then(|| { + let layer_progress = imp.request_progress(); + let layer_byte_progress = imp.request_layer_progress(); + tokio::task::spawn(async move { + handle_layer_progress_print(layer_progress, layer_byte_progress).await + }) + }); + let import = imp.import(prep).await; + if let Some(printer) = printer { + let _ = printer.await; + } + let import = import?; + if let Some(msg) = + ostree_container::store::image_filtered_content_warning(repo, &imgref.imgref)? + { + eprintln!("{msg}") + } + println!("Wrote: {} => {}", imgref, import.merge_commit); + Ok(()) +} + +fn print_column(s: &str, clen: u16, remaining: &mut terminal_size::Width) { + let l: u16 = s.len().try_into().unwrap(); + let l = l.min(remaining.0); + print!("{}", &s[0..l as usize]); + if clen > 0 { + // We always want two trailing spaces + let pad = clen.saturating_sub(l) + 2; + for _ in 0..pad { + print!(" "); + } + remaining.0 = remaining.0.checked_sub(l + pad).unwrap(); + } +} + +/// Output the container image history +async fn container_history(repo: &ostree::Repo, imgref: &ImageReference) -> Result<()> { + let img = crate::container::store::query_image(repo, imgref)? + .ok_or_else(|| anyhow::anyhow!("No such image: {}", imgref))?; + let columns = [("ID", 20u16), ("SIZE", 10), ("CREATED BY", 0)]; + let width = terminal_size::terminal_size() + .map(|x| x.0) + .unwrap_or(terminal_size::Width(80)); + { + let mut remaining = width; + for (name, width) in columns.iter() { + print_column(name, *width, &mut remaining); + } + println!(); + } + + let mut history = img.configuration.history().iter(); + let layers = img.manifest.layers().iter(); + for layer in layers { + let histent = history.next(); + let created_by = histent + .and_then(|s| s.created_by().as_deref()) + .unwrap_or(""); + + let mut remaining = width; + + let digest = layer.digest().digest(); + // Verify it's OK to slice, this should all be ASCII + assert!(digest.is_ascii()); + let digest_max = columns[0].1; + let digest = &digest[0..digest_max as usize]; + print_column(digest, digest_max, &mut remaining); + let size = glib::format_size(layer.size()); + print_column(size.as_str(), columns[1].1, &mut remaining); + print_column(created_by, columns[2].1, &mut remaining); + println!(); + } + Ok(()) +} + +/// Add IMA signatures to an ostree commit, generating a new commit. +fn ima_sign(cmdopts: &ImaSignOpts) -> Result<()> { + let cancellable = gio::Cancellable::NONE; + let signopts = crate::ima::ImaOpts { + algorithm: cmdopts.algorithm.clone(), + key: cmdopts.key.clone(), + overwrite: cmdopts.overwrite, + }; + let repo = parse_repo(&cmdopts.repo)?; + let tx = repo.auto_transaction(cancellable)?; + let signed_commit = crate::ima::ima_sign(&repo, cmdopts.src_rev.as_str(), &signopts)?; + repo.transaction_set_ref( + None, + cmdopts.target_ref.as_str(), + Some(signed_commit.as_str()), + ); + let _stats = tx.commit(cancellable)?; + println!("{} => {}", cmdopts.target_ref, signed_commit); + Ok(()) +} + +#[cfg(feature = "internal-testing-api")] +async fn testing(opts: &TestingOpts) -> Result<()> { + match opts { + TestingOpts::DetectEnv => { + println!("{}", crate::integrationtest::detectenv()?); + Ok(()) + } + TestingOpts::CreateFixture => crate::integrationtest::create_fixture().await, + TestingOpts::Run => crate::integrationtest::run_tests(), + TestingOpts::RunIMA => crate::integrationtest::test_ima(), + TestingOpts::FilterTar => { + let tmpdir = cap_std_ext::cap_tempfile::TempDir::new(cap_std::ambient_authority())?; + crate::tar::filter_tar( + std::io::stdin(), + std::io::stdout(), + &Default::default(), + &tmpdir, + ) + .map(|_| {}) + } + } +} + +// Quick hack; TODO dedup this with the code in bootc or lower here +#[context("Remounting sysroot writable")] +fn container_remount_sysroot(sysroot: &Utf8Path) -> Result<()> { + if !Utf8Path::new("/run/.containerenv").exists() { + return Ok(()); + } + println!("Running in container, assuming we can remount {sysroot} writable"); + let st = Command::new("mount") + .args(["-o", "remount,rw", sysroot.as_str()]) + .status()?; + if !st.success() { + anyhow::bail!("Failed to remount {sysroot}: {st:?}"); + } + Ok(()) +} + +#[context("Serializing to output file")] +fn handle_serialize_to_file(path: Option<&Utf8Path>, obj: T) -> Result<()> { + if let Some(path) = path { + let mut out = std::fs::File::create(path) + .map(BufWriter::new) + .with_context(|| anyhow::anyhow!("Opening {path} for writing"))?; + serde_json::to_writer(&mut out, &obj).context("Serializing output")?; + } + Ok(()) +} + +/// Parse the provided arguments and execute. +/// Calls [`clap::Error::exit`] on failure, printing the error message and aborting the program. +pub async fn run_from_iter(args: I) -> Result<()> +where + I: IntoIterator, + I::Item: Into + Clone, +{ + run_from_opt(Opt::parse_from(args)).await +} + +async fn run_from_opt(opt: Opt) -> Result<()> { + match opt { + Opt::Tar(TarOpts::Import(ref opt)) => tar_import(opt).await, + Opt::Tar(TarOpts::Export(ref opt)) => tar_export(opt), + Opt::Container(o) => match o { + ContainerOpts::Info { imgref } => container_info(&imgref).await, + ContainerOpts::Commit {} => container_commit().await, + ContainerOpts::Unencapsulate { + repo, + imgref, + proxyopts, + write_ref, + quiet, + } => { + let repo = parse_repo(&repo)?; + container_import(&repo, &imgref, proxyopts, write_ref.as_deref(), quiet).await + } + ContainerOpts::Encapsulate { + repo, + rev, + imgref, + labels, + authfile, + copy_meta_keys, + copy_meta_opt_keys, + config, + cmd, + compression_fast, + contentmeta, + } => { + let labels: Result> = labels + .into_iter() + .map(|l| { + let (k, v) = l + .split_once('=') + .ok_or_else(|| anyhow::anyhow!("Missing '=' in label {}", l))?; + Ok((k.to_string(), v.to_string())) + }) + .collect(); + let repo = parse_repo(&repo)?; + container_export( + &repo, + &rev, + &imgref, + labels?, + authfile, + copy_meta_keys, + copy_meta_opt_keys, + config, + cmd, + compression_fast, + contentmeta, + ) + .await + } + ContainerOpts::Image(opts) => match opts { + ContainerImageOpts::List { repo } => { + let repo = parse_repo(&repo)?; + for image in crate::container::store::list_images(&repo)? { + println!("{}", image); + } + Ok(()) + } + ContainerImageOpts::Pull { + repo, + imgref, + proxyopts, + quiet, + check, + } => { + let repo = parse_repo(&repo)?; + container_store(&repo, &imgref, proxyopts, quiet, check).await + } + ContainerImageOpts::Reexport { + repo, + src_imgref, + dest_imgref, + authfile, + compression_fast, + } => { + let repo = &parse_repo(&repo)?; + let opts = ExportToOCIOpts { + authfile, + skip_compression: compression_fast, + ..Default::default() + }; + let digest = ostree_container::store::export( + repo, + &src_imgref, + &dest_imgref, + Some(opts), + ) + .await?; + println!("Exported: {digest}"); + Ok(()) + } + ContainerImageOpts::History { repo, imgref } => { + let repo = parse_repo(&repo)?; + container_history(&repo, &imgref).await + } + ContainerImageOpts::Metadata { + repo, + imgref, + config, + } => { + let repo = parse_repo(&repo)?; + let image = crate::container::store::query_image(&repo, &imgref)? + .ok_or_else(|| anyhow::anyhow!("No such image"))?; + let stdout = std::io::stdout().lock(); + let mut stdout = std::io::BufWriter::new(stdout); + if config { + serde_json::to_writer(&mut stdout, &image.configuration)?; + } else { + serde_json::to_writer(&mut stdout, &image.manifest)?; + } + stdout.flush()?; + Ok(()) + } + ContainerImageOpts::Remove { + repo, + imgrefs, + skip_gc, + } => { + let nimgs = imgrefs.len(); + let repo = parse_repo(&repo)?; + crate::container::store::remove_images(&repo, imgrefs.iter())?; + if !skip_gc { + let nlayers = crate::container::store::gc_image_layers(&repo)?; + println!("Removed images: {nimgs} layers: {nlayers}"); + } else { + println!("Removed images: {nimgs}"); + } + Ok(()) + } + ContainerImageOpts::PruneLayers { repo } => { + let repo = parse_repo(&repo)?; + let nlayers = crate::container::store::gc_image_layers(&repo)?; + println!("Removed layers: {nlayers}"); + Ok(()) + } + ContainerImageOpts::PruneImages { + sysroot, + and_layers, + full, + } => { + let sysroot = &ostree::Sysroot::new(Some(&gio::File::for_path(&sysroot))); + sysroot.load(gio::Cancellable::NONE)?; + let sysroot = &SysrootLock::new_from_sysroot(sysroot).await?; + if full { + let res = crate::container::deploy::prune(sysroot)?; + if res.is_empty() { + println!("No content was pruned."); + } else { + println!("Removed images: {}", res.n_images); + println!("Removed layers: {}", res.n_layers); + println!("Removed objects: {}", res.n_objects_pruned); + let objsize = glib::format_size(res.objsize); + println!("Freed: {objsize}"); + } + } else { + let removed = crate::container::deploy::remove_undeployed_images(sysroot)?; + match removed.as_slice() { + [] => { + println!("No unreferenced images."); + return Ok(()); + } + o => { + for imgref in o { + println!("Removed: {imgref}"); + } + } + } + if and_layers { + let nlayers = + crate::container::store::gc_image_layers(&sysroot.repo())?; + println!("Removed layers: {nlayers}"); + } + } + Ok(()) + } + ContainerImageOpts::Copy { + src_repo, + dest_repo, + imgref, + } => { + let src_repo = parse_repo(&src_repo)?; + let dest_repo = parse_repo(&dest_repo)?; + let imgref = &imgref.imgref; + crate::container::store::copy(&src_repo, imgref, &dest_repo, imgref).await + } + ContainerImageOpts::ReplaceDetachedMetadata { + src, + dest, + contents, + } => { + let contents = contents.map(std::fs::read).transpose()?; + let digest = crate::container::update_detached_metadata( + &src, + &dest, + contents.as_deref(), + ) + .await?; + println!("Pushed: {}", digest); + Ok(()) + } + ContainerImageOpts::Deploy { + sysroot, + stateroot, + imgref, + image, + transport, + mut no_signature_verification, + enforce_container_sigpolicy, + ostree_remote, + target_imgref, + no_imgref, + karg, + proxyopts, + write_commitid_to, + } => { + // As of recent releases, signature verification enforcement is + // off by default, and must be explicitly enabled. + no_signature_verification = !enforce_container_sigpolicy; + let sysroot = &if let Some(sysroot) = sysroot { + ostree::Sysroot::new(Some(&gio::File::for_path(sysroot))) + } else { + ostree::Sysroot::new_default() + }; + sysroot.load(gio::Cancellable::NONE)?; + let repo = &sysroot.repo(); + let kargs = karg.as_deref(); + let kargs = kargs.map(|v| { + let r: Vec<_> = v.iter().map(|s| s.as_str()).collect(); + r + }); + + // If the user specified a stateroot, we always use that. + let stateroot = if let Some(stateroot) = stateroot.as_deref() { + Cow::Borrowed(stateroot) + } else { + // Otherwise, if we're booted via ostree, use the booted. + // If that doesn't hold, then use `default`. + let booted_stateroot = sysroot + .booted_deployment() + .map(|d| Cow::Owned(d.osname().to_string())); + booted_stateroot.unwrap_or({ + Cow::Borrowed(crate::container::deploy::STATEROOT_DEFAULT) + }) + }; + + let imgref = if let Some(image) = image { + let transport = transport.as_deref().unwrap_or("registry"); + let transport = ostree_container::Transport::try_from(transport)?; + let imgref = ostree_container::ImageReference { + transport, + name: image, + }; + let sigverify = if no_signature_verification { + ostree_container::SignatureSource::ContainerPolicyAllowInsecure + } else if let Some(remote) = ostree_remote.as_ref() { + ostree_container::SignatureSource::OstreeRemote(remote.to_string()) + } else { + ostree_container::SignatureSource::ContainerPolicy + }; + ostree_container::OstreeImageReference { sigverify, imgref } + } else { + // SAFETY: We use the clap required_unless_present flag, so this must be set + // because --image is not. + let imgref = imgref.expect("imgref option should be set"); + imgref.as_str().try_into()? + }; + + #[allow(clippy::needless_update)] + let options = crate::container::deploy::DeployOpts { + kargs: kargs.as_deref(), + target_imgref: target_imgref.as_ref(), + proxy_cfg: Some(proxyopts.into()), + no_imgref, + ..Default::default() + }; + let state = crate::container::deploy::deploy( + sysroot, + &stateroot, + &imgref, + Some(options), + ) + .await?; + let wrote_imgref = target_imgref.as_ref().unwrap_or(&imgref); + if let Some(msg) = ostree_container::store::image_filtered_content_warning( + repo, + &wrote_imgref.imgref, + )? { + eprintln!("{msg}") + } + if let Some(p) = write_commitid_to { + std::fs::write(&p, state.merge_commit.as_bytes()) + .with_context(|| format!("Failed to write commitid to {}", p))?; + } + Ok(()) + } + }, + ContainerOpts::Compare { + imgref_old, + imgref_new, + } => { + let (manifest_old, _) = crate::container::fetch_manifest(&imgref_old).await?; + let (manifest_new, _) = crate::container::fetch_manifest(&imgref_new).await?; + let manifest_diff = + crate::container::ManifestDiff::new(&manifest_old, &manifest_new); + manifest_diff.print(); + Ok(()) + } + }, + Opt::ImaSign(ref opts) => ima_sign(opts), + #[cfg(feature = "internal-testing-api")] + Opt::InternalOnlyForTesting(ref opts) => testing(opts).await, + #[cfg(feature = "docgen")] + Opt::Man(manopts) => crate::docgen::generate_manpages(&manopts.directory), + Opt::ProvisionalRepair(opts) => match opts { + ProvisionalRepairOpts::AnalyzeInodes { + repo, + verbose, + write_result_to, + } => { + let repo = parse_repo(&repo)?; + let check_res = crate::repair::check_inode_collision(&repo, verbose)?; + handle_serialize_to_file(write_result_to.as_deref(), &check_res)?; + if check_res.collisions.is_empty() { + println!("OK: No colliding objects found."); + } else { + eprintln!( + "warning: {} potentially colliding inodes found", + check_res.collisions.len() + ); + } + Ok(()) + } + ProvisionalRepairOpts::Repair { + sysroot, + verbose, + dry_run, + write_result_to, + } => { + container_remount_sysroot(&sysroot)?; + let sysroot = &ostree::Sysroot::new(Some(&gio::File::for_path(&sysroot))); + sysroot.load(gio::Cancellable::NONE)?; + let sysroot = &SysrootLock::new_from_sysroot(sysroot).await?; + let result = crate::repair::analyze_for_repair(sysroot, verbose)?; + handle_serialize_to_file(write_result_to.as_deref(), &result)?; + if dry_run { + result.check() + } else { + result.repair(sysroot) + } + } + }, + } +} diff --git a/ostree-ext/src/commit.rs b/ostree-ext/src/commit.rs new file mode 100644 index 00000000..babe9017 --- /dev/null +++ b/ostree-ext/src/commit.rs @@ -0,0 +1,181 @@ +//! This module contains the functions to implement the commit +//! procedures as part of building an ostree container image. +//! + +use crate::container_utils::require_ostree_container; +use crate::mountutil::is_mountpoint; +use anyhow::Context; +use anyhow::Result; +use cap_std::fs::Dir; +use cap_std::fs::MetadataExt; +use cap_std_ext::cap_std; +use cap_std_ext::dirext::CapStdExtDirExt; +use std::path::Path; +use std::path::PathBuf; +use tokio::task; + +/// Directories for which we will always remove all content. +const FORCE_CLEAN_PATHS: &[&str] = &["run", "tmp", "var/tmp", "var/cache"]; + +/// Recursively remove the target directory, but avoid traversing across mount points. +fn remove_all_on_mount_recurse(root: &Dir, rootdev: u64, path: &Path) -> Result { + let mut skipped = false; + for entry in root + .read_dir(path) + .with_context(|| format!("Reading {path:?}"))? + { + let entry = entry?; + let metadata = entry.metadata()?; + if metadata.dev() != rootdev { + skipped = true; + continue; + } + let name = entry.file_name(); + let path = &path.join(name); + + if metadata.is_dir() { + skipped |= remove_all_on_mount_recurse(root, rootdev, path.as_path())?; + } else { + root.remove_file(path) + .with_context(|| format!("Removing {path:?}"))?; + } + } + if !skipped { + root.remove_dir(path) + .with_context(|| format!("Removing {path:?}"))?; + } + Ok(skipped) +} + +fn clean_subdir(root: &Dir, rootdev: u64) -> Result<()> { + for entry in root.entries()? { + let entry = entry?; + let metadata = entry.metadata()?; + let dev = metadata.dev(); + let path = PathBuf::from(entry.file_name()); + // Ignore other filesystem mounts, e.g. podman injects /run/.containerenv + if dev != rootdev { + tracing::trace!("Skipping entry in foreign dev {path:?}"); + continue; + } + // Also ignore bind mounts, if we have a new enough kernel with statx() + // that will tell us. + if is_mountpoint(root, &path)?.unwrap_or_default() { + tracing::trace!("Skipping mount point {path:?}"); + continue; + } + if metadata.is_dir() { + remove_all_on_mount_recurse(root, rootdev, &path)?; + } else { + root.remove_file(&path) + .with_context(|| format!("Removing {path:?}"))?; + } + } + Ok(()) +} + +fn clean_paths_in(root: &Dir, rootdev: u64) -> Result<()> { + for path in FORCE_CLEAN_PATHS { + let subdir = if let Some(subdir) = root.open_dir_optional(path)? { + subdir + } else { + continue; + }; + clean_subdir(&subdir, rootdev).with_context(|| format!("Cleaning {path}"))?; + } + Ok(()) +} + +/// Given a root filesystem, clean out empty directories and warn about +/// files in /var. /run, /tmp, and /var/tmp have their contents recursively cleaned. +pub fn prepare_ostree_commit_in(root: &Dir) -> Result<()> { + let rootdev = root.dir_metadata()?.dev(); + clean_paths_in(root, rootdev) +} + +/// Like [`prepare_ostree_commit_in`] but only emits warnings about unsupported +/// files in `/var` and will not error. +pub fn prepare_ostree_commit_in_nonstrict(root: &Dir) -> Result<()> { + let rootdev = root.dir_metadata()?.dev(); + clean_paths_in(root, rootdev) +} + +/// Entrypoint to the commit procedures, initially we just +/// have one validation but we expect more in the future. +pub(crate) async fn container_commit() -> Result<()> { + task::spawn_blocking(move || { + require_ostree_container()?; + let rootdir = Dir::open_ambient_dir("/", cap_std::ambient_authority())?; + prepare_ostree_commit_in(&rootdir) + }) + .await? +} + +#[cfg(test)] +mod tests { + use super::*; + use camino::Utf8Path; + + use cap_std_ext::cap_tempfile; + + #[test] + fn commit() -> Result<()> { + let td = &cap_tempfile::tempdir(cap_std::ambient_authority())?; + + // Handle the empty case + prepare_ostree_commit_in(td).unwrap(); + prepare_ostree_commit_in_nonstrict(td).unwrap(); + + let var = Utf8Path::new("var"); + let run = Utf8Path::new("run"); + let tmp = Utf8Path::new("tmp"); + let vartmp_foobar = &var.join("tmp/foo/bar"); + let runsystemd = &run.join("systemd"); + let resolvstub = &runsystemd.join("resolv.conf"); + + for p in [var, run, tmp] { + td.create_dir(p)?; + } + + td.create_dir_all(vartmp_foobar)?; + td.write(vartmp_foobar.join("a"), "somefile")?; + td.write(vartmp_foobar.join("b"), "somefile2")?; + td.create_dir_all(runsystemd)?; + td.write(resolvstub, "stub resolv")?; + prepare_ostree_commit_in(td).unwrap(); + assert!(td.try_exists(var)?); + assert!(td.try_exists(var.join("tmp"))?); + assert!(!td.try_exists(vartmp_foobar)?); + assert!(td.try_exists(run)?); + assert!(!td.try_exists(runsystemd)?); + + let systemd = run.join("systemd"); + td.create_dir_all(&systemd)?; + prepare_ostree_commit_in(td).unwrap(); + assert!(td.try_exists(var)?); + assert!(!td.try_exists(&systemd)?); + + td.remove_dir_all(var)?; + td.create_dir(var)?; + td.write(var.join("foo"), "somefile")?; + prepare_ostree_commit_in(td).unwrap(); + // Right now we don't auto-create var/tmp if it didn't exist, but maybe + // we will in the future. + assert!(!td.try_exists(var.join("tmp"))?); + assert!(td.try_exists(var)?); + + td.write(var.join("foo"), "somefile")?; + prepare_ostree_commit_in_nonstrict(td).unwrap(); + assert!(td.try_exists(var)?); + + let nested = Utf8Path::new("var/lib/nested"); + td.create_dir_all(nested)?; + td.write(nested.join("foo"), "test1")?; + td.write(nested.join("foo2"), "test2")?; + prepare_ostree_commit_in(td).unwrap(); + assert!(td.try_exists(var)?); + assert!(td.try_exists(nested)?); + + Ok(()) + } +} diff --git a/ostree-ext/src/container/deploy.rs b/ostree-ext/src/container/deploy.rs new file mode 100644 index 00000000..4d0ec1bf --- /dev/null +++ b/ostree-ext/src/container/deploy.rs @@ -0,0 +1,221 @@ +//! Perform initial setup for a container image based system root + +use std::collections::HashSet; + +use anyhow::Result; +use fn_error_context::context; +use ostree::glib; + +use super::store::{gc_image_layers, LayeredImageState}; +use super::{ImageReference, OstreeImageReference}; +use crate::container::store::PrepareResult; +use crate::keyfileext::KeyFileExt; +use crate::sysroot::SysrootLock; + +/// The key in the OSTree origin which holds a serialized [`super::OstreeImageReference`]. +pub const ORIGIN_CONTAINER: &str = "container-image-reference"; + +/// The name of the default stateroot. +// xref https://github.com/ostreedev/ostree/issues/2794 +pub const STATEROOT_DEFAULT: &str = "default"; + +/// Options configuring deployment. +#[derive(Debug, Default)] +#[non_exhaustive] +pub struct DeployOpts<'a> { + /// Kernel arguments to use. + pub kargs: Option<&'a [&'a str]>, + /// Target image reference, as distinct from the source. + /// + /// In many cases, one may want a workflow where a system is provisioned from + /// an image with a specific digest (e.g. `quay.io/example/os@sha256:...) for + /// reproducibilty. However, one would want `ostree admin upgrade` to fetch + /// `quay.io/example/os:latest`. + /// + /// To implement this, use this option for the latter `:latest` tag. + pub target_imgref: Option<&'a OstreeImageReference>, + + /// Configuration for fetching containers. + pub proxy_cfg: Option, + + /// If true, then no image reference will be written; but there will be refs + /// for the fetched layers. This ensures that if the machine is later updated + /// to a different container image, the fetch process will reuse shared layers, but + /// it will not be necessary to remove the previous image. + pub no_imgref: bool, + + /// Do not cleanup deployments + pub no_clean: bool, +} + +/// Write a container image to an OSTree deployment. +/// +/// This API is currently intended for only an initial deployment. +#[context("Performing deployment")] +pub async fn deploy( + sysroot: &ostree::Sysroot, + stateroot: &str, + imgref: &OstreeImageReference, + options: Option>, +) -> Result> { + let cancellable = ostree::gio::Cancellable::NONE; + let options = options.unwrap_or_default(); + let repo = &sysroot.repo(); + let merge_deployment = sysroot.merge_deployment(Some(stateroot)); + let mut imp = + super::store::ImageImporter::new(repo, imgref, options.proxy_cfg.unwrap_or_default()) + .await?; + imp.require_bootable(); + if let Some(target) = options.target_imgref { + imp.set_target(target); + } + if options.no_imgref { + imp.set_no_imgref(); + } + let state = match imp.prepare().await? { + PrepareResult::AlreadyPresent(r) => r, + PrepareResult::Ready(prep) => { + if let Some(warning) = prep.deprecated_warning() { + crate::cli::print_deprecated_warning(warning).await; + } + + imp.import(prep).await? + } + }; + let commit = state.merge_commit.as_str(); + let origin = glib::KeyFile::new(); + let target_imgref = options.target_imgref.unwrap_or(imgref); + origin.set_string("origin", ORIGIN_CONTAINER, &target_imgref.to_string()); + + let opts = ostree::SysrootDeployTreeOpts { + override_kernel_argv: options.kargs, + ..Default::default() + }; + + if sysroot.booted_deployment().is_some() { + sysroot.stage_tree_with_options( + Some(stateroot), + commit, + Some(&origin), + merge_deployment.as_ref(), + &opts, + cancellable, + )?; + } else { + let deployment = &sysroot.deploy_tree_with_options( + Some(stateroot), + commit, + Some(&origin), + merge_deployment.as_ref(), + Some(&opts), + cancellable, + )?; + let flags = if options.no_clean { + ostree::SysrootSimpleWriteDeploymentFlags::NO_CLEAN + } else { + ostree::SysrootSimpleWriteDeploymentFlags::NONE + }; + sysroot.simple_write_deployment( + Some(stateroot), + deployment, + merge_deployment.as_ref(), + flags, + cancellable, + )?; + if !options.no_clean { + sysroot.cleanup(cancellable)?; + } + } + + Ok(state) +} + +/// Query the container image reference for a deployment +fn deployment_origin_container( + deploy: &ostree::Deployment, +) -> Result> { + let origin = deploy + .origin() + .map(|o| o.optional_string("origin", ORIGIN_CONTAINER)) + .transpose()? + .flatten(); + let r = origin + .map(|v| OstreeImageReference::try_from(v.as_str())) + .transpose()?; + Ok(r) +} + +/// Remove all container images which are not the target of a deployment. +/// This acts equivalently to [`super::store::remove_images()`] - the underlying layers +/// are not pruned. +/// +/// The set of removed images is returned. +pub fn remove_undeployed_images(sysroot: &SysrootLock) -> Result> { + let repo = &sysroot.repo(); + let deployment_origins: Result> = sysroot + .deployments() + .into_iter() + .filter_map(|deploy| { + deployment_origin_container(&deploy) + .map(|v| v.map(|v| v.imgref)) + .transpose() + }) + .collect(); + let deployment_origins = deployment_origins?; + // TODO add an API that returns ImageReference instead + let all_images = super::store::list_images(&sysroot.repo())? + .into_iter() + .filter_map(|img| ImageReference::try_from(img.as_str()).ok()); + let mut removed = Vec::new(); + for image in all_images { + if !deployment_origins.contains(&image) { + super::store::remove_image(repo, &image)?; + removed.push(image); + } + } + Ok(removed) +} + +/// The result of a prune operation +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Pruned { + /// The number of images that were pruned + pub n_images: u32, + /// The number of image layers that were pruned + pub n_layers: u32, + /// The number of OSTree objects that were pruned + pub n_objects_pruned: u32, + /// The total size of pruned objects + pub objsize: u64, +} + +impl Pruned { + /// Whether this prune was a no-op (i.e. no images, layers or objects were pruned). + pub fn is_empty(&self) -> bool { + self.n_images == 0 && self.n_layers == 0 && self.n_objects_pruned == 0 + } +} + +/// This combines the functionality of [`remove_undeployed_images()`] with [`super::store::gc_image_layers()`]. +pub fn prune(sysroot: &SysrootLock) -> Result { + let repo = &sysroot.repo(); + // Prune container images which are not deployed. + // SAFETY: There should never be more than u32 images + let n_images = remove_undeployed_images(sysroot)?.len().try_into().unwrap(); + // Prune unreferenced layer branches. + let n_layers = gc_image_layers(repo)?; + // Prune the objects in the repo; the above just removed refs (branches). + let (_, n_objects_pruned, objsize) = repo.prune( + ostree::RepoPruneFlags::REFS_ONLY, + 0, + ostree::gio::Cancellable::NONE, + )?; + // SAFETY: The number of pruned objects should never be negative + let n_objects_pruned = u32::try_from(n_objects_pruned).unwrap(); + Ok(Pruned { + n_images, + n_layers, + n_objects_pruned, + objsize, + }) +} diff --git a/ostree-ext/src/container/encapsulate.rs b/ostree-ext/src/container/encapsulate.rs new file mode 100644 index 00000000..5345151f --- /dev/null +++ b/ostree-ext/src/container/encapsulate.rs @@ -0,0 +1,430 @@ +//! APIs for creating container images from OSTree commits + +use super::{ImageReference, SignatureSource, OSTREE_COMMIT_LABEL}; +use super::{OstreeImageReference, Transport, COMPONENT_SEPARATOR, CONTENT_ANNOTATION}; +use crate::chunking::{Chunk, Chunking, ObjectMetaSized}; +use crate::container::skopeo; +use crate::tar as ostree_tar; +use anyhow::{anyhow, Context, Result}; +use camino::Utf8Path; +use cap_std::fs::Dir; +use cap_std_ext::cap_std; +use chrono::DateTime; +use containers_image_proxy::oci_spec; +use flate2::Compression; +use fn_error_context::context; +use gio::glib; +use oci_spec::image as oci_image; +use ocidir::{Layer, OciDir}; +use ostree::gio; +use std::borrow::Cow; +use std::collections::{BTreeMap, HashMap}; +use std::num::NonZeroU32; +use tracing::instrument; + +/// The label which may be used in addition to the standard OCI label. +pub const LEGACY_VERSION_LABEL: &str = "version"; +/// The label which indicates where the ostree layers stop, and the +/// derived ones start. +pub const DIFFID_LABEL: &str = "ostree.final-diffid"; +/// The label for bootc. +pub const BOOTC_LABEL: &str = "containers.bootc"; + +/// Annotation injected into the layer to say that this is an ostree commit. +/// However, because this gets lost when converted to D2S2 https://docs.docker.com/registry/spec/manifest-v2-2/ +/// schema, it's not actually useful today. But, we keep it +/// out of principle. +const BLOB_OSTREE_ANNOTATION: &str = "ostree.encapsulated"; +/// Configuration for the generated container. +#[derive(Debug, Default)] +pub struct Config { + /// Additional labels. + pub labels: Option>, + /// The equivalent of a `Dockerfile`'s `CMD` instruction. + pub cmd: Option>, +} + +fn commit_meta_to_labels<'a>( + meta: &glib::VariantDict, + keys: impl IntoIterator, + opt_keys: impl IntoIterator, + labels: &mut HashMap, +) -> Result<()> { + for k in keys { + let v = meta + .lookup::(k) + .context("Expected string for commit metadata value")? + .ok_or_else(|| anyhow!("Could not find commit metadata key: {}", k))?; + labels.insert(k.to_string(), v); + } + for k in opt_keys { + let v = meta + .lookup::(k) + .context("Expected string for commit metadata value")?; + if let Some(v) = v { + labels.insert(k.to_string(), v); + } + } + // Copy standard metadata keys `ostree.bootable` and `ostree.linux`. + // Bootable is an odd one out in being a boolean. + #[allow(clippy::explicit_auto_deref)] + if let Some(v) = meta.lookup::(*ostree::METADATA_KEY_BOOTABLE)? { + labels.insert(ostree::METADATA_KEY_BOOTABLE.to_string(), v.to_string()); + labels.insert(BOOTC_LABEL.into(), "1".into()); + } + // Handle any other string-typed values here. + for k in &[&ostree::METADATA_KEY_LINUX] { + if let Some(v) = meta.lookup::(k)? { + labels.insert(k.to_string(), v); + } + } + Ok(()) +} + +fn export_chunks( + repo: &ostree::Repo, + commit: &str, + ociw: &mut OciDir, + chunks: Vec, + opts: &ExportOpts, +) -> Result)>> { + chunks + .into_iter() + .enumerate() + .map(|(i, chunk)| -> Result<_> { + let mut w = ociw.create_layer(Some(opts.compression()))?; + ostree_tar::export_chunk(repo, commit, chunk.content, &mut w) + .with_context(|| format!("Exporting chunk {i}"))?; + let w = w.into_inner()?; + Ok((w.complete()?, chunk.name, chunk.packages)) + }) + .collect() +} + +/// Write an ostree commit to an OCI blob +#[context("Writing ostree root to blob")] +#[allow(clippy::too_many_arguments)] +pub(crate) fn export_chunked( + repo: &ostree::Repo, + commit: &str, + ociw: &mut OciDir, + manifest: &mut oci_image::ImageManifest, + imgcfg: &mut oci_image::ImageConfiguration, + labels: &mut HashMap, + mut chunking: Chunking, + opts: &ExportOpts, + description: &str, +) -> Result<()> { + let layers = export_chunks(repo, commit, ociw, chunking.take_chunks(), opts)?; + let compression = Some(opts.compression()); + + // In V1, the ostree layer comes first + let mut w = ociw.create_layer(compression)?; + ostree_tar::export_final_chunk(repo, commit, chunking.remainder, &mut w)?; + let w = w.into_inner()?; + let ostree_layer = w.complete()?; + + // Then, we have a label that points to the last chunk. + // Note in the pathological case of a single layer chunked v1 image, this could be the ostree layer. + let last_digest = layers + .last() + .map(|v| &v.0) + .unwrap_or(&ostree_layer) + .uncompressed_sha256 + .clone(); + + // Add the ostree layer + ociw.push_layer(manifest, imgcfg, ostree_layer, description, None); + // Add the component/content layers + let mut buf = [0; 8]; + let sep = COMPONENT_SEPARATOR.encode_utf8(&mut buf); + for (layer, name, mut packages) in layers { + let mut annotation_component_layer = HashMap::new(); + packages.sort(); + annotation_component_layer.insert(CONTENT_ANNOTATION.to_string(), packages.join(sep)); + ociw.push_layer( + manifest, + imgcfg, + layer, + name.as_str(), + Some(annotation_component_layer), + ); + } + + // This label (mentioned above) points to the last layer that is part of + // the ostree commit. + labels.insert( + DIFFID_LABEL.into(), + format!("sha256:{}", last_digest.digest()), + ); + Ok(()) +} + +/// Generate an OCI image from a given ostree root +#[context("Building oci")] +#[allow(clippy::too_many_arguments)] +fn build_oci( + repo: &ostree::Repo, + rev: &str, + writer: &mut OciDir, + tag: Option<&str>, + config: &Config, + opts: ExportOpts, +) -> Result<()> { + let commit = repo.require_rev(rev)?; + let commit = commit.as_str(); + let (commit_v, _) = repo.load_commit(commit)?; + let commit_timestamp = DateTime::from_timestamp( + ostree::commit_get_timestamp(&commit_v).try_into().unwrap(), + 0, + ) + .unwrap(); + let commit_subject = commit_v.child_value(3); + let commit_subject = commit_subject.str().ok_or_else(|| { + anyhow::anyhow!( + "Corrupted commit {}; expecting string value for subject", + commit + ) + })?; + let commit_meta = &commit_v.child_value(0); + let commit_meta = glib::VariantDict::new(Some(commit_meta)); + + let mut ctrcfg = opts.container_config.clone().unwrap_or_default(); + let mut imgcfg = oci_image::ImageConfiguration::default(); + + let created_at = opts + .created + .clone() + .unwrap_or_else(|| commit_timestamp.format("%Y-%m-%dT%H:%M:%SZ").to_string()); + imgcfg.set_created(Some(created_at)); + let mut labels = HashMap::new(); + + commit_meta_to_labels( + &commit_meta, + opts.copy_meta_keys.iter().map(|k| k.as_str()), + opts.copy_meta_opt_keys.iter().map(|k| k.as_str()), + &mut labels, + )?; + + let mut manifest = ocidir::new_empty_manifest().build().unwrap(); + + let chunking = opts + .contentmeta + .as_ref() + .map(|meta| { + crate::chunking::Chunking::from_mapping( + repo, + commit, + meta, + &opts.max_layers, + opts.prior_build, + ) + }) + .transpose()?; + // If no chunking was provided, create a logical single chunk. + let chunking = chunking + .map(Ok) + .unwrap_or_else(|| crate::chunking::Chunking::new(repo, commit))?; + + if let Some(version) = commit_meta.lookup::("version")? { + if opts.legacy_version_label { + labels.insert(LEGACY_VERSION_LABEL.into(), version.clone()); + } + labels.insert(oci_image::ANNOTATION_VERSION.into(), version); + } + labels.insert(OSTREE_COMMIT_LABEL.into(), commit.into()); + + for (k, v) in config.labels.iter().flat_map(|k| k.iter()) { + labels.insert(k.into(), v.into()); + } + + let mut annos = HashMap::new(); + annos.insert(BLOB_OSTREE_ANNOTATION.to_string(), "true".to_string()); + let description = if commit_subject.is_empty() { + Cow::Owned(format!("ostree export of commit {}", commit)) + } else { + Cow::Borrowed(commit_subject) + }; + + export_chunked( + repo, + commit, + writer, + &mut manifest, + &mut imgcfg, + &mut labels, + chunking, + &opts, + &description, + )?; + + // Lookup the cmd embedded in commit metadata + let cmd = commit_meta.lookup::>(ostree::COMMIT_META_CONTAINER_CMD)?; + // But support it being overridden by CLI options + + // https://github.com/rust-lang/rust-clippy/pull/7639#issuecomment-1050340564 + #[allow(clippy::unnecessary_lazy_evaluations)] + let cmd = config.cmd.as_ref().or_else(|| cmd.as_ref()); + if let Some(cmd) = cmd { + ctrcfg.set_cmd(Some(cmd.clone())); + } + + ctrcfg + .labels_mut() + .get_or_insert_with(Default::default) + .extend(labels.clone()); + imgcfg.set_config(Some(ctrcfg)); + let ctrcfg = writer.write_config(imgcfg)?; + manifest.set_config(ctrcfg); + manifest.set_annotations(Some(labels)); + let platform = oci_image::Platform::default(); + if let Some(tag) = tag { + writer.insert_manifest(manifest, Some(tag), platform)?; + } else { + writer.replace_with_single_manifest(manifest, platform)?; + } + + Ok(()) +} + +/// Interpret a filesystem path as optionally including a tag. Paths +/// such as `/foo/bar` will return `("/foo/bar"`, None)`, whereas +/// e.g. `/foo/bar:latest` will return `("/foo/bar", Some("latest"))`. +pub(crate) fn parse_oci_path_and_tag(path: &str) -> (&str, Option<&str>) { + match path.split_once(':') { + Some((path, tag)) => (path, Some(tag)), + None => (path, None), + } +} + +/// Helper for `build()` that avoids generics +#[instrument(level = "debug", skip_all)] +async fn build_impl( + repo: &ostree::Repo, + ostree_ref: &str, + config: &Config, + opts: Option>, + dest: &ImageReference, +) -> Result { + let mut opts = opts.unwrap_or_default(); + if dest.transport == Transport::ContainerStorage { + opts.skip_compression = true; + } + let digest = if dest.transport == Transport::OciDir { + let (path, tag) = parse_oci_path_and_tag(dest.name.as_str()); + tracing::debug!("using OCI path={path} tag={tag:?}"); + if !Utf8Path::new(path).exists() { + std::fs::create_dir(path)?; + } + let ocidir = Dir::open_ambient_dir(path, cap_std::ambient_authority())?; + let mut ocidir = OciDir::ensure(&ocidir)?; + build_oci(repo, ostree_ref, &mut ocidir, tag, config, opts)?; + None + } else { + let tempdir = { + let vartmp = Dir::open_ambient_dir("/var/tmp", cap_std::ambient_authority())?; + cap_std_ext::cap_tempfile::tempdir_in(&vartmp)? + }; + let mut ocidir = OciDir::ensure(&tempdir)?; + + // Minor TODO: refactor to avoid clone + let authfile = opts.authfile.clone(); + build_oci(repo, ostree_ref, &mut ocidir, None, config, opts)?; + drop(ocidir); + + // Pass the temporary oci directory as the current working directory for the skopeo process + let target_fd = 3i32; + let tempoci = ImageReference { + transport: Transport::OciDir, + name: format!("/proc/self/fd/{target_fd}"), + }; + let digest = skopeo::copy( + &tempoci, + dest, + authfile.as_deref(), + Some((std::sync::Arc::new(tempdir.try_clone()?.into()), target_fd)), + false, + ) + .await?; + Some(digest) + }; + if let Some(digest) = digest { + Ok(digest) + } else { + // If `skopeo copy` doesn't have `--digestfile` yet, then fall back + // to running an inspect cycle. + let imgref = OstreeImageReference { + sigverify: SignatureSource::ContainerPolicyAllowInsecure, + imgref: dest.to_owned(), + }; + let (_, digest) = super::unencapsulate::fetch_manifest(&imgref) + .await + .context("Querying manifest after push")?; + Ok(digest) + } +} + +/// Options controlling commit export into OCI +#[derive(Clone, Debug, Default)] +#[non_exhaustive] +pub struct ExportOpts<'m, 'o> { + /// If true, do not perform gzip compression of the tar layers. + pub skip_compression: bool, + /// A set of commit metadata keys to copy as image labels. + pub copy_meta_keys: Vec, + /// A set of optionally-present commit metadata keys to copy as image labels. + pub copy_meta_opt_keys: Vec, + /// Maximum number of layers to use + pub max_layers: Option, + /// Path to Docker-formatted authentication file. + pub authfile: Option, + /// Also include the legacy `version` label. + pub legacy_version_label: bool, + /// Image runtime configuration that will be used as a base + pub container_config: Option, + /// A reference to the metadata for a previous build; used to optimize + /// the packing structure. + pub prior_build: Option<&'m oci_image::ImageManifest>, + /// Metadata mapping between objects and their owning component/package; + /// used to optimize packing. + pub contentmeta: Option<&'o ObjectMetaSized>, + /// Sets the created tag in the image manifest. + pub created: Option, +} + +impl<'m, 'o> ExportOpts<'m, 'o> { + /// Return the gzip compression level to use, as configured by the export options. + fn compression(&self) -> Compression { + if self.skip_compression { + Compression::fast() + } else { + Compression::default() + } + } +} + +/// Given an OSTree repository and ref, generate a container image. +/// +/// The returned `ImageReference` will contain a digested (e.g. `@sha256:`) version of the destination. +pub async fn encapsulate>( + repo: &ostree::Repo, + ostree_ref: S, + config: &Config, + opts: Option>, + dest: &ImageReference, +) -> Result { + build_impl(repo, ostree_ref.as_ref(), config, opts, dest).await +} + +#[test] +fn test_parse_ocipath() { + let default = "/foo/bar"; + let untagged = "/foo/bar:baz"; + let tagged = "/foo/bar:baz:latest"; + assert_eq!(parse_oci_path_and_tag(default), ("/foo/bar", None)); + assert_eq!( + parse_oci_path_and_tag(tagged), + ("/foo/bar", Some("baz:latest")) + ); + assert_eq!(parse_oci_path_and_tag(untagged), ("/foo/bar", Some("baz"))); +} diff --git a/ostree-ext/src/container/mod.rs b/ostree-ext/src/container/mod.rs new file mode 100644 index 00000000..c1d31ee2 --- /dev/null +++ b/ostree-ext/src/container/mod.rs @@ -0,0 +1,646 @@ +//! # APIs bridging OSTree and container images +//! +//! This module contains APIs to bidirectionally map between a single OSTree commit and a container image wrapping it. +//! Because container images are just layers of tarballs, this builds on the [`crate::tar`] module. +//! +//! To emphasize this, the current high level model is that this is a one-to-one mapping - an ostree commit +//! can be exported (wrapped) into a container image, which will have exactly one layer. Upon import +//! back into an ostree repository, all container metadata except for its digested checksum will be discarded. +//! +//! ## Signatures +//! +//! OSTree supports GPG and ed25519 signatures natively, and it's expected by default that +//! when booting from a fetched container image, one verifies ostree-level signatures. +//! For ostree, a signing configuration is specified via an ostree remote. In order to +//! pair this configuration together, this library defines a "URL-like" string schema: +//! +//! `ostree-remote-registry::` +//! +//! A concrete instantiation might be e.g.: `ostree-remote-registry:fedora:quay.io/coreos/fedora-coreos:stable` +//! +//! To parse and generate these strings, see [`OstreeImageReference`]. +//! +//! ## Layering +//! +//! A key feature of container images is support for layering. At the moment, support +//! for this is [planned but not implemented](https://github.com/ostreedev/ostree-rs-ext/issues/12). + +use anyhow::anyhow; +use cap_std_ext::cap_std; +use cap_std_ext::cap_std::fs::Dir; +use containers_image_proxy::oci_spec; +use ostree::glib; +use serde::Serialize; + +use std::borrow::Cow; +use std::collections::HashMap; +use std::fmt::Debug; +use std::ops::Deref; +use std::str::FromStr; + +/// The label injected into a container image that contains the ostree commit SHA-256. +pub const OSTREE_COMMIT_LABEL: &str = "ostree.commit"; + +/// The name of an annotation attached to a layer which names the packages/components +/// which are part of it. +pub(crate) const CONTENT_ANNOTATION: &str = "ostree.components"; +/// The character we use to separate values in [`CONTENT_ANNOTATION`]. +pub(crate) const COMPONENT_SEPARATOR: char = ','; + +/// Our generic catchall fatal error, expected to be converted +/// to a string to output to a terminal or logs. +type Result = anyhow::Result; + +/// A backend/transport for OCI/Docker images. +#[derive(Copy, Clone, Hash, Debug, PartialEq, Eq)] +pub enum Transport { + /// A remote Docker/OCI registry (`registry:` or `docker://`) + Registry, + /// A local OCI directory (`oci:`) + OciDir, + /// A local OCI archive tarball (`oci-archive:`) + OciArchive, + /// A local Docker archive tarball (`docker-archive:`) + DockerArchive, + /// Local container storage (`containers-storage:`) + ContainerStorage, + /// Local directory (`dir:`) + Dir, +} + +/// Combination of a remote image reference and transport. +/// +/// For example, +#[derive(Debug, Clone, Hash, PartialEq, Eq)] +pub struct ImageReference { + /// The storage and transport for the image + pub transport: Transport, + /// The image name (e.g. `quay.io/somerepo/someimage:latest`) + pub name: String, +} + +/// Policy for signature verification. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum SignatureSource { + /// Fetches will use the named ostree remote for signature verification of the ostree commit. + OstreeRemote(String), + /// Fetches will defer to the `containers-policy.json`, but we make a best effort to reject `default: insecureAcceptAnything` policy. + ContainerPolicy, + /// NOT RECOMMENDED. Fetches will defer to the `containers-policy.json` default which is usually `insecureAcceptAnything`. + ContainerPolicyAllowInsecure, +} + +/// A commonly used pre-OCI label for versions. +pub const LABEL_VERSION: &str = "version"; + +/// Combination of a signature verification mechanism, and a standard container image reference. +/// +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct OstreeImageReference { + /// The signature verification mechanism. + pub sigverify: SignatureSource, + /// The container image reference. + pub imgref: ImageReference, +} + +impl TryFrom<&str> for Transport { + type Error = anyhow::Error; + + fn try_from(value: &str) -> Result { + Ok(match value { + Self::REGISTRY_STR | "docker" => Self::Registry, + Self::OCI_STR => Self::OciDir, + Self::OCI_ARCHIVE_STR => Self::OciArchive, + Self::DOCKER_ARCHIVE_STR => Self::DockerArchive, + Self::CONTAINERS_STORAGE_STR => Self::ContainerStorage, + Self::LOCAL_DIRECTORY_STR => Self::Dir, + o => return Err(anyhow!("Unknown transport '{}'", o)), + }) + } +} + +impl Transport { + const OCI_STR: &'static str = "oci"; + const OCI_ARCHIVE_STR: &'static str = "oci-archive"; + const DOCKER_ARCHIVE_STR: &'static str = "docker-archive"; + const CONTAINERS_STORAGE_STR: &'static str = "containers-storage"; + const LOCAL_DIRECTORY_STR: &'static str = "dir"; + const REGISTRY_STR: &'static str = "registry"; + + /// Retrieve an identifier that can then be re-parsed from [`Transport::try_from::<&str>`]. + pub fn serializable_name(&self) -> &'static str { + match self { + Transport::Registry => Self::REGISTRY_STR, + Transport::OciDir => Self::OCI_STR, + Transport::OciArchive => Self::OCI_ARCHIVE_STR, + Transport::DockerArchive => Self::DOCKER_ARCHIVE_STR, + Transport::ContainerStorage => Self::CONTAINERS_STORAGE_STR, + Transport::Dir => Self::LOCAL_DIRECTORY_STR, + } + } +} + +impl TryFrom<&str> for ImageReference { + type Error = anyhow::Error; + + fn try_from(value: &str) -> Result { + let (transport_name, mut name) = value + .split_once(':') + .ok_or_else(|| anyhow!("Missing ':' in {}", value))?; + let transport: Transport = transport_name.try_into()?; + if name.is_empty() { + return Err(anyhow!("Invalid empty name in {}", value)); + } + if transport_name == "docker" { + name = name + .strip_prefix("//") + .ok_or_else(|| anyhow!("Missing // in docker:// in {}", value))?; + } + Ok(Self { + transport, + name: name.to_string(), + }) + } +} + +impl FromStr for ImageReference { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + Self::try_from(s) + } +} + +impl TryFrom<&str> for SignatureSource { + type Error = anyhow::Error; + + fn try_from(value: &str) -> Result { + match value { + "ostree-image-signed" => Ok(Self::ContainerPolicy), + "ostree-unverified-image" => Ok(Self::ContainerPolicyAllowInsecure), + o => match o.strip_prefix("ostree-remote-image:") { + Some(rest) => Ok(Self::OstreeRemote(rest.to_string())), + _ => Err(anyhow!("Invalid signature source: {}", o)), + }, + } + } +} + +impl FromStr for SignatureSource { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + Self::try_from(s) + } +} + +impl TryFrom<&str> for OstreeImageReference { + type Error = anyhow::Error; + + fn try_from(value: &str) -> Result { + let (first, second) = value + .split_once(':') + .ok_or_else(|| anyhow!("Missing ':' in {}", value))?; + let (sigverify, rest) = match first { + "ostree-image-signed" => (SignatureSource::ContainerPolicy, Cow::Borrowed(second)), + "ostree-unverified-image" => ( + SignatureSource::ContainerPolicyAllowInsecure, + Cow::Borrowed(second), + ), + // Shorthand for ostree-unverified-image:registry: + "ostree-unverified-registry" => ( + SignatureSource::ContainerPolicyAllowInsecure, + Cow::Owned(format!("registry:{second}")), + ), + // This is a shorthand for ostree-remote-image with registry: + "ostree-remote-registry" => { + let (remote, rest) = second + .split_once(':') + .ok_or_else(|| anyhow!("Missing second ':' in {}", value))?; + ( + SignatureSource::OstreeRemote(remote.to_string()), + Cow::Owned(format!("registry:{rest}")), + ) + } + "ostree-remote-image" => { + let (remote, rest) = second + .split_once(':') + .ok_or_else(|| anyhow!("Missing second ':' in {}", value))?; + ( + SignatureSource::OstreeRemote(remote.to_string()), + Cow::Borrowed(rest), + ) + } + o => { + return Err(anyhow!("Invalid ostree image reference scheme: {}", o)); + } + }; + let imgref = rest.deref().try_into()?; + Ok(Self { sigverify, imgref }) + } +} + +impl FromStr for OstreeImageReference { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + Self::try_from(s) + } +} + +impl std::fmt::Display for Transport { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let s = match self { + // TODO once skopeo supports this, canonicalize as registry: + Self::Registry => "docker://", + Self::OciArchive => "oci-archive:", + Self::DockerArchive => "docker-archive:", + Self::OciDir => "oci:", + Self::ContainerStorage => "containers-storage:", + Self::Dir => "dir:", + }; + f.write_str(s) + } +} + +impl std::fmt::Display for ImageReference { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}{}", self.transport, self.name) + } +} + +impl std::fmt::Display for SignatureSource { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + SignatureSource::OstreeRemote(r) => write!(f, "ostree-remote-image:{r}"), + SignatureSource::ContainerPolicy => write!(f, "ostree-image-signed"), + SignatureSource::ContainerPolicyAllowInsecure => { + write!(f, "ostree-unverified-image") + } + } + } +} + +impl std::fmt::Display for OstreeImageReference { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match (&self.sigverify, &self.imgref) { + (SignatureSource::ContainerPolicyAllowInsecure, imgref) + if imgref.transport == Transport::Registry => + { + // Because allow-insecure is the effective default, allow formatting + // without it. Note this formatting is asymmetric and cannot be + // re-parsed. + if f.alternate() { + write!(f, "{}", self.imgref) + } else { + write!(f, "ostree-unverified-registry:{}", self.imgref.name) + } + } + (sigverify, imgref) => { + write!(f, "{}:{}", sigverify, imgref) + } + } + } +} + +/// Represents the difference in layer/blob content between two OCI image manifests. +#[derive(Debug, Serialize)] +pub struct ManifestDiff<'a> { + /// The source container image manifest. + #[serde(skip)] + pub from: &'a oci_spec::image::ImageManifest, + /// The target container image manifest. + #[serde(skip)] + pub to: &'a oci_spec::image::ImageManifest, + /// Layers which are present in the old image but not the new image. + #[serde(skip)] + pub removed: Vec<&'a oci_spec::image::Descriptor>, + /// Layers which are present in the new image but not the old image. + #[serde(skip)] + pub added: Vec<&'a oci_spec::image::Descriptor>, + /// Total number of layers + pub total: u64, + /// Size of total number of layers. + pub total_size: u64, + /// Number of layers removed + pub n_removed: u64, + /// Size of the number of layers removed + pub removed_size: u64, + /// Number of packages added + pub n_added: u64, + /// Size of the number of layers added + pub added_size: u64, +} + +impl<'a> ManifestDiff<'a> { + /// Compute the layer difference between two OCI image manifests. + pub fn new( + src: &'a oci_spec::image::ImageManifest, + dest: &'a oci_spec::image::ImageManifest, + ) -> Self { + let src_layers = src + .layers() + .iter() + .map(|l| (l.digest().digest(), l)) + .collect::>(); + let dest_layers = dest + .layers() + .iter() + .map(|l| (l.digest().digest(), l)) + .collect::>(); + let mut removed = Vec::new(); + let mut added = Vec::new(); + for (blobid, &descriptor) in src_layers.iter() { + if !dest_layers.contains_key(blobid) { + removed.push(descriptor); + } + } + removed.sort_by(|a, b| a.digest().digest().cmp(b.digest().digest())); + for (blobid, &descriptor) in dest_layers.iter() { + if !src_layers.contains_key(blobid) { + added.push(descriptor); + } + } + added.sort_by(|a, b| a.digest().digest().cmp(b.digest().digest())); + + fn layersum<'a, I: Iterator>(layers: I) -> u64 { + layers.map(|layer| layer.size()).sum() + } + let total = dest_layers.len() as u64; + let total_size = layersum(dest.layers().iter()); + let n_removed = removed.len() as u64; + let n_added = added.len() as u64; + let removed_size = layersum(removed.iter().copied()); + let added_size = layersum(added.iter().copied()); + ManifestDiff { + from: src, + to: dest, + removed, + added, + total, + total_size, + n_removed, + removed_size, + n_added, + added_size, + } + } +} + +impl<'a> ManifestDiff<'a> { + /// Prints the total, removed and added content between two OCI images + pub fn print(&self) { + let print_total = self.total; + let print_total_size = glib::format_size(self.total_size); + let print_n_removed = self.n_removed; + let print_removed_size = glib::format_size(self.removed_size); + let print_n_added = self.n_added; + let print_added_size = glib::format_size(self.added_size); + println!("Total new layers: {print_total:<4} Size: {print_total_size}"); + println!("Removed layers: {print_n_removed:<4} Size: {print_removed_size}"); + println!("Added layers: {print_n_added:<4} Size: {print_added_size}"); + } +} + +/// Apply default configuration for container image pulls to an existing configuration. +/// For example, if `authfile` is not set, and `auth_anonymous` is `false`, and a global configuration file exists, it will be used. +/// +/// If there is no configured explicit subprocess for skopeo, and the process is running +/// as root, then a default isolation of running the process via `nobody` will be applied. +pub fn merge_default_container_proxy_opts( + config: &mut containers_image_proxy::ImageProxyConfig, +) -> Result<()> { + let user = rustix::process::getuid() + .is_root() + .then_some(isolation::DEFAULT_UNPRIVILEGED_USER); + merge_default_container_proxy_opts_with_isolation(config, user) +} + +/// Apply default configuration for container image pulls, with optional support +/// for isolation as an unprivileged user. +pub fn merge_default_container_proxy_opts_with_isolation( + config: &mut containers_image_proxy::ImageProxyConfig, + isolation_user: Option<&str>, +) -> Result<()> { + let auth_specified = + config.auth_anonymous || config.authfile.is_some() || config.auth_data.is_some(); + if !auth_specified { + let root = &Dir::open_ambient_dir("/", cap_std::ambient_authority())?; + config.auth_data = crate::globals::get_global_authfile(root)?.map(|a| a.1); + // If there's no auth data, then force on anonymous pulls to ensure + // that the container stack doesn't try to find it in the standard + // container paths. + if config.auth_data.is_none() { + config.auth_anonymous = true; + } + } + // By default, drop privileges, unless the higher level code + // has configured the skopeo command explicitly. + let isolation_user = config + .skopeo_cmd + .is_none() + .then_some(isolation_user.as_ref()) + .flatten(); + if let Some(user) = isolation_user { + // Read the default authfile if it exists and pass it via file descriptor + // which will ensure it's readable when we drop privileges. + if let Some(authfile) = config.authfile.take() { + config.auth_data = Some(std::fs::File::open(authfile)?); + } + let cmd = crate::isolation::unprivileged_subprocess("skopeo", user); + config.skopeo_cmd = Some(cmd); + } + Ok(()) +} + +/// Convenience helper to return the labels, if present. +pub(crate) fn labels_of( + config: &oci_spec::image::ImageConfiguration, +) -> Option<&HashMap> { + config.config().as_ref().and_then(|c| c.labels().as_ref()) +} + +/// Retrieve the version number from an image configuration. +pub fn version_for_config(config: &oci_spec::image::ImageConfiguration) -> Option<&str> { + if let Some(labels) = labels_of(config) { + for k in [oci_spec::image::ANNOTATION_VERSION, LABEL_VERSION] { + if let Some(v) = labels.get(k) { + return Some(v.as_str()); + } + } + } + None +} + +pub mod deploy; +mod encapsulate; +pub use encapsulate::*; +mod unencapsulate; +pub use unencapsulate::*; +mod skopeo; +pub mod store; +mod update_detachedmeta; +pub use update_detachedmeta::*; + +use crate::isolation; + +#[cfg(test)] +mod tests { + use std::process::Command; + + use containers_image_proxy::ImageProxyConfig; + + use super::*; + + #[test] + fn test_serializable_transport() { + for v in [ + Transport::Registry, + Transport::ContainerStorage, + Transport::OciArchive, + Transport::DockerArchive, + Transport::OciDir, + ] { + assert_eq!(Transport::try_from(v.serializable_name()).unwrap(), v); + } + } + + const INVALID_IRS: &[&str] = &["", "foo://", "docker:blah", "registry:", "foo:bar"]; + const VALID_IRS: &[&str] = &[ + "containers-storage:localhost/someimage", + "docker://quay.io/exampleos/blah:sometag", + ]; + + #[test] + fn test_imagereference() { + let ir: ImageReference = "registry:quay.io/exampleos/blah".try_into().unwrap(); + assert_eq!(ir.transport, Transport::Registry); + assert_eq!(ir.name, "quay.io/exampleos/blah"); + assert_eq!(ir.to_string(), "docker://quay.io/exampleos/blah"); + + for &v in VALID_IRS { + ImageReference::try_from(v).unwrap(); + } + + for &v in INVALID_IRS { + if ImageReference::try_from(v).is_ok() { + panic!("Should fail to parse: {}", v) + } + } + struct Case { + s: &'static str, + transport: Transport, + name: &'static str, + } + for case in [ + Case { + s: "oci:somedir", + transport: Transport::OciDir, + name: "somedir", + }, + Case { + s: "dir:/some/dir/blah", + transport: Transport::Dir, + name: "/some/dir/blah", + }, + Case { + s: "oci-archive:/path/to/foo.ociarchive", + transport: Transport::OciArchive, + name: "/path/to/foo.ociarchive", + }, + Case { + s: "docker-archive:/path/to/foo.dockerarchive", + transport: Transport::DockerArchive, + name: "/path/to/foo.dockerarchive", + }, + Case { + s: "containers-storage:localhost/someimage:blah", + transport: Transport::ContainerStorage, + name: "localhost/someimage:blah", + }, + ] { + let ir: ImageReference = case.s.try_into().unwrap(); + assert_eq!(ir.transport, case.transport); + assert_eq!(ir.name, case.name); + let reserialized = ir.to_string(); + assert_eq!(case.s, reserialized.as_str()); + } + } + + #[test] + fn test_ostreeimagereference() { + // Test both long form `ostree-remote-image:$myremote:registry` and the + // shorthand `ostree-remote-registry:$myremote`. + let ir_s = "ostree-remote-image:myremote:registry:quay.io/exampleos/blah"; + let ir_registry = "ostree-remote-registry:myremote:quay.io/exampleos/blah"; + for &ir_s in &[ir_s, ir_registry] { + let ir: OstreeImageReference = ir_s.try_into().unwrap(); + assert_eq!( + ir.sigverify, + SignatureSource::OstreeRemote("myremote".to_string()) + ); + assert_eq!(ir.imgref.transport, Transport::Registry); + assert_eq!(ir.imgref.name, "quay.io/exampleos/blah"); + assert_eq!( + ir.to_string(), + "ostree-remote-image:myremote:docker://quay.io/exampleos/blah" + ); + } + + // Also verify our FromStr impls + + let ir: OstreeImageReference = ir_s.try_into().unwrap(); + assert_eq!(ir, OstreeImageReference::from_str(ir_s).unwrap()); + // test our Eq implementation + assert_eq!(&ir, &OstreeImageReference::try_from(ir_registry).unwrap()); + + let ir_s = "ostree-image-signed:docker://quay.io/exampleos/blah"; + let ir: OstreeImageReference = ir_s.try_into().unwrap(); + assert_eq!(ir.sigverify, SignatureSource::ContainerPolicy); + assert_eq!(ir.imgref.transport, Transport::Registry); + assert_eq!(ir.imgref.name, "quay.io/exampleos/blah"); + assert_eq!(ir.to_string(), ir_s); + assert_eq!(format!("{:#}", &ir), ir_s); + + let ir_s = "ostree-unverified-image:docker://quay.io/exampleos/blah"; + let ir: OstreeImageReference = ir_s.try_into().unwrap(); + assert_eq!(ir.sigverify, SignatureSource::ContainerPolicyAllowInsecure); + assert_eq!(ir.imgref.transport, Transport::Registry); + assert_eq!(ir.imgref.name, "quay.io/exampleos/blah"); + assert_eq!( + ir.to_string(), + "ostree-unverified-registry:quay.io/exampleos/blah" + ); + let ir_shorthand = + OstreeImageReference::try_from("ostree-unverified-registry:quay.io/exampleos/blah") + .unwrap(); + assert_eq!(&ir_shorthand, &ir); + assert_eq!(format!("{:#}", &ir), "docker://quay.io/exampleos/blah"); + } + + #[test] + fn test_merge_authopts() { + // Verify idempotence of authentication processing + let mut c = ImageProxyConfig::default(); + let authf = std::fs::File::open("/dev/null").unwrap(); + c.auth_data = Some(authf); + super::merge_default_container_proxy_opts_with_isolation(&mut c, None).unwrap(); + assert!(!c.auth_anonymous); + assert!(c.authfile.is_none()); + assert!(c.auth_data.is_some()); + assert!(c.skopeo_cmd.is_none()); + super::merge_default_container_proxy_opts_with_isolation(&mut c, None).unwrap(); + assert!(!c.auth_anonymous); + assert!(c.authfile.is_none()); + assert!(c.auth_data.is_some()); + assert!(c.skopeo_cmd.is_none()); + + // Verify interaction with explicit isolation + let mut c = ImageProxyConfig { + skopeo_cmd: Some(Command::new("skopeo")), + ..Default::default() + }; + super::merge_default_container_proxy_opts_with_isolation(&mut c, Some("foo")).unwrap(); + assert_eq!(c.skopeo_cmd.unwrap().get_program(), "skopeo"); + } +} diff --git a/ostree-ext/src/container/skopeo.rs b/ostree-ext/src/container/skopeo.rs new file mode 100644 index 00000000..7ac46250 --- /dev/null +++ b/ostree-ext/src/container/skopeo.rs @@ -0,0 +1,156 @@ +//! Fork skopeo as a subprocess + +use super::ImageReference; +use anyhow::{Context, Result}; +use cap_std_ext::cmdext::CapStdExtCommandExt; +use containers_image_proxy::oci_spec::image as oci_image; +use fn_error_context::context; +use io_lifetimes::OwnedFd; +use serde::Deserialize; +use std::io::Read; +use std::path::Path; +use std::process::Stdio; +use std::str::FromStr; +use tokio::process::Command; + +// See `man containers-policy.json` and +// https://github.com/containers/image/blob/main/signature/policy_types.go +// Ideally we add something like `skopeo pull --disallow-insecure-accept-anything` +// but for now we parse the policy. +const POLICY_PATH: &str = "/etc/containers/policy.json"; +const INSECURE_ACCEPT_ANYTHING: &str = "insecureAcceptAnything"; + +#[derive(Deserialize)] +struct PolicyEntry { + #[serde(rename = "type")] + ty: String, +} +#[derive(Deserialize)] +struct ContainerPolicy { + default: Option>, +} + +impl ContainerPolicy { + fn is_default_insecure(&self) -> bool { + if let Some(default) = self.default.as_deref() { + match default.split_first() { + Some((v, &[])) => v.ty == INSECURE_ACCEPT_ANYTHING, + _ => false, + } + } else { + false + } + } +} + +pub(crate) fn container_policy_is_default_insecure() -> Result { + let r = std::io::BufReader::new(std::fs::File::open(POLICY_PATH)?); + let policy: ContainerPolicy = serde_json::from_reader(r)?; + Ok(policy.is_default_insecure()) +} + +/// Create a Command builder for skopeo. +pub(crate) fn new_cmd() -> std::process::Command { + let mut cmd = std::process::Command::new("skopeo"); + cmd.stdin(Stdio::null()); + cmd +} + +/// Spawn the child process +pub(crate) fn spawn(mut cmd: Command) -> Result { + let cmd = cmd.stdin(Stdio::null()).stderr(Stdio::piped()); + cmd.spawn().context("Failed to exec skopeo") +} + +/// Use skopeo to copy a container image. +#[context("Skopeo copy")] +pub(crate) async fn copy( + src: &ImageReference, + dest: &ImageReference, + authfile: Option<&Path>, + add_fd: Option<(std::sync::Arc, i32)>, + progress: bool, +) -> Result { + let digestfile = tempfile::NamedTempFile::new()?; + let mut cmd = new_cmd(); + cmd.arg("copy"); + if !progress { + cmd.stdout(std::process::Stdio::null()); + } + cmd.arg("--digestfile"); + cmd.arg(digestfile.path()); + if let Some((add_fd, n)) = add_fd { + cmd.take_fd_n(add_fd, n); + } + if let Some(authfile) = authfile { + cmd.arg("--authfile"); + cmd.arg(authfile); + } + cmd.args(&[src.to_string(), dest.to_string()]); + let mut cmd = tokio::process::Command::from(cmd); + cmd.kill_on_drop(true); + let proc = super::skopeo::spawn(cmd)?; + let output = proc.wait_with_output().await?; + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + return Err(anyhow::anyhow!("skopeo failed: {}\n", stderr)); + } + let mut digestfile = digestfile.into_file(); + let mut r = String::new(); + digestfile.read_to_string(&mut r)?; + Ok(oci_image::Digest::from_str(r.trim())?) +} + +#[cfg(test)] +mod tests { + use super::*; + + // Default value as of the Fedora 34 containers-common-1-21.fc34.noarch package. + const DEFAULT_POLICY: &str = indoc::indoc! {r#" + { + "default": [ + { + "type": "insecureAcceptAnything" + } + ], + "transports": + { + "docker-daemon": + { + "": [{"type":"insecureAcceptAnything"}] + } + } + } + "#}; + + // Stripped down copy from the manual. + const REASONABLY_LOCKED_DOWN: &str = indoc::indoc! { r#" + { + "default": [{"type": "reject"}], + "transports": { + "dir": { + "": [{"type": "insecureAcceptAnything"}] + }, + "atomic": { + "hostname:5000/myns/official": [ + { + "type": "signedBy", + "keyType": "GPGKeys", + "keyPath": "/path/to/official-pubkey.gpg" + } + ] + } + } + } + "#}; + + #[test] + fn policy_is_insecure() { + let p: ContainerPolicy = serde_json::from_str(DEFAULT_POLICY).unwrap(); + assert!(p.is_default_insecure()); + for &v in &["{}", REASONABLY_LOCKED_DOWN] { + let p: ContainerPolicy = serde_json::from_str(v).unwrap(); + assert!(!p.is_default_insecure()); + } + } +} diff --git a/ostree-ext/src/container/store.rs b/ostree-ext/src/container/store.rs new file mode 100644 index 00000000..c715f8ca --- /dev/null +++ b/ostree-ext/src/container/store.rs @@ -0,0 +1,1862 @@ +//! APIs for storing (layered) container images as OSTree commits +//! +//! # Extension of encapsulation support +//! +//! This code supports ingesting arbitrary layered container images from an ostree-exported +//! base. See [`encapsulate`][`super::encapsulate()`] for more information on encaspulation of images. + +use super::*; +use crate::chunking::{self, Chunk}; +use crate::logging::system_repo_journal_print; +use crate::refescape; +use crate::sysroot::SysrootLock; +use crate::utils::ResultExt; +use anyhow::{anyhow, Context}; +use camino::{Utf8Path, Utf8PathBuf}; +use cap_std_ext::cap_std; +use cap_std_ext::cap_std::fs::{Dir, MetadataExt}; +use cap_std_ext::cmdext::CapStdExtCommandExt; +use containers_image_proxy::{ImageProxy, OpenedImage}; +use flate2::Compression; +use fn_error_context::context; +use futures_util::TryFutureExt; +use oci_spec::image::{ + self as oci_image, Arch, Descriptor, Digest, History, ImageConfiguration, ImageManifest, +}; +use ostree::prelude::{Cast, FileEnumeratorExt, FileExt, ToVariant}; +use ostree::{gio, glib}; +use std::collections::{BTreeSet, HashMap}; +use std::iter::FromIterator; +use tokio::sync::mpsc::{Receiver, Sender}; + +/// Configuration for the proxy. +/// +/// We re-export this rather than inventing our own wrapper +/// in the interest of avoiding duplication. +pub use containers_image_proxy::ImageProxyConfig; + +/// The ostree ref prefix for blobs. +const LAYER_PREFIX: &str = "ostree/container/blob"; +/// The ostree ref prefix for image references. +const IMAGE_PREFIX: &str = "ostree/container/image"; +/// The ostree ref prefix for "base" image references that are used by derived images. +/// If you maintain tooling which is locally building derived commits, write a ref +/// with this prefix that is owned by your code. It's a best practice to prefix the +/// ref with the project name, so the final ref may be of the form e.g. `ostree/container/baseimage/bootc/foo`. +pub const BASE_IMAGE_PREFIX: &str = "ostree/container/baseimage"; + +/// The key injected into the merge commit for the manifest digest. +pub(crate) const META_MANIFEST_DIGEST: &str = "ostree.manifest-digest"; +/// The key injected into the merge commit with the manifest serialized as JSON. +const META_MANIFEST: &str = "ostree.manifest"; +/// The key injected into the merge commit with the image configuration serialized as JSON. +const META_CONFIG: &str = "ostree.container.image-config"; +/// Value of type `a{sa{su}}` containing number of filtered out files +pub const META_FILTERED: &str = "ostree.tar-filtered"; +/// The type used to store content filtering information with `META_FILTERED`. +pub type MetaFilteredData = HashMap>; + +/// The ref prefixes which point to ostree deployments. (TODO: Add an official API for this) +const OSTREE_BASE_DEPLOYMENT_REFS: &[&str] = &["ostree/0", "ostree/1"]; +/// A layering violation we'll carry for a bit to band-aid over https://github.com/coreos/rpm-ostree/issues/4185 +const RPMOSTREE_BASE_REFS: &[&str] = &["rpmostree/base"]; + +/// Convert e.g. sha256:12345... into `/ostree/container/blob/sha256_2B12345...`. +fn ref_for_blob_digest(d: &str) -> Result { + refescape::prefix_escape_for_ref(LAYER_PREFIX, d) +} + +/// Convert e.g. sha256:12345... into `/ostree/container/blob/sha256_2B12345...`. +fn ref_for_layer(l: &oci_image::Descriptor) -> Result { + ref_for_blob_digest(&l.digest().to_string()) +} + +/// Convert e.g. sha256:12345... into `/ostree/container/blob/sha256_2B12345...`. +fn ref_for_image(l: &ImageReference) -> Result { + refescape::prefix_escape_for_ref(IMAGE_PREFIX, &l.to_string()) +} + +/// Sent across a channel to track start and end of a container fetch. +#[derive(Debug)] +pub enum ImportProgress { + /// Started fetching this layer. + OstreeChunkStarted(Descriptor), + /// Successfully completed the fetch of this layer. + OstreeChunkCompleted(Descriptor), + /// Started fetching this layer. + DerivedLayerStarted(Descriptor), + /// Successfully completed the fetch of this layer. + DerivedLayerCompleted(Descriptor), +} + +impl ImportProgress { + /// Returns `true` if this message signifies the start of a new layer being fetched. + pub fn is_starting(&self) -> bool { + match self { + ImportProgress::OstreeChunkStarted(_) => true, + ImportProgress::OstreeChunkCompleted(_) => false, + ImportProgress::DerivedLayerStarted(_) => true, + ImportProgress::DerivedLayerCompleted(_) => false, + } + } +} + +/// Sent across a channel to track the byte-level progress of a layer fetch. +#[derive(Debug)] +pub struct LayerProgress { + /// Index of the layer in the manifest + pub layer_index: usize, + /// Number of bytes downloaded + pub fetched: u64, + /// Total number of bytes outstanding + pub total: u64, +} + +/// State of an already pulled layered image. +#[derive(Debug, PartialEq, Eq)] +pub struct LayeredImageState { + /// The base ostree commit + pub base_commit: String, + /// The merge commit unions all layers + pub merge_commit: String, + /// The digest of the original manifest + pub manifest_digest: Digest, + /// The image manfiest + pub manifest: ImageManifest, + /// The image configuration + pub configuration: ImageConfiguration, + /// Metadata for (cached, previously fetched) updates to the image, if any. + pub cached_update: Option, +} + +impl LayeredImageState { + /// Return the merged ostree commit for this image. + /// + /// This is not the same as the underlying base ostree commit. + pub fn get_commit(&self) -> &str { + self.merge_commit.as_str() + } + + /// Retrieve the container image version. + pub fn version(&self) -> Option<&str> { + super::version_for_config(&self.configuration) + } +} + +/// Locally cached metadata for an update to an existing image. +#[derive(Debug, PartialEq, Eq)] +pub struct CachedImageUpdate { + /// The image manifest + pub manifest: ImageManifest, + /// The image configuration + pub config: ImageConfiguration, + /// The digest of the manifest + pub manifest_digest: Digest, +} + +impl CachedImageUpdate { + /// Retrieve the container image version. + pub fn version(&self) -> Option<&str> { + super::version_for_config(&self.config) + } +} + +/// Context for importing a container image. +#[derive(Debug)] +pub struct ImageImporter { + repo: ostree::Repo, + pub(crate) proxy: ImageProxy, + imgref: OstreeImageReference, + target_imgref: Option, + no_imgref: bool, // If true, do not write final image ref + disable_gc: bool, // If true, don't prune unused image layers + /// If true, require the image has the bootable flag + require_bootable: bool, + /// If true, we have ostree v2024.3 or newer. + ostree_v2024_3: bool, + pub(crate) proxy_img: OpenedImage, + + layer_progress: Option>, + layer_byte_progress: Option>>, +} + +/// Result of invoking [`ImageImporter::prepare`]. +#[derive(Debug)] +pub enum PrepareResult { + /// The image reference is already present; the contained string is the OSTree commit. + AlreadyPresent(Box), + /// The image needs to be downloaded + Ready(Box), +} + +/// A container image layer with associated downloaded-or-not state. +#[derive(Debug)] +pub struct ManifestLayerState { + /// The underlying layer descriptor. + pub(crate) layer: oci_image::Descriptor, + // TODO semver: Make this readonly via an accessor + /// The ostree ref name for this layer. + pub ostree_ref: String, + // TODO semver: Make this readonly via an accessor + /// The ostree commit that caches this layer, if present. + pub commit: Option, +} + +impl ManifestLayerState { + /// Return the layer descriptor. + pub fn layer(&self) -> &oci_image::Descriptor { + &self.layer + } +} + +/// Information about which layers need to be downloaded. +#[derive(Debug)] +pub struct PreparedImport { + /// The manifest digest that was found + pub manifest_digest: Digest, + /// The deserialized manifest. + pub manifest: oci_image::ImageManifest, + /// The deserialized configuration. + pub config: oci_image::ImageConfiguration, + /// The previous manifest + pub previous_state: Option>, + /// The previously stored manifest digest. + pub previous_manifest_digest: Option, + /// The previously stored image ID. + pub previous_imageid: Option, + /// The layers containing split objects + pub ostree_layers: Vec, + /// The layer for the ostree commit. + pub ostree_commit_layer: ManifestLayerState, + /// Any further non-ostree (derived) layers. + pub layers: Vec, +} + +impl PreparedImport { + /// Iterate over all layers; the commit layer, the ostree split object layers, and any non-ostree layers. + pub fn all_layers(&self) -> impl Iterator { + std::iter::once(&self.ostree_commit_layer) + .chain(self.ostree_layers.iter()) + .chain(self.layers.iter()) + } + + /// Retrieve the container image version. + pub fn version(&self) -> Option<&str> { + super::version_for_config(&self.config) + } + + /// If this image is using any deprecated features, return a message saying so. + pub fn deprecated_warning(&self) -> Option<&'static str> { + None + } + + /// Iterate over all layers paired with their history entry. + /// An error will be returned if the history does not cover all entries. + pub fn layers_with_history( + &self, + ) -> impl Iterator> { + // FIXME use .filter(|h| h.empty_layer.unwrap_or_default()) after https://github.com/containers/oci-spec-rs/pull/100 lands. + let truncated = std::iter::once_with(|| Err(anyhow::anyhow!("Truncated history"))); + let history = self.config.history().iter().map(Ok).chain(truncated); + self.all_layers() + .zip(history) + .map(|(s, h)| h.map(|h| (s, h))) + } + + /// Iterate over all layers that are not present, along with their history description. + pub fn layers_to_fetch(&self) -> impl Iterator> { + self.layers_with_history().filter_map(|r| { + r.map(|(l, h)| { + l.commit.is_none().then(|| { + let comment = h.created_by().as_deref().unwrap_or(""); + (l, comment) + }) + }) + .transpose() + }) + } + + /// Common helper to format a string for the status + pub(crate) fn format_layer_status(&self) -> Option { + let (stored, to_fetch, to_fetch_size) = + self.all_layers() + .fold((0u32, 0u32, 0u64), |(stored, to_fetch, sz), v| { + if v.commit.is_some() { + (stored + 1, to_fetch, sz) + } else { + (stored, to_fetch + 1, sz + v.layer().size()) + } + }); + (to_fetch > 0).then(|| { + let size = crate::glib::format_size(to_fetch_size); + format!("layers already present: {stored}; layers needed: {to_fetch} ({size})") + }) + } +} + +// Given a manifest, compute its ostree ref name and cached ostree commit +pub(crate) fn query_layer( + repo: &ostree::Repo, + layer: oci_image::Descriptor, +) -> Result { + let ostree_ref = ref_for_layer(&layer)?; + let commit = repo.resolve_rev(&ostree_ref, true)?.map(|s| s.to_string()); + Ok(ManifestLayerState { + layer, + ostree_ref, + commit, + }) +} + +#[context("Reading manifest data from commit")] +fn manifest_data_from_commitmeta( + commit_meta: &glib::VariantDict, +) -> Result<(oci_image::ImageManifest, Digest)> { + let digest = commit_meta + .lookup::(META_MANIFEST_DIGEST)? + .ok_or_else(|| anyhow!("Missing {} metadata on merge commit", META_MANIFEST_DIGEST))?; + let digest = Digest::from_str(&digest)?; + let manifest_bytes: String = commit_meta + .lookup::(META_MANIFEST)? + .ok_or_else(|| anyhow!("Failed to find {} metadata key", META_MANIFEST))?; + let r = serde_json::from_str(&manifest_bytes)?; + Ok((r, digest)) +} + +fn image_config_from_commitmeta(commit_meta: &glib::VariantDict) -> Result { + let config = if let Some(config) = commit_meta + .lookup::(META_CONFIG)? + .filter(|v| v != "null") // Format v0 apparently old versions injected `null` here sadly... + .map(|v| serde_json::from_str(&v).map_err(anyhow::Error::msg)) + .transpose()? + { + config + } else { + tracing::debug!("No image configuration found"); + Default::default() + }; + Ok(config) +} + +/// Return the original digest of the manifest stored in the commit metadata. +/// This will be a string of the form e.g. `sha256:`. +/// +/// This can be used to uniquely identify the image. For example, it can be used +/// in a "digested pull spec" like `quay.io/someuser/exampleos@sha256:...`. +pub fn manifest_digest_from_commit(commit: &glib::Variant) -> Result { + let commit_meta = &commit.child_value(0); + let commit_meta = &glib::VariantDict::new(Some(commit_meta)); + Ok(manifest_data_from_commitmeta(commit_meta)?.1) +} + +/// Given a target diffid, return its corresponding layer. In our current model, +/// we require a 1-to-1 mapping between the two up until the ostree level. +/// For a bit more information on this, see https://github.com/opencontainers/image-spec/blob/main/config.md +fn layer_from_diffid<'a>( + manifest: &'a ImageManifest, + config: &ImageConfiguration, + diffid: &str, +) -> Result<&'a Descriptor> { + let idx = config + .rootfs() + .diff_ids() + .iter() + .position(|x| x.as_str() == diffid) + .ok_or_else(|| anyhow!("Missing {} {}", DIFFID_LABEL, diffid))?; + manifest.layers().get(idx).ok_or_else(|| { + anyhow!( + "diffid position {} exceeds layer count {}", + idx, + manifest.layers().len() + ) + }) +} + +#[context("Parsing manifest layout")] +pub(crate) fn parse_manifest_layout<'a>( + manifest: &'a ImageManifest, + config: &ImageConfiguration, +) -> Result<(&'a Descriptor, Vec<&'a Descriptor>, Vec<&'a Descriptor>)> { + let config_labels = super::labels_of(config); + + let first_layer = manifest + .layers() + .first() + .ok_or_else(|| anyhow!("No layers in manifest"))?; + let target_diffid = config_labels + .and_then(|labels| labels.get(DIFFID_LABEL)) + .ok_or_else(|| { + anyhow!( + "No {} label found, not an ostree encapsulated container", + DIFFID_LABEL + ) + })?; + + let target_layer = layer_from_diffid(manifest, config, target_diffid.as_str())?; + let mut chunk_layers = Vec::new(); + let mut derived_layers = Vec::new(); + let mut after_target = false; + // Gather the ostree layer + let ostree_layer = first_layer; + for layer in manifest.layers() { + if layer == target_layer { + if after_target { + anyhow::bail!("Multiple entries for {}", layer.digest()); + } + after_target = true; + if layer != ostree_layer { + chunk_layers.push(layer); + } + } else if !after_target { + if layer != ostree_layer { + chunk_layers.push(layer); + } + } else { + derived_layers.push(layer); + } + } + + Ok((ostree_layer, chunk_layers, derived_layers)) +} + +/// Find the timestamp of the manifest (or config), ignoring errors. +fn timestamp_of_manifest_or_config( + manifest: &ImageManifest, + config: &ImageConfiguration, +) -> Option { + // The manifest timestamp seems to not be widely used, but let's + // try it in preference to the config one. + let timestamp = manifest + .annotations() + .as_ref() + .and_then(|a| a.get(oci_image::ANNOTATION_CREATED)) + .or_else(|| config.created().as_ref()); + // Try to parse the timestamp + timestamp + .map(|t| { + chrono::DateTime::parse_from_rfc3339(t) + .context("Failed to parse manifest timestamp") + .map(|t| t.timestamp() as u64) + }) + .transpose() + .log_err_default() +} + +impl ImageImporter { + /// The metadata key used in ostree commit metadata to serialize + const CACHED_KEY_MANIFEST_DIGEST: &'static str = "ostree-ext.cached.manifest-digest"; + const CACHED_KEY_MANIFEST: &'static str = "ostree-ext.cached.manifest"; + const CACHED_KEY_CONFIG: &'static str = "ostree-ext.cached.config"; + + /// Create a new importer. + #[context("Creating importer")] + pub async fn new( + repo: &ostree::Repo, + imgref: &OstreeImageReference, + mut config: ImageProxyConfig, + ) -> Result { + if imgref.imgref.transport == Transport::ContainerStorage { + // Fetching from containers-storage, may require privileges to read files + merge_default_container_proxy_opts_with_isolation(&mut config, None)?; + } else { + // Apply our defaults to the proxy config + merge_default_container_proxy_opts(&mut config)?; + } + let proxy = ImageProxy::new_with_config(config).await?; + + system_repo_journal_print( + repo, + libsystemd::logging::Priority::Info, + &format!("Fetching {}", imgref), + ); + + let proxy_img = proxy.open_image(&imgref.imgref.to_string()).await?; + let repo = repo.clone(); + Ok(ImageImporter { + repo, + proxy, + proxy_img, + target_imgref: None, + no_imgref: false, + ostree_v2024_3: ostree::check_version(2024, 3), + disable_gc: false, + require_bootable: false, + imgref: imgref.clone(), + layer_progress: None, + layer_byte_progress: None, + }) + } + + /// Write cached data as if the image came from this source. + pub fn set_target(&mut self, target: &OstreeImageReference) { + self.target_imgref = Some(target.clone()) + } + + /// Do not write the final image ref, but do write refs for shared layers. + /// This is useful in scenarios where you want to "pre-pull" an image, + /// but in such a way that it does not need to be manually removed later. + pub fn set_no_imgref(&mut self) { + self.no_imgref = true; + } + + /// Require that the image has the bootable metadata field + pub fn require_bootable(&mut self) { + self.require_bootable = true; + } + + /// Override the ostree version being targeted + pub fn set_ostree_version(&mut self, year: u32, v: u32) { + self.ostree_v2024_3 = (year > 2024) || (year == 2024 && v >= 3) + } + + /// Do not prune image layers. + pub fn disable_gc(&mut self) { + self.disable_gc = true; + } + + /// Determine if there is a new manifest, and if so return its digest. + /// This will also serialize the new manifest and configuration into + /// metadata associated with the image, so that invocations of `[query_cached]` + /// can re-fetch it without accessing the network. + #[context("Preparing import")] + pub async fn prepare(&mut self) -> Result { + self.prepare_internal(false).await + } + + /// Create a channel receiver that will get notifications for layer fetches. + pub fn request_progress(&mut self) -> Receiver { + assert!(self.layer_progress.is_none()); + let (s, r) = tokio::sync::mpsc::channel(2); + self.layer_progress = Some(s); + r + } + + /// Create a channel receiver that will get notifications for byte-level progress of layer fetches. + pub fn request_layer_progress( + &mut self, + ) -> tokio::sync::watch::Receiver> { + assert!(self.layer_byte_progress.is_none()); + let (s, r) = tokio::sync::watch::channel(None); + self.layer_byte_progress = Some(s); + r + } + + /// Serialize the metadata about a pending fetch as detached metadata on the commit object, + /// so it can be retrieved later offline + #[context("Writing cached pending manifest")] + pub(crate) async fn cache_pending( + &self, + commit: &str, + manifest_digest: &Digest, + manifest: &ImageManifest, + config: &ImageConfiguration, + ) -> Result<()> { + let commitmeta = glib::VariantDict::new(None); + commitmeta.insert( + Self::CACHED_KEY_MANIFEST_DIGEST, + manifest_digest.to_string(), + ); + let cached_manifest = serde_json::to_string(manifest).context("Serializing manifest")?; + commitmeta.insert(Self::CACHED_KEY_MANIFEST, cached_manifest); + let cached_config = serde_json::to_string(config).context("Serializing config")?; + commitmeta.insert(Self::CACHED_KEY_CONFIG, cached_config); + let commitmeta = commitmeta.to_variant(); + // Clone these to move into blocking method + let commit = commit.to_string(); + let repo = self.repo.clone(); + crate::tokio_util::spawn_blocking_cancellable_flatten(move |cancellable| { + repo.write_commit_detached_metadata(&commit, Some(&commitmeta), Some(cancellable)) + .map_err(anyhow::Error::msg) + }) + .await + } + + /// Given existing metadata (manifest, config, previous image statE) generate a PreparedImport structure + /// which e.g. includes a diff of the layers. + fn create_prepared_import( + &mut self, + manifest_digest: Digest, + manifest: ImageManifest, + config: ImageConfiguration, + previous_state: Option>, + previous_imageid: Option, + ) -> Result> { + let config_labels = super::labels_of(&config); + if self.require_bootable { + let bootable_key = *ostree::METADATA_KEY_BOOTABLE; + let bootable = config_labels.map_or(false, |l| { + l.contains_key(bootable_key) || l.contains_key(BOOTC_LABEL) + }); + if !bootable { + anyhow::bail!("Target image does not have {bootable_key} label"); + } + let container_arch = config.architecture(); + let target_arch = &Arch::default(); + if container_arch != target_arch { + anyhow::bail!("Image has architecture {container_arch}; expected {target_arch}"); + } + } + + let (commit_layer, component_layers, remaining_layers) = + parse_manifest_layout(&manifest, &config)?; + + let query = |l: &Descriptor| query_layer(&self.repo, l.clone()); + let commit_layer = query(commit_layer)?; + let component_layers = component_layers + .into_iter() + .map(query) + .collect::>>()?; + let remaining_layers = remaining_layers + .into_iter() + .map(query) + .collect::>>()?; + + let previous_manifest_digest = previous_state.as_ref().map(|s| s.manifest_digest.clone()); + let imp = PreparedImport { + manifest_digest, + manifest, + config, + previous_state, + previous_manifest_digest, + previous_imageid, + ostree_layers: component_layers, + ostree_commit_layer: commit_layer, + layers: remaining_layers, + }; + Ok(Box::new(imp)) + } + + /// Determine if there is a new manifest, and if so return its digest. + #[context("Fetching manifest")] + pub(crate) async fn prepare_internal(&mut self, verify_layers: bool) -> Result { + match &self.imgref.sigverify { + SignatureSource::ContainerPolicy if skopeo::container_policy_is_default_insecure()? => { + return Err(anyhow!("containers-policy.json specifies a default of `insecureAcceptAnything`; refusing usage")); + } + SignatureSource::OstreeRemote(_) if verify_layers => { + return Err(anyhow!( + "Cannot currently verify layered containers via ostree remote" + )); + } + _ => {} + } + + let (manifest_digest, manifest) = self.proxy.fetch_manifest(&self.proxy_img).await?; + let manifest_digest = Digest::from_str(&manifest_digest)?; + let new_imageid = manifest.config().digest(); + + // Query for previous stored state + + let (previous_state, previous_imageid) = + if let Some(previous_state) = try_query_image(&self.repo, &self.imgref.imgref)? { + // If the manifest digests match, we're done. + if previous_state.manifest_digest == manifest_digest { + return Ok(PrepareResult::AlreadyPresent(previous_state)); + } + // Failing that, if they have the same imageID, we're also done. + let previous_imageid = previous_state.manifest.config().digest(); + if previous_imageid == new_imageid { + return Ok(PrepareResult::AlreadyPresent(previous_state)); + } + let previous_imageid = previous_imageid.to_string(); + (Some(previous_state), Some(previous_imageid)) + } else { + (None, None) + }; + + let config = self.proxy.fetch_config(&self.proxy_img).await?; + + // If there is a currently fetched image, cache the new pending manifest+config + // as detached commit metadata, so that future fetches can query it offline. + if let Some(previous_state) = previous_state.as_ref() { + self.cache_pending( + previous_state.merge_commit.as_str(), + &manifest_digest, + &manifest, + &config, + ) + .await?; + } + + let imp = self.create_prepared_import( + manifest_digest, + manifest, + config, + previous_state, + previous_imageid, + )?; + Ok(PrepareResult::Ready(imp)) + } + + /// Extract the base ostree commit. + #[context("Unencapsulating base")] + pub(crate) async fn unencapsulate_base( + &mut self, + import: &mut store::PreparedImport, + write_refs: bool, + ) -> Result<()> { + tracing::debug!("Fetching base"); + if matches!(self.imgref.sigverify, SignatureSource::ContainerPolicy) + && skopeo::container_policy_is_default_insecure()? + { + return Err(anyhow!("containers-policy.json specifies a default of `insecureAcceptAnything`; refusing usage")); + } + let remote = match &self.imgref.sigverify { + SignatureSource::OstreeRemote(remote) => Some(remote.clone()), + SignatureSource::ContainerPolicy | SignatureSource::ContainerPolicyAllowInsecure => { + None + } + }; + let des_layers = self.proxy.get_layer_info(&self.proxy_img).await?; + for layer in import.ostree_layers.iter_mut() { + if layer.commit.is_some() { + continue; + } + if let Some(p) = self.layer_progress.as_ref() { + p.send(ImportProgress::OstreeChunkStarted(layer.layer.clone())) + .await?; + } + let (blob, driver, media_type) = fetch_layer( + &self.proxy, + &self.proxy_img, + &import.manifest, + &layer.layer, + self.layer_byte_progress.as_ref(), + des_layers.as_ref(), + self.imgref.imgref.transport, + ) + .await?; + let repo = self.repo.clone(); + let target_ref = layer.ostree_ref.clone(); + let import_task = + crate::tokio_util::spawn_blocking_cancellable_flatten(move |cancellable| { + let txn = repo.auto_transaction(Some(cancellable))?; + let mut importer = crate::tar::Importer::new_for_object_set(&repo); + let blob = tokio_util::io::SyncIoBridge::new(blob); + let blob = super::unencapsulate::decompressor(&media_type, blob)?; + let mut archive = tar::Archive::new(blob); + importer.import_objects(&mut archive, Some(cancellable))?; + let commit = if write_refs { + let commit = importer.finish_import_object_set()?; + repo.transaction_set_ref(None, &target_ref, Some(commit.as_str())); + tracing::debug!("Wrote {} => {}", target_ref, commit); + Some(commit) + } else { + None + }; + txn.commit(Some(cancellable))?; + Ok::<_, anyhow::Error>(commit) + }) + .map_err(|e| e.context(format!("Layer {}", layer.layer.digest()))); + let commit = super::unencapsulate::join_fetch(import_task, driver).await?; + layer.commit = commit; + if let Some(p) = self.layer_progress.as_ref() { + p.send(ImportProgress::OstreeChunkCompleted(layer.layer.clone())) + .await?; + } + } + if import.ostree_commit_layer.commit.is_none() { + if let Some(p) = self.layer_progress.as_ref() { + p.send(ImportProgress::OstreeChunkStarted( + import.ostree_commit_layer.layer.clone(), + )) + .await?; + } + let (blob, driver, media_type) = fetch_layer( + &self.proxy, + &self.proxy_img, + &import.manifest, + &import.ostree_commit_layer.layer, + self.layer_byte_progress.as_ref(), + des_layers.as_ref(), + self.imgref.imgref.transport, + ) + .await?; + let repo = self.repo.clone(); + let target_ref = import.ostree_commit_layer.ostree_ref.clone(); + let import_task = + crate::tokio_util::spawn_blocking_cancellable_flatten(move |cancellable| { + let txn = repo.auto_transaction(Some(cancellable))?; + let mut importer = crate::tar::Importer::new_for_commit(&repo, remote); + let blob = tokio_util::io::SyncIoBridge::new(blob); + let blob = super::unencapsulate::decompressor(&media_type, blob)?; + let mut archive = tar::Archive::new(blob); + importer.import_commit(&mut archive, Some(cancellable))?; + let commit = importer.finish_import_commit(); + if write_refs { + repo.transaction_set_ref(None, &target_ref, Some(commit.as_str())); + tracing::debug!("Wrote {} => {}", target_ref, commit); + } + repo.mark_commit_partial(&commit, false)?; + txn.commit(Some(cancellable))?; + Ok::<_, anyhow::Error>(commit) + }); + let commit = super::unencapsulate::join_fetch(import_task, driver).await?; + import.ostree_commit_layer.commit = Some(commit); + if let Some(p) = self.layer_progress.as_ref() { + p.send(ImportProgress::OstreeChunkCompleted( + import.ostree_commit_layer.layer.clone(), + )) + .await?; + } + }; + Ok(()) + } + + /// Retrieve an inner ostree commit. + /// + /// This does not write cached references for each blob, and errors out if + /// the image has any non-ostree layers. + pub async fn unencapsulate(mut self) -> Result { + let mut prep = match self.prepare_internal(false).await? { + PrepareResult::AlreadyPresent(_) => { + panic!("Should not have image present for unencapsulation") + } + PrepareResult::Ready(r) => r, + }; + if !prep.layers.is_empty() { + anyhow::bail!("Image has {} non-ostree layers", prep.layers.len()); + } + let deprecated_warning = prep.deprecated_warning().map(ToOwned::to_owned); + self.unencapsulate_base(&mut prep, false).await?; + // TODO change the imageproxy API to ensure this happens automatically when + // the image reference is dropped + self.proxy.close_image(&self.proxy_img).await?; + let ostree_commit = prep.ostree_commit_layer.commit.unwrap(); + let image_digest = prep.manifest_digest; + Ok(Import { + ostree_commit, + image_digest, + deprecated_warning, + }) + } + + /// Import a layered container image. + /// + /// If enabled, this will also prune unused container image layers. + #[context("Importing")] + pub async fn import( + mut self, + mut import: Box, + ) -> Result> { + if let Some(status) = import.format_layer_status() { + system_repo_journal_print(&self.repo, libsystemd::logging::Priority::Info, &status); + } + // First download all layers for the base image (if necessary) - we need the SELinux policy + // there to label all following layers. + self.unencapsulate_base(&mut import, true).await?; + let des_layers = self.proxy.get_layer_info(&self.proxy_img).await?; + let proxy = self.proxy; + let proxy_img = self.proxy_img; + let target_imgref = self.target_imgref.as_ref().unwrap_or(&self.imgref); + let base_commit = import.ostree_commit_layer.commit.clone().unwrap(); + + let root_is_transient = { + let rootf = self + .repo + .read_commit(&base_commit, gio::Cancellable::NONE)? + .0; + let rootf = rootf.downcast_ref::().unwrap(); + crate::ostree_prepareroot::overlayfs_root_enabled(rootf)? + }; + tracing::debug!("Base rootfs is transient: {root_is_transient}"); + + let ostree_ref = ref_for_image(&target_imgref.imgref)?; + + let mut layer_commits = Vec::new(); + let mut layer_filtered_content: MetaFilteredData = HashMap::new(); + let have_derived_layers = !import.layers.is_empty(); + for layer in import.layers { + if let Some(c) = layer.commit { + tracing::debug!("Reusing fetched commit {}", c); + layer_commits.push(c.to_string()); + } else { + if let Some(p) = self.layer_progress.as_ref() { + p.send(ImportProgress::DerivedLayerStarted(layer.layer.clone())) + .await?; + } + let (blob, driver, media_type) = super::unencapsulate::fetch_layer( + &proxy, + &proxy_img, + &import.manifest, + &layer.layer, + self.layer_byte_progress.as_ref(), + des_layers.as_ref(), + self.imgref.imgref.transport, + ) + .await?; + // An important aspect of this is that we SELinux label the derived layers using + // the base policy. + let opts = crate::tar::WriteTarOptions { + base: Some(base_commit.clone()), + selinux: true, + allow_nonusr: root_is_transient, + retain_var: self.ostree_v2024_3, + }; + let r = crate::tar::write_tar( + &self.repo, + blob, + media_type, + layer.ostree_ref.as_str(), + Some(opts), + ); + let r = super::unencapsulate::join_fetch(r, driver) + .await + .with_context(|| format!("Parsing layer blob {}", layer.layer.digest()))?; + layer_commits.push(r.commit); + if !r.filtered.is_empty() { + let filtered = HashMap::from_iter(r.filtered.into_iter()); + tracing::debug!("Found {} filtered toplevels", filtered.len()); + layer_filtered_content.insert(layer.layer.digest().to_string(), filtered); + } else { + tracing::debug!("No filtered content"); + } + if let Some(p) = self.layer_progress.as_ref() { + p.send(ImportProgress::DerivedLayerCompleted(layer.layer.clone())) + .await?; + } + } + } + + // TODO change the imageproxy API to ensure this happens automatically when + // the image reference is dropped + proxy.close_image(&proxy_img).await?; + + // We're done with the proxy, make sure it didn't have any errors. + proxy.finalize().await?; + tracing::debug!("finalized proxy"); + + let serialized_manifest = serde_json::to_string(&import.manifest)?; + let serialized_config = serde_json::to_string(&import.config)?; + let mut metadata = HashMap::new(); + metadata.insert( + META_MANIFEST_DIGEST, + import.manifest_digest.to_string().to_variant(), + ); + metadata.insert(META_MANIFEST, serialized_manifest.to_variant()); + metadata.insert(META_CONFIG, serialized_config.to_variant()); + metadata.insert( + "ostree.importer.version", + env!("CARGO_PKG_VERSION").to_variant(), + ); + let filtered = layer_filtered_content.to_variant(); + metadata.insert(META_FILTERED, filtered); + let metadata = metadata.to_variant(); + + let timestamp = timestamp_of_manifest_or_config(&import.manifest, &import.config) + .unwrap_or_else(|| chrono::offset::Utc::now().timestamp() as u64); + // Destructure to transfer ownership to thread + let repo = self.repo; + let state = crate::tokio_util::spawn_blocking_cancellable_flatten( + move |cancellable| -> Result> { + use rustix::fd::AsRawFd; + + let cancellable = Some(cancellable); + let repo = &repo; + let txn = repo.auto_transaction(cancellable)?; + + let devino = ostree::RepoDevInoCache::new(); + let repodir = Dir::reopen_dir(&repo.dfd_borrow())?; + let repo_tmp = repodir.open_dir("tmp")?; + let td = cap_std_ext::cap_tempfile::TempDir::new_in(&repo_tmp)?; + + let rootpath = "root"; + let checkout_mode = if repo.mode() == ostree::RepoMode::Bare { + ostree::RepoCheckoutMode::None + } else { + ostree::RepoCheckoutMode::User + }; + let mut checkout_opts = ostree::RepoCheckoutAtOptions { + mode: checkout_mode, + overwrite_mode: ostree::RepoCheckoutOverwriteMode::UnionFiles, + devino_to_csum_cache: Some(devino.clone()), + no_copy_fallback: true, + force_copy_zerosized: true, + process_whiteouts: false, + ..Default::default() + }; + repo.checkout_at( + Some(&checkout_opts), + (*td).as_raw_fd(), + rootpath, + &base_commit, + cancellable, + ) + .context("Checking out base commit")?; + + // Layer all subsequent commits + checkout_opts.process_whiteouts = true; + for commit in layer_commits { + repo.checkout_at( + Some(&checkout_opts), + (*td).as_raw_fd(), + rootpath, + &commit, + cancellable, + ) + .with_context(|| format!("Checking out layer {commit}"))?; + } + + let modifier = + ostree::RepoCommitModifier::new(ostree::RepoCommitModifierFlags::CONSUME, None); + modifier.set_devino_cache(&devino); + // If we have derived layers, then we need to handle the case where + // the derived layers include custom policy. Just relabel everything + // in this case. + if have_derived_layers { + let rootpath = td.open_dir(rootpath)?; + let sepolicy = ostree::SePolicy::new_at(rootpath.as_raw_fd(), cancellable)?; + tracing::debug!("labeling from merged tree"); + modifier.set_sepolicy(Some(&sepolicy)); + } else { + tracing::debug!("labeling from base tree"); + // TODO: We can likely drop this; we know all labels should be pre-computed. + modifier.set_sepolicy_from_commit(repo, &base_commit, cancellable)?; + } + + let mt = ostree::MutableTree::new(); + repo.write_dfd_to_mtree( + (*td).as_raw_fd(), + rootpath, + &mt, + Some(&modifier), + cancellable, + ) + .context("Writing merged filesystem to mtree")?; + + let merged_root = repo + .write_mtree(&mt, cancellable) + .context("Writing mtree")?; + let merged_root = merged_root.downcast::().unwrap(); + let merged_commit = repo + .write_commit_with_time( + None, + None, + None, + Some(&metadata), + &merged_root, + timestamp, + cancellable, + ) + .context("Writing commit")?; + if !self.no_imgref { + repo.transaction_set_ref(None, &ostree_ref, Some(merged_commit.as_str())); + } + txn.commit(cancellable)?; + + if !self.disable_gc { + let n: u32 = gc_image_layers_impl(repo, cancellable)?; + tracing::debug!("pruned {n} layers"); + } + + // Here we re-query state just to run through the same code path, + // though it'd be cheaper to synthesize it from the data we already have. + let state = query_image_commit(repo, &merged_commit)?; + Ok(state) + }, + ) + .await?; + Ok(state) + } +} + +/// List all images stored +pub fn list_images(repo: &ostree::Repo) -> Result> { + let cancellable = gio::Cancellable::NONE; + let refs = repo.list_refs_ext( + Some(IMAGE_PREFIX), + ostree::RepoListRefsExtFlags::empty(), + cancellable, + )?; + refs.keys() + .map(|imgname| refescape::unprefix_unescape_ref(IMAGE_PREFIX, imgname)) + .collect() +} + +/// Attempt to query metadata for a pulled image; if it is corrupted, +/// the error is printed to stderr and None is returned. +fn try_query_image( + repo: &ostree::Repo, + imgref: &ImageReference, +) -> Result>> { + let ostree_ref = &ref_for_image(imgref)?; + if let Some(merge_rev) = repo.resolve_rev(ostree_ref, true)? { + match query_image_commit(repo, merge_rev.as_str()) { + Ok(r) => Ok(Some(r)), + Err(e) => { + eprintln!("error: failed to query image commit: {e}"); + Ok(None) + } + } + } else { + Ok(None) + } +} + +/// Query metadata for a pulled image. +#[context("Querying image {imgref}")] +pub fn query_image( + repo: &ostree::Repo, + imgref: &ImageReference, +) -> Result>> { + let ostree_ref = &ref_for_image(imgref)?; + let merge_rev = repo.resolve_rev(ostree_ref, true)?; + merge_rev + .map(|r| query_image_commit(repo, r.as_str())) + .transpose() +} + +/// Given detached commit metadata, parse the data that we serialized for a pending update (if any). +fn parse_cached_update(meta: &glib::VariantDict) -> Result> { + // Try to retrieve the manifest digest key from the commit detached metadata. + let manifest_digest = + if let Some(d) = meta.lookup::(ImageImporter::CACHED_KEY_MANIFEST_DIGEST)? { + d + } else { + // It's possible that something *else* wrote detached metadata, but without + // our key; gracefully handle that. + return Ok(None); + }; + let manifest_digest = Digest::from_str(&manifest_digest)?; + // If we found the cached manifest digest key, then we must have the manifest and config; + // otherwise that's an error. + let manifest = meta.lookup_value(ImageImporter::CACHED_KEY_MANIFEST, None); + let manifest: oci_image::ImageManifest = manifest + .as_ref() + .and_then(|v| v.str()) + .map(serde_json::from_str) + .transpose()? + .ok_or_else(|| { + anyhow!( + "Expected cached manifest {}", + ImageImporter::CACHED_KEY_MANIFEST + ) + })?; + let config = meta.lookup_value(ImageImporter::CACHED_KEY_CONFIG, None); + let config: oci_image::ImageConfiguration = config + .as_ref() + .and_then(|v| v.str()) + .map(serde_json::from_str) + .transpose()? + .ok_or_else(|| { + anyhow!( + "Expected cached manifest {}", + ImageImporter::CACHED_KEY_CONFIG + ) + })?; + Ok(Some(CachedImageUpdate { + manifest, + config, + manifest_digest, + })) +} + +/// Query metadata for a pulled image via an OSTree commit digest. +/// The digest must refer to a pulled container image's merge commit. +pub fn query_image_commit(repo: &ostree::Repo, commit: &str) -> Result> { + let merge_commit = commit.to_string(); + let merge_commit_obj = repo.load_commit(commit)?.0; + let commit_meta = &merge_commit_obj.child_value(0); + let commit_meta = &ostree::glib::VariantDict::new(Some(commit_meta)); + let (manifest, manifest_digest) = manifest_data_from_commitmeta(commit_meta)?; + let configuration = image_config_from_commitmeta(commit_meta)?; + let mut layers = manifest.layers().iter().cloned(); + // We require a base layer. + let base_layer = layers.next().ok_or_else(|| anyhow!("No layers found"))?; + let base_layer = query_layer(repo, base_layer)?; + let ostree_ref = base_layer.ostree_ref.as_str(); + let base_commit = base_layer + .commit + .ok_or_else(|| anyhow!("Missing base image ref {ostree_ref}"))?; + + let detached_commitmeta = + repo.read_commit_detached_metadata(&merge_commit, gio::Cancellable::NONE)?; + let detached_commitmeta = detached_commitmeta + .as_ref() + .map(|v| glib::VariantDict::new(Some(v))); + let cached_update = detached_commitmeta + .as_ref() + .map(parse_cached_update) + .transpose()? + .flatten(); + let state = Box::new(LayeredImageState { + base_commit, + merge_commit, + manifest_digest, + manifest, + configuration, + cached_update, + }); + tracing::debug!("Wrote merge commit {}", state.merge_commit); + Ok(state) +} + +fn manifest_for_image(repo: &ostree::Repo, imgref: &ImageReference) -> Result { + let ostree_ref = ref_for_image(imgref)?; + let rev = repo.require_rev(&ostree_ref)?; + let (commit_obj, _) = repo.load_commit(rev.as_str())?; + let commit_meta = &glib::VariantDict::new(Some(&commit_obj.child_value(0))); + Ok(manifest_data_from_commitmeta(commit_meta)?.0) +} + +/// Copy a downloaded image from one repository to another, while also +/// optionally changing the image reference type. +#[context("Copying image")] +pub async fn copy( + src_repo: &ostree::Repo, + src_imgref: &ImageReference, + dest_repo: &ostree::Repo, + dest_imgref: &ImageReference, +) -> Result<()> { + let src_ostree_ref = ref_for_image(src_imgref)?; + let src_commit = src_repo.require_rev(&src_ostree_ref)?; + let manifest = manifest_for_image(src_repo, src_imgref)?; + // Create a task to copy each layer, plus the final ref + let layer_refs = manifest + .layers() + .iter() + .map(ref_for_layer) + .chain(std::iter::once(Ok(src_commit.to_string()))); + for ostree_ref in layer_refs { + let ostree_ref = ostree_ref?; + let src_repo = src_repo.clone(); + let dest_repo = dest_repo.clone(); + crate::tokio_util::spawn_blocking_cancellable_flatten(move |cancellable| -> Result<_> { + let cancellable = Some(cancellable); + let srcfd = &format!("file:///proc/self/fd/{}", src_repo.dfd()); + let flags = ostree::RepoPullFlags::MIRROR; + let opts = glib::VariantDict::new(None); + let refs = [ostree_ref.as_str()]; + // Some older archives may have bindings, we don't need to verify them. + opts.insert("disable-verify-bindings", true); + opts.insert("refs", &refs[..]); + opts.insert("flags", flags.bits() as i32); + let options = opts.to_variant(); + dest_repo.pull_with_options(srcfd, &options, None, cancellable)?; + Ok(()) + }) + .await?; + } + + let dest_ostree_ref = ref_for_image(dest_imgref)?; + dest_repo.set_ref_immediate( + None, + &dest_ostree_ref, + Some(&src_commit), + gio::Cancellable::NONE, + )?; + + Ok(()) +} + +/// Options controlling commit export into OCI +#[derive(Clone, Debug, Default)] +#[non_exhaustive] +pub struct ExportToOCIOpts { + /// If true, do not perform gzip compression of the tar layers. + pub skip_compression: bool, + /// Path to Docker-formatted authentication file. + pub authfile: Option, + /// Output progress to stdout + pub progress_to_stdout: bool, +} + +/// The way we store "chunk" layers in ostree is by writing a commit +/// whose filenames are their own object identifier. This function parses +/// what is written by the `ImporterMode::ObjectSet` logic, turning +/// it back into a "chunked" structure that is used by the export code. +fn chunking_from_layer_committed( + repo: &ostree::Repo, + l: &Descriptor, + chunking: &mut chunking::Chunking, +) -> Result<()> { + let mut chunk = Chunk::default(); + let layer_ref = &ref_for_layer(l)?; + let root = repo.read_commit(layer_ref, gio::Cancellable::NONE)?.0; + let e = root.enumerate_children( + "standard::name,standard::size", + gio::FileQueryInfoFlags::NOFOLLOW_SYMLINKS, + gio::Cancellable::NONE, + )?; + for child in e.clone() { + let child = &child?; + // The name here should be a valid checksum + let name = child.name(); + // SAFETY: ostree doesn't give us non-UTF8 filenames + let name = Utf8Path::from_path(&name).unwrap(); + ostree::validate_checksum_string(name.as_str())?; + chunking.remainder.move_obj(&mut chunk, name.as_str()); + } + chunking.chunks.push(chunk); + Ok(()) +} + +/// Export an imported container image to a target OCI directory. +#[context("Copying image")] +pub(crate) fn export_to_oci( + repo: &ostree::Repo, + imgref: &ImageReference, + dest_oci: &Dir, + tag: Option<&str>, + opts: ExportToOCIOpts, +) -> Result { + let srcinfo = query_image(repo, imgref)?.ok_or_else(|| anyhow!("No such image"))?; + let (commit_layer, component_layers, remaining_layers) = + parse_manifest_layout(&srcinfo.manifest, &srcinfo.configuration)?; + let commit_chunk_ref = ref_for_layer(commit_layer)?; + let commit_chunk_rev = repo.require_rev(&commit_chunk_ref)?; + let mut chunking = chunking::Chunking::new(repo, &commit_chunk_rev)?; + for layer in component_layers { + chunking_from_layer_committed(repo, layer, &mut chunking)?; + } + // Unfortunately today we can't guarantee we reserialize the same tar stream + // or compression, so we'll need to generate a new copy of the manifest and config + // with the layers reset. + let mut new_manifest = srcinfo.manifest.clone(); + new_manifest.layers_mut().clear(); + let mut new_config = srcinfo.configuration.clone(); + new_config.history_mut().clear(); + + let mut dest_oci = ocidir::OciDir::ensure(dest_oci)?; + + let opts = ExportOpts { + skip_compression: opts.skip_compression, + authfile: opts.authfile, + ..Default::default() + }; + + let mut labels = HashMap::new(); + + // Given the object chunking information we recomputed from what + // we found on disk, re-serialize to layers (tarballs). + export_chunked( + repo, + &srcinfo.base_commit, + &mut dest_oci, + &mut new_manifest, + &mut new_config, + &mut labels, + chunking, + &opts, + "", + )?; + + // Now, handle the non-ostree layers; this is a simple conversion of + // + let compression = opts.skip_compression.then_some(Compression::none()); + for (i, layer) in remaining_layers.iter().enumerate() { + let layer_ref = &ref_for_layer(layer)?; + let mut target_blob = dest_oci.create_gzip_layer(compression)?; + // Sadly the libarchive stuff isn't exposed via Rust due to type unsafety, + // so we'll just fork off the CLI. + let repo_dfd = repo.dfd_borrow(); + let repo_dir = cap_std_ext::cap_std::fs::Dir::reopen_dir(&repo_dfd)?; + let mut subproc = std::process::Command::new("ostree") + .args(["--repo=.", "export", layer_ref.as_str()]) + .stdout(std::process::Stdio::piped()) + .cwd_dir(repo_dir) + .spawn()?; + // SAFETY: we piped just above + let mut stdout = subproc.stdout.take().unwrap(); + std::io::copy(&mut stdout, &mut target_blob).context("Creating blob")?; + let layer = target_blob.complete()?; + let previous_annotations = srcinfo + .manifest + .layers() + .get(i) + .and_then(|l| l.annotations().as_ref()) + .cloned(); + let previous_description = srcinfo + .configuration + .history() + .get(i) + .and_then(|h| h.comment().as_deref()) + .unwrap_or_default(); + dest_oci.push_layer( + &mut new_manifest, + &mut new_config, + layer, + previous_description, + previous_annotations, + ) + } + + let new_config = dest_oci.write_config(new_config)?; + new_manifest.set_config(new_config); + + Ok(dest_oci.insert_manifest(new_manifest, tag, oci_image::Platform::default())?) +} + +/// Given a container image reference which is stored in `repo`, export it to the +/// target image location. +#[context("Export")] +pub async fn export( + repo: &ostree::Repo, + src_imgref: &ImageReference, + dest_imgref: &ImageReference, + opts: Option, +) -> Result { + let opts = opts.unwrap_or_default(); + let target_oci = dest_imgref.transport == Transport::OciDir; + let tempdir = if !target_oci { + let vartmp = cap_std::fs::Dir::open_ambient_dir("/var/tmp", cap_std::ambient_authority())?; + let td = cap_std_ext::cap_tempfile::TempDir::new_in(&vartmp)?; + // Always skip compression when making a temporary copy + let opts = ExportToOCIOpts { + skip_compression: true, + progress_to_stdout: opts.progress_to_stdout, + ..Default::default() + }; + export_to_oci(repo, src_imgref, &td, None, opts)?; + td + } else { + let (path, tag) = parse_oci_path_and_tag(dest_imgref.name.as_str()); + tracing::debug!("using OCI path={path} tag={tag:?}"); + let path = Dir::open_ambient_dir(path, cap_std::ambient_authority()) + .with_context(|| format!("Opening {path}"))?; + let descriptor = export_to_oci(repo, src_imgref, &path, tag, opts)?; + return Ok(descriptor.digest().clone()); + }; + // Pass the temporary oci directory as the current working directory for the skopeo process + let target_fd = 3i32; + let tempoci = ImageReference { + transport: Transport::OciDir, + name: format!("/proc/self/fd/{target_fd}"), + }; + let authfile = opts.authfile.as_deref(); + skopeo::copy( + &tempoci, + dest_imgref, + authfile, + Some((std::sync::Arc::new(tempdir.try_clone()?.into()), target_fd)), + opts.progress_to_stdout, + ) + .await +} + +/// Iterate over deployment commits, returning the manifests from +/// commits which point to a container image. +#[context("Listing deployment manifests")] +fn list_container_deployment_manifests( + repo: &ostree::Repo, + cancellable: Option<&gio::Cancellable>, +) -> Result> { + // Gather all refs which start with ostree/0/ or ostree/1/ or rpmostree/base/ + // and create a set of the commits which they reference. + let commits = OSTREE_BASE_DEPLOYMENT_REFS + .iter() + .chain(RPMOSTREE_BASE_REFS) + .chain(std::iter::once(&BASE_IMAGE_PREFIX)) + .try_fold( + std::collections::HashSet::new(), + |mut acc, &p| -> Result<_> { + let refs = repo.list_refs_ext( + Some(p), + ostree::RepoListRefsExtFlags::empty(), + cancellable, + )?; + for (_, v) in refs { + acc.insert(v); + } + Ok(acc) + }, + )?; + // Loop over the commits - if they refer to a container image, add that to our return value. + let mut r = Vec::new(); + for commit in commits { + let commit_obj = repo.load_commit(&commit)?.0; + let commit_meta = &glib::VariantDict::new(Some(&commit_obj.child_value(0))); + if commit_meta + .lookup::(META_MANIFEST_DIGEST)? + .is_some() + { + tracing::trace!("Commit {commit} is a container image"); + let manifest = manifest_data_from_commitmeta(commit_meta)?.0; + r.push(manifest); + } + } + Ok(r) +} + +/// Garbage collect unused image layer references. +/// +/// This function assumes no transaction is active on the repository. +/// The underlying objects are *not* pruned; that requires a separate invocation +/// of [`ostree::Repo::prune`]. +pub fn gc_image_layers(repo: &ostree::Repo) -> Result { + gc_image_layers_impl(repo, gio::Cancellable::NONE) +} + +#[context("Pruning image layers")] +fn gc_image_layers_impl( + repo: &ostree::Repo, + cancellable: Option<&gio::Cancellable>, +) -> Result { + let all_images = list_images(repo)?; + let deployment_commits = list_container_deployment_manifests(repo, cancellable)?; + let all_manifests = all_images + .into_iter() + .map(|img| { + ImageReference::try_from(img.as_str()).and_then(|ir| manifest_for_image(repo, &ir)) + }) + .chain(deployment_commits.into_iter().map(Ok)) + .collect::>>()?; + tracing::debug!("Images found: {}", all_manifests.len()); + let mut referenced_layers = BTreeSet::new(); + for m in all_manifests.iter() { + for layer in m.layers() { + referenced_layers.insert(layer.digest().to_string()); + } + } + tracing::debug!("Referenced layers: {}", referenced_layers.len()); + let found_layers = repo + .list_refs_ext( + Some(LAYER_PREFIX), + ostree::RepoListRefsExtFlags::empty(), + cancellable, + )? + .into_iter() + .map(|v| v.0); + tracing::debug!("Found layers: {}", found_layers.len()); + let mut pruned = 0u32; + for layer_ref in found_layers { + let layer_digest = refescape::unprefix_unescape_ref(LAYER_PREFIX, &layer_ref)?; + if referenced_layers.remove(layer_digest.as_str()) { + continue; + } + pruned += 1; + tracing::debug!("Pruning: {}", layer_ref.as_str()); + repo.set_ref_immediate(None, layer_ref.as_str(), None, cancellable)?; + } + + Ok(pruned) +} + +#[cfg(feature = "internal-testing-api")] +/// Return how many container blobs (layers) are stored +pub fn count_layer_references(repo: &ostree::Repo) -> Result { + let cancellable = gio::Cancellable::NONE; + let n = repo + .list_refs_ext( + Some(LAYER_PREFIX), + ostree::RepoListRefsExtFlags::empty(), + cancellable, + )? + .len(); + Ok(n as u32) +} + +/// Given an image, if it has any non-ostree compatible content, return a suitable +/// warning message. +pub fn image_filtered_content_warning( + repo: &ostree::Repo, + image: &ImageReference, +) -> Result> { + use std::fmt::Write; + + let ostree_ref = ref_for_image(image)?; + let rev = repo.require_rev(&ostree_ref)?; + let commit_obj = repo.load_commit(rev.as_str())?.0; + let commit_meta = &glib::VariantDict::new(Some(&commit_obj.child_value(0))); + + let r = commit_meta + .lookup::(META_FILTERED)? + .filter(|v| !v.is_empty()) + .map(|v| { + let mut filtered = HashMap::<&String, u32>::new(); + for paths in v.values() { + for (k, v) in paths { + let e = filtered.entry(k).or_default(); + *e += v; + } + } + let mut buf = "Image contains non-ostree compatible file paths:".to_string(); + for (k, v) in filtered { + write!(buf, " {k}: {v}").unwrap(); + } + buf + }); + Ok(r) +} + +/// Remove the specified image reference. If the image is already +/// not present, this function will successfully perform no operation. +/// +/// This function assumes no transaction is active on the repository. +/// The underlying layers are *not* pruned; that requires a separate invocation +/// of [`gc_image_layers`]. +#[context("Pruning {img}")] +pub fn remove_image(repo: &ostree::Repo, img: &ImageReference) -> Result { + let ostree_ref = &ref_for_image(img)?; + let found = repo.resolve_rev(ostree_ref, true)?.is_some(); + // Note this API is already idempotent, but we might as well avoid another + // trip into ostree. + if found { + repo.set_ref_immediate(None, ostree_ref, None, gio::Cancellable::NONE)?; + } + Ok(found) +} + +/// Remove the specified image references. If an image is not found, further +/// images will be removed, but an error will be returned. +/// +/// This function assumes no transaction is active on the repository. +/// The underlying layers are *not* pruned; that requires a separate invocation +/// of [`gc_image_layers`]. +pub fn remove_images<'a>( + repo: &ostree::Repo, + imgs: impl IntoIterator, +) -> Result<()> { + let mut missing = Vec::new(); + for img in imgs.into_iter() { + let found = remove_image(repo, img)?; + if !found { + missing.push(img); + } + } + if !missing.is_empty() { + let missing = missing.into_iter().fold("".to_string(), |mut a, v| { + a.push_str(&v.to_string()); + a + }); + return Err(anyhow::anyhow!("Missing images: {missing}")); + } + Ok(()) +} + +#[derive(Debug, Default)] +struct CompareState { + verified: BTreeSet, + inode_corrupted: BTreeSet, + unknown_corrupted: BTreeSet, +} + +impl CompareState { + fn is_ok(&self) -> bool { + self.inode_corrupted.is_empty() && self.unknown_corrupted.is_empty() + } +} + +fn compare_file_info(src: &gio::FileInfo, target: &gio::FileInfo) -> bool { + if src.file_type() != target.file_type() { + return false; + } + if src.size() != target.size() { + return false; + } + for attr in ["unix::uid", "unix::gid", "unix::mode"] { + if src.attribute_uint32(attr) != target.attribute_uint32(attr) { + return false; + } + } + true +} + +#[context("Querying object inode")] +fn inode_of_object(repo: &ostree::Repo, checksum: &str) -> Result { + let repodir = Dir::reopen_dir(&repo.dfd_borrow())?; + let (prefix, suffix) = checksum.split_at(2); + let objpath = format!("objects/{}/{}.file", prefix, suffix); + let metadata = repodir.symlink_metadata(objpath)?; + Ok(metadata.ino()) +} + +fn compare_commit_trees( + repo: &ostree::Repo, + root: &Utf8Path, + target: &ostree::RepoFile, + expected: &ostree::RepoFile, + exact: bool, + colliding_inodes: &BTreeSet, + state: &mut CompareState, +) -> Result<()> { + let cancellable = gio::Cancellable::NONE; + let queryattrs = "standard::name,standard::type"; + let queryflags = gio::FileQueryInfoFlags::NOFOLLOW_SYMLINKS; + let expected_iter = expected.enumerate_children(queryattrs, queryflags, cancellable)?; + + while let Some(expected_info) = expected_iter.next_file(cancellable)? { + let expected_child = expected_iter.child(&expected_info); + let name = expected_info.name(); + let name = name.to_str().expect("UTF-8 ostree name"); + let path = Utf8PathBuf::from(format!("{root}{name}")); + let target_child = target.child(name); + let target_info = crate::diff::query_info_optional(&target_child, queryattrs, queryflags) + .context("querying optional to")?; + let is_dir = matches!(expected_info.file_type(), gio::FileType::Directory); + if let Some(target_info) = target_info { + let to_child = target_child + .downcast::() + .expect("downcast"); + to_child.ensure_resolved()?; + let from_child = expected_child + .downcast::() + .expect("downcast"); + from_child.ensure_resolved()?; + + if is_dir { + let from_contents_checksum = from_child.tree_get_contents_checksum(); + let to_contents_checksum = to_child.tree_get_contents_checksum(); + if from_contents_checksum != to_contents_checksum { + let subpath = Utf8PathBuf::from(format!("{}/", path)); + compare_commit_trees( + repo, + &subpath, + &from_child, + &to_child, + exact, + colliding_inodes, + state, + )?; + } + } else { + let from_checksum = from_child.checksum(); + let to_checksum = to_child.checksum(); + let matches = if exact { + from_checksum == to_checksum + } else { + compare_file_info(&target_info, &expected_info) + }; + if !matches { + let from_inode = inode_of_object(repo, &from_checksum)?; + let to_inode = inode_of_object(repo, &to_checksum)?; + if colliding_inodes.contains(&from_inode) + || colliding_inodes.contains(&to_inode) + { + state.inode_corrupted.insert(path); + } else { + state.unknown_corrupted.insert(path); + } + } else { + state.verified.insert(path); + } + } + } else { + eprintln!("Missing {path}"); + state.unknown_corrupted.insert(path); + } + } + Ok(()) +} + +#[context("Verifying container image state")] +pub(crate) fn verify_container_image( + sysroot: &SysrootLock, + imgref: &ImageReference, + state: &LayeredImageState, + colliding_inodes: &BTreeSet, + verbose: bool, +) -> Result { + let cancellable = gio::Cancellable::NONE; + let repo = &sysroot.repo(); + let merge_commit = state.merge_commit.as_str(); + let merge_commit_root = repo.read_commit(merge_commit, gio::Cancellable::NONE)?.0; + let merge_commit_root = merge_commit_root + .downcast::() + .expect("downcast"); + merge_commit_root.ensure_resolved()?; + + let (commit_layer, _component_layers, remaining_layers) = + parse_manifest_layout(&state.manifest, &state.configuration)?; + + let mut comparison_state = CompareState::default(); + + let query = |l: &Descriptor| query_layer(repo, l.clone()); + + let base_tree = repo + .read_commit(&state.base_commit, cancellable)? + .0 + .downcast::() + .expect("downcast"); + println!( + "Verifying with base ostree layer {}", + ref_for_layer(commit_layer)? + ); + compare_commit_trees( + repo, + "/".into(), + &merge_commit_root, + &base_tree, + true, + colliding_inodes, + &mut comparison_state, + )?; + + let remaining_layers = remaining_layers + .into_iter() + .map(query) + .collect::>>()?; + + println!("Image has {} derived layers", remaining_layers.len()); + + for layer in remaining_layers.iter().rev() { + let layer_ref = layer.ostree_ref.as_str(); + let layer_commit = layer + .commit + .as_deref() + .ok_or_else(|| anyhow!("Missing layer {layer_ref}"))?; + let layer_tree = repo + .read_commit(layer_commit, cancellable)? + .0 + .downcast::() + .expect("downcast"); + compare_commit_trees( + repo, + "/".into(), + &merge_commit_root, + &layer_tree, + false, + colliding_inodes, + &mut comparison_state, + )?; + } + + let n_verified = comparison_state.verified.len(); + if comparison_state.is_ok() { + println!("OK image {imgref} (verified={n_verified})"); + println!(); + } else { + let n_inode = comparison_state.inode_corrupted.len(); + let n_other = comparison_state.unknown_corrupted.len(); + eprintln!("warning: Found corrupted merge commit"); + eprintln!(" inode clashes: {n_inode}"); + eprintln!(" unknown: {n_other}"); + eprintln!(" ok: {n_verified}"); + if verbose { + eprintln!("Mismatches:"); + for path in comparison_state.inode_corrupted { + eprintln!(" inode: {path}"); + } + for path in comparison_state.unknown_corrupted { + eprintln!(" other: {path}"); + } + } + eprintln!(); + return Ok(false); + } + + Ok(true) +} + +#[cfg(test)] +mod tests { + use oci_image::{DescriptorBuilder, MediaType, Sha256Digest}; + + use super::*; + + #[test] + fn test_ref_for_descriptor() { + let d = DescriptorBuilder::default() + .size(42u64) + .media_type(MediaType::ImageManifest) + .digest( + Sha256Digest::from_str( + "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae", + ) + .unwrap(), + ) + .build() + .unwrap(); + assert_eq!(ref_for_layer(&d).unwrap(), "ostree/container/blob/sha256_3A_2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae"); + } +} diff --git a/ostree-ext/src/container/tests/it/fixtures/exampleos.tar.zst b/ostree-ext/src/container/tests/it/fixtures/exampleos.tar.zst new file mode 100644 index 00000000..8e8969d8 Binary files /dev/null and b/ostree-ext/src/container/tests/it/fixtures/exampleos.tar.zst differ diff --git a/ostree-ext/src/container/unencapsulate.rs b/ostree-ext/src/container/unencapsulate.rs new file mode 100644 index 00000000..9e0818a8 --- /dev/null +++ b/ostree-ext/src/container/unencapsulate.rs @@ -0,0 +1,276 @@ +//! APIs for "unencapsulating" OSTree commits from container images +//! +//! This code only operates on container images that were created via +//! [`encapsulate`]. +//! +//! # External depenendency on container-image-proxy +//! +//! This code requires +//! installed as a binary in $PATH. +//! +//! The rationale for this is that while there exist Rust crates to speak +//! the Docker distribution API, the Go library +//! supports key things we want for production use like: +//! +//! - Image mirroring and remapping; effectively `man containers-registries.conf` +//! For example, we need to support an administrator mirroring an ostree-container +//! into a disconnected registry, without changing all the pull specs. +//! - Signing +//! +//! Additionally, the proxy "upconverts" manifests into OCI, so we don't need to care +//! about parsing the Docker manifest format (as used by most registries still). +//! +//! [`encapsulate`]: [`super::encapsulate()`] + +// # Implementation +// +// First, we support explicitly fetching just the manifest: https://github.com/opencontainers/image-spec/blob/main/manifest.md +// This will give us information about the layers it contains, and crucially the digest (sha256) of +// the manifest is how higher level software can detect changes. +// +// Once we have the manifest, we expect it to point to a single `application/vnd.oci.image.layer.v1.tar+gzip` layer, +// which is exactly what is exported by the [`crate::tar::export`] process. + +use crate::container::store::LayerProgress; + +use super::*; +use containers_image_proxy::{ImageProxy, OpenedImage}; +use fn_error_context::context; +use futures_util::{Future, FutureExt}; +use oci_spec::image::{self as oci_image, Digest}; +use std::io::Read; +use std::sync::{Arc, Mutex}; +use tokio::{ + io::{AsyncBufRead, AsyncRead}, + sync::watch::{Receiver, Sender}, +}; +use tracing::instrument; + +/// The legacy MIME type returned by the skopeo/(containers/storage) code +/// when we have local uncompressed docker-formatted image. +/// TODO: change the skopeo code to shield us from this correctly +const DOCKER_TYPE_LAYER_TAR: &str = "application/vnd.docker.image.rootfs.diff.tar"; + +type Progress = tokio::sync::watch::Sender; + +/// A read wrapper that updates the download progress. +#[pin_project::pin_project] +#[derive(Debug)] +pub(crate) struct ProgressReader { + #[pin] + pub(crate) reader: T, + #[pin] + pub(crate) progress: Arc>, +} + +impl ProgressReader { + pub(crate) fn new(reader: T) -> (Self, Receiver) { + let (progress, r) = tokio::sync::watch::channel(1); + let progress = Arc::new(Mutex::new(progress)); + (ProgressReader { reader, progress }, r) + } +} + +impl AsyncRead for ProgressReader { + fn poll_read( + self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + buf: &mut tokio::io::ReadBuf<'_>, + ) -> std::task::Poll> { + let this = self.project(); + let len = buf.filled().len(); + match this.reader.poll_read(cx, buf) { + v @ std::task::Poll::Ready(Ok(_)) => { + let progress = this.progress.lock().unwrap(); + let state = { + let mut state = *progress.borrow(); + let newlen = buf.filled().len(); + debug_assert!(newlen >= len); + let read = (newlen - len) as u64; + state += read; + state + }; + // Ignore errors, if the caller disconnected from progress that's OK. + let _ = progress.send(state); + v + } + o => o, + } + } +} + +async fn fetch_manifest_impl( + proxy: &mut ImageProxy, + imgref: &OstreeImageReference, +) -> Result<(oci_image::ImageManifest, oci_image::Digest)> { + let oi = &proxy.open_image(&imgref.imgref.to_string()).await?; + let (digest, manifest) = proxy.fetch_manifest(oi).await?; + proxy.close_image(oi).await?; + Ok((manifest, oci_image::Digest::from_str(digest.as_str())?)) +} + +/// Download the manifest for a target image and its sha256 digest. +#[context("Fetching manifest")] +pub async fn fetch_manifest( + imgref: &OstreeImageReference, +) -> Result<(oci_image::ImageManifest, oci_image::Digest)> { + let mut proxy = ImageProxy::new().await?; + fetch_manifest_impl(&mut proxy, imgref).await +} + +/// Download the manifest for a target image and its sha256 digest, as well as the image configuration. +#[context("Fetching manifest and config")] +pub async fn fetch_manifest_and_config( + imgref: &OstreeImageReference, +) -> Result<( + oci_image::ImageManifest, + oci_image::Digest, + oci_image::ImageConfiguration, +)> { + let proxy = ImageProxy::new().await?; + let oi = &proxy.open_image(&imgref.imgref.to_string()).await?; + let (digest, manifest) = proxy.fetch_manifest(oi).await?; + let digest = oci_image::Digest::from_str(&digest)?; + let config = proxy.fetch_config(oi).await?; + Ok((manifest, digest, config)) +} + +/// The result of an import operation +#[derive(Debug)] +pub struct Import { + /// The ostree commit that was imported + pub ostree_commit: String, + /// The image digest retrieved + pub image_digest: Digest, + + /// Any deprecation warning + pub deprecated_warning: Option, +} + +/// Use this to process potential errors from a worker and a driver. +/// This is really a brutal hack around the fact that an error can occur +/// on either our side or in the proxy. But if an error occurs on our +/// side, then we will close the pipe, which will *also* cause the proxy +/// to error out. +/// +/// What we really want is for the proxy to tell us when it got an +/// error from us closing the pipe. Or, we could store that state +/// on our side. Both are slightly tricky, so we have this (again) +/// hacky thing where we just search for `broken pipe` in the error text. +/// +/// Or to restate all of the above - what this function does is check +/// to see if the worker function had an error *and* if the proxy +/// had an error, but if the proxy's error ends in `broken pipe` +/// then it means the real only error is from the worker. +pub(crate) async fn join_fetch( + worker: impl Future>, + driver: impl Future>, +) -> Result { + let (worker, driver) = tokio::join!(worker, driver); + match (worker, driver) { + (Ok(t), Ok(())) => Ok(t), + (Err(worker), Err(driver)) => { + let text = driver.root_cause().to_string(); + if text.ends_with("broken pipe") { + tracing::trace!("Ignoring broken pipe failure from driver"); + Err(worker) + } else { + Err(worker.context(format!("proxy failure: {} and client error", text))) + } + } + (Ok(_), Err(driver)) => Err(driver), + (Err(worker), Ok(())) => Err(worker), + } +} + +/// Fetch a container image and import its embedded OSTree commit. +#[context("Importing {}", imgref)] +#[instrument(level = "debug", skip(repo))] +pub async fn unencapsulate(repo: &ostree::Repo, imgref: &OstreeImageReference) -> Result { + let importer = super::store::ImageImporter::new(repo, imgref, Default::default()).await?; + importer.unencapsulate().await +} + +/// Create a decompressor for this MIME type, given a stream of input. +pub(crate) fn decompressor( + media_type: &oci_image::MediaType, + src: impl Read + Send + 'static, +) -> Result> { + let r: Box = match media_type { + m @ (oci_image::MediaType::ImageLayerGzip | oci_image::MediaType::ImageLayerZstd) => { + if matches!(m, oci_image::MediaType::ImageLayerZstd) { + Box::new(zstd::stream::read::Decoder::new(src)?) + } else { + Box::new(flate2::bufread::GzDecoder::new(std::io::BufReader::new( + src, + ))) + } + } + oci_image::MediaType::ImageLayer => Box::new(src), + oci_image::MediaType::Other(t) if t.as_str() == DOCKER_TYPE_LAYER_TAR => Box::new(src), + o => anyhow::bail!("Unhandled layer type: {}", o), + }; + Ok(r) +} + +/// A wrapper for [`get_blob`] which fetches a layer and decompresses it. +pub(crate) async fn fetch_layer<'a>( + proxy: &'a ImageProxy, + img: &OpenedImage, + manifest: &oci_image::ImageManifest, + layer: &'a oci_image::Descriptor, + progress: Option<&'a Sender>>, + layer_info: Option<&Vec>, + transport_src: Transport, +) -> Result<( + Box, + impl Future> + 'a, + oci_image::MediaType, +)> { + use futures_util::future::Either; + tracing::debug!("fetching {}", layer.digest()); + let layer_index = manifest.layers().iter().position(|x| x == layer).unwrap(); + let (blob, driver, size); + let media_type: oci_image::MediaType; + match transport_src { + Transport::ContainerStorage => { + let layer_info = layer_info + .ok_or_else(|| anyhow!("skopeo too old to pull from containers-storage"))?; + let n_layers = layer_info.len(); + let layer_blob = layer_info.get(layer_index).ok_or_else(|| { + anyhow!("blobid position {layer_index} exceeds diffid count {n_layers}") + })?; + size = layer_blob.size; + media_type = layer_blob.media_type.clone(); + (blob, driver) = proxy.get_blob(img, &layer_blob.digest, size).await?; + } + _ => { + size = layer.size(); + media_type = layer.media_type().clone(); + (blob, driver) = proxy.get_blob(img, layer.digest(), size).await?; + } + }; + + let driver = async { driver.await.map_err(Into::into) }; + + if let Some(progress) = progress { + let (readprogress, mut readwatch) = ProgressReader::new(blob); + let readprogress = tokio::io::BufReader::new(readprogress); + let readproxy = async move { + while let Ok(()) = readwatch.changed().await { + let fetched = readwatch.borrow_and_update(); + let status = LayerProgress { + layer_index, + fetched: *fetched, + total: size, + }; + progress.send_replace(Some(status)); + } + }; + let reader = Box::new(readprogress); + let driver = futures_util::future::join(readproxy, driver).map(|r| r.1); + Ok((reader, Either::Left(driver), media_type)) + } else { + Ok((Box::new(blob), Either::Right(driver), media_type)) + } +} diff --git a/ostree-ext/src/container/update_detachedmeta.rs b/ostree-ext/src/container/update_detachedmeta.rs new file mode 100644 index 00000000..2803fab1 --- /dev/null +++ b/ostree-ext/src/container/update_detachedmeta.rs @@ -0,0 +1,138 @@ +use super::ImageReference; +use crate::container::{skopeo, DIFFID_LABEL}; +use crate::container::{store as container_store, Transport}; +use anyhow::{anyhow, Context, Result}; +use camino::Utf8Path; +use cap_std::fs::Dir; +use cap_std_ext::cap_std; +use containers_image_proxy::oci_spec::image as oci_image; +use std::io::{BufReader, BufWriter}; + +/// Given an OSTree container image reference, update the detached metadata (e.g. GPG signature) +/// while preserving all other container image metadata. +/// +/// The return value is the manifest digest of (e.g. `@sha256:`) the image. +pub async fn update_detached_metadata( + src: &ImageReference, + dest: &ImageReference, + detached_buf: Option<&[u8]>, +) -> Result { + // For now, convert the source to a temporary OCI directory, so we can directly + // parse and manipulate it. In the future this will be replaced by https://github.com/ostreedev/ostree-rs-ext/issues/153 + // and other work to directly use the containers/image API via containers-image-proxy. + let tempdir = tempfile::tempdir_in("/var/tmp")?; + let tempsrc = tempdir.path().join("src"); + let tempsrc_utf8 = Utf8Path::from_path(&tempsrc).ok_or_else(|| anyhow!("Invalid tempdir"))?; + let tempsrc_ref = ImageReference { + transport: Transport::OciDir, + name: tempsrc_utf8.to_string(), + }; + + // Full copy of the source image + let pulled_digest = skopeo::copy(src, &tempsrc_ref, None, None, false) + .await + .context("Creating temporary copy to OCI dir")?; + + // Copy to the thread + let detached_buf = detached_buf.map(Vec::from); + let tempsrc_ref_path = tempsrc_ref.name.clone(); + // Fork a thread to do the heavy lifting of filtering the tar stream, rewriting the manifest/config. + crate::tokio_util::spawn_blocking_cancellable_flatten(move |cancellable| { + // Open the temporary OCI directory. + let tempsrc = Dir::open_ambient_dir(tempsrc_ref_path, cap_std::ambient_authority()) + .context("Opening src")?; + let tempsrc = ocidir::OciDir::open(&tempsrc)?; + + // Load the manifest, platform, and config + let idx = tempsrc + .read_index()? + .ok_or(anyhow!("Reading image index from source"))?; + let manifest_descriptor = idx + .manifests() + .first() + .ok_or(anyhow!("No manifests in index"))?; + let mut manifest: oci_image::ImageManifest = tempsrc + .read_json_blob(manifest_descriptor) + .context("Reading manifest json blob")?; + + anyhow::ensure!(manifest_descriptor.digest() == &pulled_digest); + let platform = manifest_descriptor + .platform() + .as_ref() + .cloned() + .unwrap_or_default(); + let mut config: oci_image::ImageConfiguration = + tempsrc.read_json_blob(manifest.config())?; + let mut ctrcfg = config + .config() + .as_ref() + .cloned() + .ok_or_else(|| anyhow!("Image is missing container configuration"))?; + + // Find the OSTree commit layer we want to replace + let (commit_layer, _, _) = container_store::parse_manifest_layout(&manifest, &config)?; + let commit_layer_idx = manifest + .layers() + .iter() + .position(|x| x == commit_layer) + .unwrap(); + + // Create a new layer + let out_layer = { + // Create tar streams for source and destination + let src_layer = BufReader::new(tempsrc.read_blob(commit_layer)?); + let mut src_layer = flate2::read::GzDecoder::new(src_layer); + let mut out_layer = BufWriter::new(tempsrc.create_gzip_layer(None)?); + + // Process the tar stream and inject our new detached metadata + crate::tar::update_detached_metadata( + &mut src_layer, + &mut out_layer, + detached_buf.as_deref(), + Some(cancellable), + )?; + + // Flush all wrappers, and finalize the layer + out_layer + .into_inner() + .map_err(|_| anyhow!("Failed to flush buffer"))? + .complete()? + }; + // Get the diffid and descriptor for our new tar layer + let out_layer_diffid = format!("sha256:{}", out_layer.uncompressed_sha256.digest()); + let out_layer_descriptor = out_layer + .descriptor() + .media_type(oci_image::MediaType::ImageLayerGzip) + .build() + .unwrap(); // SAFETY: We pass all required fields + + // Splice it into both the manifest and config + manifest.layers_mut()[commit_layer_idx] = out_layer_descriptor; + config.rootfs_mut().diff_ids_mut()[commit_layer_idx].clone_from(&out_layer_diffid); + + let labels = ctrcfg.labels_mut().get_or_insert_with(Default::default); + // Nothing to do except in the special case where there's somehow only one + // chunked layer. + if manifest.layers().len() == 1 { + labels.insert(DIFFID_LABEL.into(), out_layer_diffid); + } + config.set_config(Some(ctrcfg)); + + // Write the config and manifest + let new_config_descriptor = tempsrc.write_config(config)?; + manifest.set_config(new_config_descriptor); + // This entirely replaces the single entry in the OCI directory, which skopeo will find by default. + tempsrc + .replace_with_single_manifest(manifest, platform) + .context("Writing manifest")?; + Ok(()) + }) + .await + .context("Regenerating commit layer")?; + + // Finally, copy the mutated image back to the target. For chunked images, + // because we only changed one layer, skopeo should know not to re-upload shared blobs. + crate::container::skopeo::copy(&tempsrc_ref, dest, None, None, false) + .await + .context("Copying to destination") +} diff --git a/ostree-ext/src/container_utils.rs b/ostree-ext/src/container_utils.rs new file mode 100644 index 00000000..f4c7ed93 --- /dev/null +++ b/ostree-ext/src/container_utils.rs @@ -0,0 +1,89 @@ +//! Helpers for interacting with containers at runtime. + +use crate::keyfileext::KeyFileExt; +use anyhow::Result; +use ostree::glib; +use std::io::Read; +use std::path::Path; + +// See https://github.com/coreos/rpm-ostree/pull/3285#issuecomment-999101477 +// For compatibility with older ostree, we stick this in /sysroot where +// it will be ignored. +const V0_REPO_CONFIG: &str = "/sysroot/config"; +const V1_REPO_CONFIG: &str = "/sysroot/ostree/repo/config"; + +/// Attempts to detect if the current process is running inside a container. +/// This looks for the `container` environment variable or the presence +/// of Docker or podman's more generic `/run/.containerenv`. +/// This is a best-effort function, as there is not a 100% reliable way +/// to determine this. +pub fn running_in_container() -> bool { + if std::env::var_os("container").is_some() { + return true; + } + // https://stackoverflow.com/questions/20010199/how-to-determine-if-a-process-runs-inside-lxc-docker + for p in ["/run/.containerenv", "/.dockerenv"] { + if std::path::Path::new(p).exists() { + return true; + } + } + false +} + +// https://docs.rs/openat-ext/0.1.10/openat_ext/trait.OpenatDirExt.html#tymethod.open_file_optional +// https://users.rust-lang.org/t/why-i-use-anyhow-error-even-in-libraries/68592 +pub(crate) fn open_optional(path: impl AsRef) -> std::io::Result> { + match std::fs::File::open(path.as_ref()) { + Ok(r) => Ok(Some(r)), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(e) => Err(e), + } +} + +/// Returns `true` if the current root filesystem has an ostree repository in `bare-split-xattrs` mode. +/// This will be the case in a running ostree-native container. +pub fn is_bare_split_xattrs() -> Result { + if let Some(configf) = open_optional(V1_REPO_CONFIG) + .transpose() + .or_else(|| open_optional(V0_REPO_CONFIG).transpose()) + { + let configf = configf?; + let mut bufr = std::io::BufReader::new(configf); + let mut s = String::new(); + bufr.read_to_string(&mut s)?; + let kf = glib::KeyFile::new(); + kf.load_from_data(&s, glib::KeyFileFlags::NONE)?; + let r = if let Some(mode) = kf.optional_string("core", "mode")? { + mode == crate::tar::BARE_SPLIT_XATTRS_MODE + } else { + false + }; + Ok(r) + } else { + Ok(false) + } +} + +/// Returns `true` if the current booted filesystem appears to be an ostree-native container. +/// +/// This just invokes [`is_bare_split_xattrs`] and [`running_in_container`]. +pub fn is_ostree_container() -> Result { + let is_container_ostree = is_bare_split_xattrs()?; + let running_in_systemd = std::env::var_os("INVOCATION_ID").is_some(); + // If we have a container-ostree repo format, then we'll assume we're + // running in a container unless there's strong evidence not (we detect + // we're part of a systemd unit or are in a booted ostree system). + let maybe_container = running_in_container() + || (!running_in_systemd && !Path::new("/run/ostree-booted").exists()); + Ok(is_container_ostree && maybe_container) +} + +/// Returns an error unless the current filesystem is an ostree-based container +/// +/// This just wraps [`is_ostree_container`]. +pub fn require_ostree_container() -> Result<()> { + if !is_ostree_container()? { + anyhow::bail!("Not in an ostree-based container environment"); + } + Ok(()) +} diff --git a/ostree-ext/src/diff.rs b/ostree-ext/src/diff.rs new file mode 100644 index 00000000..655adc38 --- /dev/null +++ b/ostree-ext/src/diff.rs @@ -0,0 +1,181 @@ +//! Compute the difference between two OSTree commits. + +/* + * Copyright (C) 2020 Red Hat, Inc. + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use anyhow::{Context, Result}; +use fn_error_context::context; +use gio::prelude::*; +use ostree::gio; +use std::collections::BTreeSet; +use std::fmt; + +/// Like `g_file_query_info()`, but return None if the target doesn't exist. +pub(crate) fn query_info_optional( + f: &gio::File, + queryattrs: &str, + queryflags: gio::FileQueryInfoFlags, +) -> Result> { + let cancellable = gio::Cancellable::NONE; + match f.query_info(queryattrs, queryflags, cancellable) { + Ok(i) => Ok(Some(i)), + Err(e) => { + if let Some(ref e2) = e.kind::() { + match e2 { + gio::IOErrorEnum::NotFound => Ok(None), + _ => Err(e.into()), + } + } else { + Err(e.into()) + } + } + } +} + +/// A set of file paths. +pub type FileSet = BTreeSet; + +/// Diff between two ostree commits. +#[derive(Debug, Default)] +pub struct FileTreeDiff { + /// The prefix passed for diffing, e.g. /usr + pub subdir: Option, + /// Files that are new in an existing directory + pub added_files: FileSet, + /// New directories + pub added_dirs: FileSet, + /// Files removed + pub removed_files: FileSet, + /// Directories removed (recursively) + pub removed_dirs: FileSet, + /// Files that changed (in any way, metadata or content) + pub changed_files: FileSet, + /// Directories that changed mode/permissions + pub changed_dirs: FileSet, +} + +impl fmt::Display for FileTreeDiff { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "files(added:{} removed:{} changed:{}) dirs(added:{} removed:{} changed:{})", + self.added_files.len(), + self.removed_files.len(), + self.changed_files.len(), + self.added_dirs.len(), + self.removed_dirs.len(), + self.changed_dirs.len() + ) + } +} + +fn diff_recurse( + prefix: &str, + diff: &mut FileTreeDiff, + from: &ostree::RepoFile, + to: &ostree::RepoFile, +) -> Result<()> { + let cancellable = gio::Cancellable::NONE; + let queryattrs = "standard::name,standard::type"; + let queryflags = gio::FileQueryInfoFlags::NOFOLLOW_SYMLINKS; + let from_iter = from.enumerate_children(queryattrs, queryflags, cancellable)?; + + // Iterate over the source (from) directory, and compare with the + // target (to) directory. This generates removals and changes. + while let Some(from_info) = from_iter.next_file(cancellable)? { + let from_child = from_iter.child(&from_info); + let name = from_info.name(); + let name = name.to_str().expect("UTF-8 ostree name"); + let path = format!("{prefix}{name}"); + let to_child = to.child(name); + let to_info = query_info_optional(&to_child, queryattrs, queryflags) + .context("querying optional to")?; + let is_dir = matches!(from_info.file_type(), gio::FileType::Directory); + if to_info.is_some() { + let to_child = to_child.downcast::().expect("downcast"); + to_child.ensure_resolved()?; + let from_child = from_child.downcast::().expect("downcast"); + from_child.ensure_resolved()?; + + if is_dir { + let from_contents_checksum = from_child.tree_get_contents_checksum(); + let to_contents_checksum = to_child.tree_get_contents_checksum(); + if from_contents_checksum != to_contents_checksum { + let subpath = format!("{}/", path); + diff_recurse(&subpath, diff, &from_child, &to_child)?; + } + let from_meta_checksum = from_child.tree_get_metadata_checksum(); + let to_meta_checksum = to_child.tree_get_metadata_checksum(); + if from_meta_checksum != to_meta_checksum { + diff.changed_dirs.insert(path); + } + } else { + let from_checksum = from_child.checksum(); + let to_checksum = to_child.checksum(); + if from_checksum != to_checksum { + diff.changed_files.insert(path); + } + } + } else if is_dir { + diff.removed_dirs.insert(path); + } else { + diff.removed_files.insert(path); + } + } + // Iterate over the target (to) directory, and find any + // files/directories which were not present in the source. + let to_iter = to.enumerate_children(queryattrs, queryflags, cancellable)?; + while let Some(to_info) = to_iter.next_file(cancellable)? { + let name = to_info.name(); + let name = name.to_str().expect("UTF-8 ostree name"); + let path = format!("{prefix}{name}"); + let from_child = from.child(name); + let from_info = query_info_optional(&from_child, queryattrs, queryflags) + .context("querying optional from")?; + if from_info.is_some() { + continue; + } + let is_dir = matches!(to_info.file_type(), gio::FileType::Directory); + if is_dir { + diff.added_dirs.insert(path); + } else { + diff.added_files.insert(path); + } + } + Ok(()) +} + +/// Given two ostree commits, compute the diff between them. +#[context("Computing ostree diff")] +pub fn diff>( + repo: &ostree::Repo, + from: &str, + to: &str, + subdir: Option

, +) -> Result { + let subdir = subdir.as_ref(); + let subdir = subdir.map(|s| s.as_ref()); + let (fromroot, _) = repo.read_commit(from, gio::Cancellable::NONE)?; + let (toroot, _) = repo.read_commit(to, gio::Cancellable::NONE)?; + let (fromroot, toroot) = if let Some(subdir) = subdir { + ( + fromroot.resolve_relative_path(subdir), + toroot.resolve_relative_path(subdir), + ) + } else { + (fromroot, toroot) + }; + let fromroot = fromroot.downcast::().expect("downcast"); + fromroot.ensure_resolved()?; + let toroot = toroot.downcast::().expect("downcast"); + toroot.ensure_resolved()?; + let mut diff = FileTreeDiff { + subdir: subdir.map(|s| s.to_string()), + ..Default::default() + }; + diff_recurse("/", &mut diff, &fromroot, &toroot)?; + Ok(diff) +} diff --git a/ostree-ext/src/docgen.rs b/ostree-ext/src/docgen.rs new file mode 100644 index 00000000..0e2d12df --- /dev/null +++ b/ostree-ext/src/docgen.rs @@ -0,0 +1,46 @@ +// Copyright 2022 Red Hat, Inc. +// +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use anyhow::{Context, Result}; +use camino::Utf8Path; +use clap::{Command, CommandFactory}; +use std::fs::OpenOptions; +use std::io::Write; + +pub fn generate_manpages(directory: &Utf8Path) -> Result<()> { + generate_one(directory, crate::cli::Opt::command()) +} + +fn generate_one(directory: &Utf8Path, cmd: Command) -> Result<()> { + let version = env!("CARGO_PKG_VERSION"); + let name = cmd.get_name(); + let path = directory.join(format!("{name}.8")); + println!("Generating {path}..."); + + let mut out = OpenOptions::new() + .create(true) + .write(true) + .truncate(true) + .open(&path) + .with_context(|| format!("opening {path}")) + .map(std::io::BufWriter::new)?; + clap_mangen::Man::new(cmd.clone()) + .title("ostree-ext") + .section("8") + .source(format!("ostree-ext {version}")) + .render(&mut out) + .with_context(|| format!("rendering {name}.8"))?; + out.flush().context("flushing man page")?; + drop(out); + + for subcmd in cmd.get_subcommands().filter(|c| !c.is_hide_set()) { + let subname = format!("{}-{}", name, subcmd.get_name()); + // SAFETY: Latest clap 4 requires names are &'static - this is + // not long-running production code, so we just leak the names here. + let subname = &*std::boxed::Box::leak(subname.into_boxed_str()); + let subcmd = subcmd.clone().name(subname).alias(subname).version(version); + generate_one(directory, subcmd)?; + } + Ok(()) +} diff --git a/ostree-ext/src/fixture.rs b/ostree-ext/src/fixture.rs new file mode 100644 index 00000000..99a2e384 --- /dev/null +++ b/ostree-ext/src/fixture.rs @@ -0,0 +1,866 @@ +//! Test suite fixture. Should only be used by this library. + +#![allow(missing_docs)] + +use crate::chunking::ObjectMetaSized; +use crate::container::store::{self, LayeredImageState}; +use crate::container::{Config, ExportOpts, ImageReference, Transport}; +use crate::objectsource::{ObjectMeta, ObjectSourceMeta}; +use crate::objgv::gv_dirtree; +use crate::prelude::*; +use crate::{gio, glib}; +use anyhow::{anyhow, Context, Result}; +use camino::{Utf8Component, Utf8Path, Utf8PathBuf}; +use cap_std::fs::Dir; +use cap_std_ext::cap_std; +use cap_std_ext::prelude::CapStdExtCommandExt; +use chrono::TimeZone; +use containers_image_proxy::oci_spec::image as oci_image; +use fn_error_context::context; +use gvariant::aligned_bytes::TryAsAligned; +use gvariant::{Marker, Structure}; +use io_lifetimes::AsFd; +use ocidir::cap_std::fs::{DirBuilder, DirBuilderExt as _}; +use once_cell::sync::Lazy; +use regex::Regex; +use std::borrow::Cow; +use std::fmt::Write as _; +use std::io::Write; +use std::ops::Add; +use std::process::{Command, Stdio}; +use std::rc::Rc; +use std::sync::Arc; +use tempfile::TempDir; + +const OSTREE_GPG_HOME: &[u8] = include_bytes!("fixtures/ostree-gpg-test-home.tar.gz"); +const TEST_GPG_KEYID_1: &str = "7FCA23D8472CDAFA"; +#[allow(dead_code)] +const TEST_GPG_KEYFPR_1: &str = "5E65DE75AB1C501862D476347FCA23D8472CDAFA"; +const TESTREF: &str = "exampleos/x86_64/stable"; + +#[derive(Debug)] +enum FileDefType { + Regular(Cow<'static, str>), + Symlink(Cow<'static, Utf8Path>), + Directory, +} + +#[derive(Debug)] +pub struct FileDef { + uid: u32, + gid: u32, + mode: u32, + path: Cow<'static, Utf8Path>, + ty: FileDefType, +} + +impl TryFrom<&'static str> for FileDef { + type Error = anyhow::Error; + + fn try_from(value: &'static str) -> Result { + let mut parts = value.split(' '); + let tydef = parts + .next() + .ok_or_else(|| anyhow!("Missing type definition"))?; + let name = parts.next().ok_or_else(|| anyhow!("Missing file name"))?; + let contents = parts.next(); + let contents = move || contents.ok_or_else(|| anyhow!("Missing file contents: {}", value)); + if parts.next().is_some() { + anyhow::bail!("Invalid filedef: {}", value); + } + let ty = match tydef { + "r" => FileDefType::Regular(contents()?.into()), + "l" => FileDefType::Symlink(Cow::Borrowed(contents()?.into())), + "d" => FileDefType::Directory, + _ => anyhow::bail!("Invalid filedef type: {}", value), + }; + Ok(FileDef { + uid: 0, + gid: 0, + mode: 0o644, + path: Cow::Borrowed(name.into()), + ty, + }) + } +} + +fn parse_mode(line: &str) -> Result<(u32, u32, u32)> { + let mut parts = line.split(' ').skip(1); + // An empty mode resets to defaults + let uid = if let Some(u) = parts.next() { + u + } else { + return Ok((0, 0, 0o644)); + }; + let gid = parts.next().ok_or_else(|| anyhow!("Missing gid"))?; + let mode = parts.next().ok_or_else(|| anyhow!("Missing mode"))?; + if parts.next().is_some() { + anyhow::bail!("Invalid mode: {}", line); + } + Ok((uid.parse()?, gid.parse()?, u32::from_str_radix(mode, 8)?)) +} + +impl FileDef { + /// Parse a list of newline-separated file definitions. + pub fn iter_from(defs: &'static str) -> impl Iterator> { + let mut uid = 0; + let mut gid = 0; + let mut mode = 0o644; + defs.lines() + .filter(|v| !(v.is_empty() || v.starts_with('#'))) + .filter_map(move |line| { + if line.starts_with('m') { + match parse_mode(line) { + Ok(r) => { + uid = r.0; + gid = r.1; + mode = r.2; + None + } + Err(e) => Some(Err(e)), + } + } else { + Some(FileDef::try_from(line).map(|mut def| { + def.uid = uid; + def.gid = gid; + def.mode = mode; + def + })) + } + }) + } +} + +/// This is like a package database, mapping our test fixture files to package names +static OWNERS: Lazy> = Lazy::new(|| { + [ + ("usr/lib/modules/.*/initramfs", "initramfs"), + ("usr/lib/modules", "kernel"), + ("usr/bin/(ba)?sh", "bash"), + ("usr/lib.*/emptyfile.*", "bash"), + ("usr/bin/hardlink.*", "testlink"), + ("usr/etc/someconfig.conf", "someconfig"), + ("usr/etc/polkit.conf", "a-polkit-config"), + ("opt", "filesystem"), + ("usr/lib/pkgdb", "pkgdb"), + ("usr/lib/sysimage/pkgdb", "pkgdb"), + ] + .iter() + .map(|(k, v)| (Regex::new(k).unwrap(), *v)) + .collect() +}); + +pub static CONTENTS_V0: &str = indoc::indoc! { r##" +r usr/lib/modules/5.10.18-200.x86_64/vmlinuz this-is-a-kernel +r usr/lib/modules/5.10.18-200.x86_64/initramfs this-is-an-initramfs +m 0 0 755 +r usr/bin/bash the-bash-shell +l usr/bin/sh bash +m 0 0 644 +# Some empty files +r usr/lib/emptyfile +r usr/lib64/emptyfile2 +# Should be the same object +r usr/bin/hardlink-a testlink +r usr/bin/hardlink-b testlink +r usr/etc/someconfig.conf someconfig +m 10 10 644 +r usr/etc/polkit.conf a-polkit-config +m 0 0 644 +# See https://github.com/coreos/fedora-coreos-tracker/issues/1258 +r usr/lib/sysimage/pkgdb some-package-database +r usr/lib/pkgdb/pkgdb some-package-database +m +d boot +d run +l opt var/opt +m 0 0 1755 +d tmp +"## }; +pub const CONTENTS_CHECKSUM_V0: &str = + "acc42fb5c796033f034941dc688643bf8beddfd9068d87165344d2b99906220a"; +// 1 for ostree commit, 2 for max frequency packages, 3 as empty layer +pub const LAYERS_V0_LEN: usize = 3usize; +pub const PKGS_V0_LEN: usize = 7usize; + +#[derive(Debug, PartialEq, Eq)] +enum SeLabel { + Root, + Usr, + UsrLibSystemd, + Boot, + Etc, + EtcSystemConf, +} + +impl SeLabel { + pub fn from_path(p: &Utf8Path) -> Self { + let rootdir = p.components().find_map(|v| { + if let Utf8Component::Normal(name) = v { + Some(name) + } else { + None + } + }); + let rootdir = if let Some(r) = rootdir { + r + } else { + return SeLabel::Root; + }; + if rootdir == "usr" { + if p.as_str().contains("systemd") { + SeLabel::UsrLibSystemd + } else { + SeLabel::Usr + } + } else if rootdir == "boot" { + SeLabel::Boot + } else if rootdir == "etc" { + if p.as_str().len() % 2 == 0 { + SeLabel::Etc + } else { + SeLabel::EtcSystemConf + } + } else { + SeLabel::Usr + } + } + + pub fn to_str(&self) -> &'static str { + match self { + SeLabel::Root => "system_u:object_r:root_t:s0", + SeLabel::Usr => "system_u:object_r:usr_t:s0", + SeLabel::UsrLibSystemd => "system_u:object_r:systemd_unit_file_t:s0", + SeLabel::Boot => "system_u:object_r:boot_t:s0", + SeLabel::Etc => "system_u:object_r:etc_t:s0", + SeLabel::EtcSystemConf => "system_u:object_r:system_conf_t:s0", + } + } + + pub fn xattrs(&self) -> Vec<(&[u8], &[u8])> { + vec![(b"security.selinux\0", self.to_str().as_bytes())] + } + + pub fn new_xattrs(&self) -> glib::Variant { + self.xattrs().to_variant() + } +} + +/// Generate directory metadata variant for root/root 0755 directory with an optional SELinux label +pub fn create_dirmeta(path: &Utf8Path, selinux: bool) -> glib::Variant { + let finfo = gio::FileInfo::new(); + finfo.set_attribute_uint32("unix::uid", 0); + finfo.set_attribute_uint32("unix::gid", 0); + finfo.set_attribute_uint32("unix::mode", libc::S_IFDIR | 0o755); + let label = if selinux { + Some(SeLabel::from_path(path)) + } else { + None + }; + let xattrs = label.map(|v| v.new_xattrs()); + ostree::create_directory_metadata(&finfo, xattrs.as_ref()) +} + +/// Wraps [`create_dirmeta`] and commits it. +#[context("Init dirmeta for {path}")] +pub fn require_dirmeta(repo: &ostree::Repo, path: &Utf8Path, selinux: bool) -> Result { + let v = create_dirmeta(path, selinux); + ostree::validate_structureof_dirmeta(&v).context("Validating dirmeta")?; + let r = repo.write_metadata( + ostree::ObjectType::DirMeta, + None, + &v, + gio::Cancellable::NONE, + )?; + Ok(r.to_hex()) +} + +fn ensure_parent_dirs( + mt: &ostree::MutableTree, + path: &Utf8Path, + metadata_checksum: &str, +) -> Result { + let parts = relative_path_components(path) + .map(|s| s.as_str()) + .collect::>(); + mt.ensure_parent_dirs(&parts, metadata_checksum) + .map_err(Into::into) +} + +fn relative_path_components(p: &Utf8Path) -> impl Iterator { + p.components() + .filter(|p| matches!(p, Utf8Component::Normal(_))) +} + +/// Walk over the whole filesystem, and generate mappings from content object checksums +/// to the package that owns them. +/// +/// In the future, we could compute this much more efficiently by walking that +/// instead. But this design is currently oriented towards accepting a single ostree +/// commit as input. +fn build_mapping_recurse( + path: &mut Utf8PathBuf, + dir: &gio::File, + ret: &mut ObjectMeta, +) -> Result<()> { + use indexmap::map::Entry; + let cancellable = gio::Cancellable::NONE; + let e = dir.enumerate_children( + "standard::name,standard::type", + gio::FileQueryInfoFlags::NOFOLLOW_SYMLINKS, + cancellable, + )?; + for child in e { + let childi = child?; + let name: Utf8PathBuf = childi.name().try_into()?; + let child = dir.child(&name); + path.push(&name); + match childi.file_type() { + gio::FileType::Regular | gio::FileType::SymbolicLink => { + let child = child.downcast::().unwrap(); + + let owner = OWNERS + .iter() + .find_map(|(r, owner)| { + if r.is_match(path.as_str()) { + Some(Rc::from(*owner)) + } else { + None + } + }) + .ok_or_else(|| anyhow!("Unowned path {}", path))?; + + if !ret.set.contains(&*owner) { + ret.set.insert(ObjectSourceMeta { + identifier: Rc::clone(&owner), + name: Rc::clone(&owner), + srcid: Rc::clone(&owner), + change_time_offset: u32::MAX, + change_frequency: u32::MAX, + }); + } + + let checksum = child.checksum().to_string(); + match ret.map.entry(checksum) { + Entry::Vacant(v) => { + v.insert(owner); + } + Entry::Occupied(v) => { + let prev_owner = v.get(); + if **prev_owner != *owner { + anyhow::bail!( + "Duplicate object ownership {} ({} and {})", + path.as_str(), + prev_owner, + owner + ); + } + } + } + } + gio::FileType::Directory => { + build_mapping_recurse(path, &child, ret)?; + } + o => anyhow::bail!("Unhandled file type: {}", o), + } + path.pop(); + } + Ok(()) +} + +/// Thin wrapper for `ostree ls -RXC` to show the full file contents +pub fn recursive_ostree_ls_text(repo: &ostree::Repo, refspec: &str) -> Result { + let o = Command::new("ostree") + .cwd_dir(Dir::reopen_dir(&repo.dfd_borrow())?) + .args(["--repo=.", "ls", "-RXC", refspec]) + .output()?; + let st = o.status; + if !st.success() { + anyhow::bail!("ostree ls failed: {st:?}"); + } + let r = String::from_utf8(o.stdout)?; + Ok(r) +} + +pub fn assert_commits_content_equal( + a_repo: &ostree::Repo, + a: &str, + b_repo: &ostree::Repo, + b: &str, +) { + let a = a_repo.require_rev(a).unwrap(); + let b = a_repo.require_rev(b).unwrap(); + let a_commit = a_repo.load_commit(&a).unwrap().0; + let b_commit = b_repo.load_commit(&b).unwrap().0; + let a_contentid = ostree::commit_get_content_checksum(&a_commit).unwrap(); + let b_contentid = ostree::commit_get_content_checksum(&b_commit).unwrap(); + if a_contentid == b_contentid { + return; + } + let a_contents = recursive_ostree_ls_text(a_repo, &a).unwrap(); + let b_contents = recursive_ostree_ls_text(b_repo, &b).unwrap(); + similar_asserts::assert_eq!(a_contents, b_contents); + panic!("Should not be reached; had different content hashes but same recursive ls") +} + +fn ls_recurse( + repo: &ostree::Repo, + path: &mut Utf8PathBuf, + buf: &mut String, + dt: &glib::Variant, +) -> Result<()> { + let dt = dt.data_as_bytes(); + let dt = dt.try_as_aligned()?; + let dt = gv_dirtree!().cast(dt); + let (files, dirs) = dt.to_tuple(); + // A reusable buffer to avoid heap allocating these + let mut hexbuf = [0u8; 64]; + for file in files { + let (name, csum) = file.to_tuple(); + path.push(name.to_str()); + hex::encode_to_slice(csum, &mut hexbuf)?; + let checksum = std::str::from_utf8(&hexbuf)?; + let meta = repo.query_file(checksum, gio::Cancellable::NONE)?.0; + let size = meta.size() as u64; + writeln!(buf, "r {path} {size}").unwrap(); + assert!(path.pop()); + } + for item in dirs { + let (name, contents_csum, _) = item.to_tuple(); + let name = name.to_str(); + // Extend our current path + path.push(name); + hex::encode_to_slice(contents_csum, &mut hexbuf)?; + let checksum_s = std::str::from_utf8(&hexbuf)?; + let child_v = repo.load_variant(ostree::ObjectType::DirTree, checksum_s)?; + ls_recurse(repo, path, buf, &child_v)?; + // We did a push above, so pop must succeed. + assert!(path.pop()); + } + Ok(()) +} + +pub fn ostree_ls(repo: &ostree::Repo, r: &str) -> Result { + let root = repo.read_commit(r, gio::Cancellable::NONE).unwrap().0; + // SAFETY: Must be a repofile + let root = root.downcast_ref::().unwrap(); + // SAFETY: must be a tree root + let root_contents = root.tree_get_contents_checksum().unwrap(); + let root_contents = repo + .load_variant(ostree::ObjectType::DirTree, &root_contents) + .unwrap(); + + let mut contents_buf = String::new(); + let mut pathbuf = Utf8PathBuf::from("/"); + ls_recurse(repo, &mut pathbuf, &mut contents_buf, &root_contents)?; + Ok(contents_buf) +} + +/// Verify the filenames (but not metadata) are the same between two commits. +/// We unfortunately need to do this because the current commit merge path +/// sets ownership of directories to the current user, which breaks in unit tests. +pub fn assert_commits_filenames_equal( + a_repo: &ostree::Repo, + a: &str, + b_repo: &ostree::Repo, + b: &str, +) { + let a_contents_buf = ostree_ls(a_repo, a).unwrap(); + let b_contents_buf = ostree_ls(b_repo, b).unwrap(); + similar_asserts::assert_eq!(a_contents_buf, b_contents_buf); +} + +#[derive(Debug)] +pub struct Fixture { + // Just holds a reference + tempdir: tempfile::TempDir, + pub dir: Arc

, + pub path: Utf8PathBuf, + srcrepo: ostree::Repo, + destrepo: ostree::Repo, + + pub selinux: bool, + pub bootable: bool, +} + +impl Fixture { + #[context("Initializing fixture")] + pub fn new_base() -> Result { + // Basic setup, allocate a tempdir + let tempdir = tempfile::tempdir_in("/var/tmp")?; + let dir = Arc::new(cap_std::fs::Dir::open_ambient_dir( + tempdir.path(), + cap_std::ambient_authority(), + )?); + let path: &Utf8Path = tempdir.path().try_into().unwrap(); + let path = path.to_path_buf(); + + // Create the src/ directory + dir.create_dir("src")?; + let srcdir_dfd = &dir.open_dir("src")?; + + // Initialize the src/gpghome/ directory + let gpgtarname = "gpghome.tgz"; + srcdir_dfd.write(gpgtarname, OSTREE_GPG_HOME)?; + let gpgtar = srcdir_dfd.open(gpgtarname)?; + srcdir_dfd.remove_file(gpgtarname)?; + srcdir_dfd.create_dir("gpghome")?; + let gpghome = srcdir_dfd.open_dir("gpghome")?; + let st = std::process::Command::new("tar") + .cwd_dir(gpghome) + .stdin(Stdio::from(gpgtar)) + .args(["-azxf", "-"]) + .status()?; + assert!(st.success()); + + let srcrepo = ostree::Repo::create_at_dir( + srcdir_dfd.as_fd(), + "repo", + ostree::RepoMode::Archive, + None, + ) + .context("Creating src/ repo")?; + + dir.create_dir("dest")?; + let destrepo = ostree::Repo::create_at_dir( + dir.as_fd(), + "dest/repo", + ostree::RepoMode::BareUser, + None, + )?; + Ok(Self { + tempdir, + dir, + path, + srcrepo, + destrepo, + selinux: true, + bootable: true, + }) + } + + pub fn srcrepo(&self) -> &ostree::Repo { + &self.srcrepo + } + + pub fn destrepo(&self) -> &ostree::Repo { + &self.destrepo + } + + pub fn new_shell(&self) -> Result { + let sh = xshell::Shell::new()?; + sh.change_dir(&self.path); + Ok(sh) + } + + /// Given the input image reference, import it into destrepo using the default + /// import config. The image must not exist already in the store. + pub async fn must_import(&self, imgref: &ImageReference) -> Result> { + let ostree_imgref = crate::container::OstreeImageReference { + sigverify: crate::container::SignatureSource::ContainerPolicyAllowInsecure, + imgref: imgref.clone(), + }; + let mut imp = + store::ImageImporter::new(self.destrepo(), &ostree_imgref, Default::default()) + .await + .unwrap(); + assert!(store::query_image(self.destrepo(), &imgref) + .unwrap() + .is_none()); + let prep = match imp.prepare().await.context("Init prep derived")? { + store::PrepareResult::AlreadyPresent(_) => panic!("should not be already imported"), + store::PrepareResult::Ready(r) => r, + }; + imp.import(prep).await + } + + // Delete all objects in the destrepo + pub fn clear_destrepo(&self) -> Result<()> { + self.destrepo() + .set_ref_immediate(None, self.testref(), None, gio::Cancellable::NONE)?; + for (r, _) in self.destrepo().list_refs(None, gio::Cancellable::NONE)? { + self.destrepo() + .set_ref_immediate(None, &r, None, gio::Cancellable::NONE)?; + } + self.destrepo() + .prune(ostree::RepoPruneFlags::REFS_ONLY, 0, gio::Cancellable::NONE)?; + Ok(()) + } + + #[context("Writing filedef {}", def.path.as_str())] + pub fn write_filedef(&self, root: &ostree::MutableTree, def: &FileDef) -> Result<()> { + let parent_path = def.path.parent(); + let parent = if let Some(parent_path) = parent_path { + let meta = require_dirmeta(&self.srcrepo, parent_path, self.selinux)?; + Some(ensure_parent_dirs(root, &def.path, meta.as_str())?) + } else { + None + }; + let parent = parent.as_ref().unwrap_or(root); + let name = def.path.file_name().expect("file name"); + let label = if self.selinux { + Some(SeLabel::from_path(&def.path)) + } else { + None + }; + let xattrs = label.map(|v| v.new_xattrs()); + let xattrs = xattrs.as_ref(); + let checksum = match &def.ty { + FileDefType::Regular(contents) => self + .srcrepo + .write_regfile_inline( + None, + 0, + 0, + libc::S_IFREG | def.mode, + xattrs, + contents.as_bytes(), + gio::Cancellable::NONE, + ) + .context("Writing regfile inline")?, + FileDefType::Symlink(target) => self.srcrepo.write_symlink( + None, + def.uid, + def.gid, + xattrs, + target.as_str(), + gio::Cancellable::NONE, + )?, + FileDefType::Directory => { + let d = parent.ensure_dir(name)?; + let meta = require_dirmeta(&self.srcrepo, &def.path, self.selinux)?; + d.set_metadata_checksum(meta.as_str()); + return Ok(()); + } + }; + parent + .replace_file(name, checksum.as_str()) + .context("Setting file")?; + Ok(()) + } + + pub fn commit_filedefs(&self, defs: impl IntoIterator>) -> Result<()> { + let root = ostree::MutableTree::new(); + let cancellable = gio::Cancellable::NONE; + let tx = self.srcrepo.auto_transaction(cancellable)?; + for def in defs { + let def = def?; + self.write_filedef(&root, &def)?; + } + let root = self.srcrepo.write_mtree(&root, cancellable)?; + let root = root.downcast_ref::().unwrap(); + // You win internet points if you understand this date reference + let ts = chrono::DateTime::parse_from_rfc2822("Fri, 29 Aug 1997 10:30:42 PST")?.timestamp(); + // Some default metadata fixtures + let metadata = glib::VariantDict::new(None); + metadata.insert( + "buildsys.checksum", + &"41af286dc0b172ed2f1ca934fd2278de4a1192302ffa07087cea2682e7d372e3", + ); + metadata.insert("ostree.container-cmd", &vec!["/usr/bin/bash"]); + metadata.insert("version", &"42.0"); + #[allow(clippy::explicit_auto_deref)] + if self.bootable { + metadata.insert(*ostree::METADATA_KEY_BOOTABLE, &true); + } + let metadata = metadata.to_variant(); + let commit = self.srcrepo.write_commit_with_time( + None, + None, + None, + Some(&metadata), + root, + ts as u64, + cancellable, + )?; + self.srcrepo + .transaction_set_ref(None, self.testref(), Some(commit.as_str())); + tx.commit(cancellable)?; + + // Add detached metadata so we can verify it makes it through + let detached = glib::VariantDict::new(None); + detached.insert("my-detached-key", &"my-detached-value"); + let detached = detached.to_variant(); + self.srcrepo.write_commit_detached_metadata( + commit.as_str(), + Some(&detached), + gio::Cancellable::NONE, + )?; + + let gpghome = self.path.join("src/gpghome"); + self.srcrepo.sign_commit( + &commit, + TEST_GPG_KEYID_1, + Some(gpghome.as_str()), + gio::Cancellable::NONE, + )?; + + Ok(()) + } + + pub fn new_v1() -> Result { + let r = Self::new_base()?; + r.commit_filedefs(FileDef::iter_from(CONTENTS_V0))?; + Ok(r) + } + + pub fn testref(&self) -> &'static str { + TESTREF + } + + #[context("Updating test repo")] + pub fn update( + &mut self, + additions: impl Iterator>, + removals: impl Iterator>, + ) -> Result<()> { + let cancellable = gio::Cancellable::NONE; + + // Load our base commit + let rev = &self.srcrepo().require_rev(self.testref())?; + let (commit, _) = self.srcrepo.load_commit(rev)?; + let metadata = commit.child_value(0); + let root = ostree::MutableTree::from_commit(self.srcrepo(), rev)?; + // Bump the commit timestamp by one day + let ts = chrono::Utc + .timestamp_opt(ostree::commit_get_timestamp(&commit) as i64, 0) + .single() + .unwrap(); + let new_ts = ts + .add(chrono::TimeDelta::try_days(1).expect("one day does not overflow")) + .timestamp() as u64; + + // Prepare a transaction + let tx = self.srcrepo.auto_transaction(cancellable)?; + for def in additions { + let def = def?; + self.write_filedef(&root, &def)?; + } + for removal in removals { + let filename = removal + .file_name() + .ok_or_else(|| anyhow!("Invalid path {}", removal))?; + // Notice that we're traversing the whole path, because that's how the walk() API works. + let p = relative_path_components(&removal); + let parts = p.map(|s| s.as_str()).collect::>(); + let parent = &root.walk(&parts, 0)?; + parent.remove(filename, false)?; + self.srcrepo.write_mtree(parent, cancellable)?; + } + let root = self + .srcrepo + .write_mtree(&root, cancellable) + .context("Writing mtree")?; + let root = root.downcast_ref::().unwrap(); + let commit = self + .srcrepo + .write_commit_with_time( + Some(rev), + None, + None, + Some(&metadata), + root, + new_ts, + cancellable, + ) + .context("Writing commit")?; + self.srcrepo + .transaction_set_ref(None, self.testref(), Some(commit.as_str())); + tx.commit(cancellable)?; + Ok(()) + } + + /// Gather object metadata for the current commit. + pub fn get_object_meta(&self) -> Result { + let cancellable = gio::Cancellable::NONE; + + // Load our base commit + let root = self.srcrepo.read_commit(self.testref(), cancellable)?.0; + + let mut ret = ObjectMeta::default(); + build_mapping_recurse(&mut Utf8PathBuf::from("/"), &root, &mut ret)?; + + Ok(ret) + } + + /// Unload all in-memory data, and return the underlying temporary directory without deleting it. + pub fn into_tempdir(self) -> tempfile::TempDir { + self.tempdir + } + + #[context("Exporting tar")] + pub fn export_tar(&self) -> Result<&'static Utf8Path> { + let cancellable = gio::Cancellable::NONE; + let (_, rev) = self.srcrepo.read_commit(self.testref(), cancellable)?; + let path = "exampleos-export.tar"; + let mut outf = std::io::BufWriter::new(self.dir.create(path)?); + #[allow(clippy::needless_update)] + let options = crate::tar::ExportOptions { + ..Default::default() + }; + crate::tar::export_commit(&self.srcrepo, rev.as_str(), &mut outf, Some(options))?; + outf.flush()?; + Ok(path.into()) + } + + /// Export the current ref as a container image. + /// This defaults to using chunking. + #[context("Exporting container")] + pub async fn export_container(&self) -> Result<(ImageReference, oci_image::Digest)> { + let name = "oci-v1"; + let container_path = &self.path.join(name); + if container_path.exists() { + std::fs::remove_dir_all(container_path)?; + } + let imgref = ImageReference { + transport: Transport::OciDir, + name: container_path.as_str().to_string(), + }; + let config = Config { + labels: Some( + [("foo", "bar"), ("test", "value")] + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(), + ), + ..Default::default() + }; + let contentmeta = self.get_object_meta().context("Computing object meta")?; + let contentmeta = ObjectMetaSized::compute_sizes(self.srcrepo(), contentmeta) + .context("Computing sizes")?; + let opts = ExportOpts { + max_layers: std::num::NonZeroU32::new(PKGS_V0_LEN as u32), + contentmeta: Some(&contentmeta), + ..Default::default() + }; + let digest = crate::container::encapsulate( + self.srcrepo(), + self.testref(), + &config, + Some(opts), + &imgref, + ) + .await + .context("exporting")?; + Ok((imgref, digest)) + } + + // Generate a directory with some test contents + #[context("Generating temp content")] + pub fn generate_test_derived_oci( + &self, + derived_path: impl AsRef, + tag: Option<&str>, + ) -> Result<()> { + let temproot = TempDir::new_in(&self.path)?; + let temprootd = Dir::open_ambient_dir(&temproot, cap_std::ambient_authority())?; + let mut db = DirBuilder::new(); + db.mode(0o755); + db.recursive(true); + temprootd.create_dir_with("usr/bin", &db)?; + temprootd.write("usr/bin/newderivedfile", "newderivedfile v0")?; + temprootd.write("usr/bin/newderivedfile3", "newderivedfile3 v0")?; + crate::integrationtest::generate_derived_oci(derived_path, temproot, tag)?; + Ok(()) + } +} diff --git a/ostree-ext/src/fixtures/fedora-coreos-contentmeta.json.gz b/ostree-ext/src/fixtures/fedora-coreos-contentmeta.json.gz new file mode 100644 index 00000000..285d587a Binary files /dev/null and b/ostree-ext/src/fixtures/fedora-coreos-contentmeta.json.gz differ diff --git a/ostree-ext/src/fixtures/ostree-gpg-test-home.tar.gz b/ostree-ext/src/fixtures/ostree-gpg-test-home.tar.gz new file mode 100644 index 00000000..1160f474 Binary files /dev/null and b/ostree-ext/src/fixtures/ostree-gpg-test-home.tar.gz differ diff --git a/ostree-ext/src/globals.rs b/ostree-ext/src/globals.rs new file mode 100644 index 00000000..ce9c53ca --- /dev/null +++ b/ostree-ext/src/globals.rs @@ -0,0 +1,184 @@ +//! Module containing access to global state. + +use super::Result; +use camino::{Utf8Path, Utf8PathBuf}; +use cap_std_ext::cap_std::fs::Dir; +use cap_std_ext::RootDir; +use once_cell::sync::OnceCell; +use ostree::glib; +use std::fs::File; + +struct ConfigPaths { + persistent: Utf8PathBuf, + runtime: Utf8PathBuf, + system: Option, +} + +/// Get the runtime and persistent config directories. In the system (root) case, these +/// system(root) case: /run/ostree /etc/ostree /usr/lib/ostree +/// user(nonroot) case: /run/user/$uid/ostree ~/.config/ostree +fn get_config_paths(root: bool) -> &'static ConfigPaths { + if root { + static PATHS_ROOT: OnceCell = OnceCell::new(); + PATHS_ROOT.get_or_init(|| ConfigPaths::new("etc", "run", Some("usr/lib"))) + } else { + static PATHS_USER: OnceCell = OnceCell::new(); + PATHS_USER.get_or_init(|| { + ConfigPaths::new( + Utf8PathBuf::try_from(glib::user_config_dir()).unwrap(), + Utf8PathBuf::try_from(glib::user_runtime_dir()).unwrap(), + None, + ) + }) + } +} + +impl ConfigPaths { + fn new>(persistent: P, runtime: P, system: Option

) -> Self { + fn relative_owned(p: &Utf8Path) -> Utf8PathBuf { + p.as_str().trim_start_matches('/').into() + } + let mut r = ConfigPaths { + persistent: relative_owned(persistent.as_ref()), + runtime: relative_owned(runtime.as_ref()), + system: system.as_ref().map(|s| relative_owned(s.as_ref())), + }; + let path = "ostree"; + r.persistent.push(path); + r.runtime.push(path); + if let Some(system) = r.system.as_mut() { + system.push(path); + } + r + } + + /// Return the path and an open fd for a config file, if it exists. + pub(crate) fn open_file( + &self, + root: &RootDir, + p: impl AsRef, + ) -> Result> { + let p = p.as_ref(); + let mut runtime = self.runtime.clone(); + runtime.push(p); + if let Some(f) = root.open_optional(&runtime)? { + return Ok(Some((runtime, f))); + } + let mut persistent = self.persistent.clone(); + persistent.push(p); + if let Some(f) = root.open_optional(&persistent)? { + return Ok(Some((persistent, f))); + } + if let Some(mut system) = self.system.clone() { + system.push(p); + if let Some(f) = root.open_optional(&system)? { + return Ok(Some((system, f))); + } + } + Ok(None) + } +} + +/// Return the path to the global container authentication file, if it exists. +pub fn get_global_authfile(root: &Dir) -> Result> { + let root = &RootDir::new(root, ".")?; + let am_uid0 = rustix::process::getuid() == rustix::process::Uid::ROOT; + get_global_authfile_impl(root, am_uid0) +} + +/// Return the path to the global container authentication file, if it exists. +fn get_global_authfile_impl(root: &RootDir, am_uid0: bool) -> Result> { + let paths = get_config_paths(am_uid0); + paths.open_file(root, "auth.json") +} + +#[cfg(test)] +mod tests { + use std::io::Read; + + use super::*; + use camino::Utf8PathBuf; + use cap_std_ext::{cap_std, cap_tempfile}; + + fn read_authfile( + root: &cap_std_ext::RootDir, + am_uid0: bool, + ) -> Result> { + let r = get_global_authfile_impl(root, am_uid0)?; + if let Some((path, mut f)) = r { + let mut s = String::new(); + f.read_to_string(&mut s)?; + Ok(Some((path.try_into()?, s))) + } else { + Ok(None) + } + } + + #[test] + fn test_config_paths() -> Result<()> { + let root = &cap_tempfile::TempDir::new(cap_std::ambient_authority())?; + let rootdir = &RootDir::new(root, ".")?; + assert!(read_authfile(rootdir, true).unwrap().is_none()); + root.create_dir_all("etc/ostree")?; + root.write("etc/ostree/auth.json", "etc ostree auth")?; + let (p, authdata) = read_authfile(rootdir, true).unwrap().unwrap(); + assert_eq!(p, "etc/ostree/auth.json"); + assert_eq!(authdata, "etc ostree auth"); + root.create_dir_all("usr/lib/ostree")?; + root.write("usr/lib/ostree/auth.json", "usrlib ostree auth")?; + // We should see /etc content still + let (p, authdata) = read_authfile(rootdir, true).unwrap().unwrap(); + assert_eq!(p, "etc/ostree/auth.json"); + assert_eq!(authdata, "etc ostree auth"); + // Now remove the /etc content, unveiling the /usr content + root.remove_file("etc/ostree/auth.json")?; + let (p, authdata) = read_authfile(rootdir, true).unwrap().unwrap(); + assert_eq!(p, "usr/lib/ostree/auth.json"); + assert_eq!(authdata, "usrlib ostree auth"); + + // Verify symlinks work, both relative... + root.create_dir_all("etc/containers")?; + root.write("etc/containers/auth.json", "etc containers ostree auth")?; + root.symlink_contents("../containers/auth.json", "etc/ostree/auth.json")?; + let (p, authdata) = read_authfile(rootdir, true).unwrap().unwrap(); + assert_eq!(p, "etc/ostree/auth.json"); + assert_eq!(authdata, "etc containers ostree auth"); + // And an absolute link + root.remove_file("etc/ostree/auth.json")?; + root.symlink_contents("/etc/containers/auth.json", "etc/ostree/auth.json")?; + assert_eq!(p, "etc/ostree/auth.json"); + assert_eq!(authdata, "etc containers ostree auth"); + + // Non-root + let mut user_runtime_dir = + Utf8Path::from_path(glib::user_runtime_dir().strip_prefix("/").unwrap()) + .unwrap() + .to_path_buf(); + user_runtime_dir.push("ostree"); + root.create_dir_all(&user_runtime_dir)?; + user_runtime_dir.push("auth.json"); + root.write(&user_runtime_dir, "usr_runtime_dir ostree auth")?; + + let mut user_config_dir = + Utf8Path::from_path(glib::user_config_dir().strip_prefix("/").unwrap()) + .unwrap() + .to_path_buf(); + user_config_dir.push("ostree"); + root.create_dir_all(&user_config_dir)?; + user_config_dir.push("auth.json"); + root.write(&user_config_dir, "usr_config_dir ostree auth")?; + + // We should see runtime_dir content still + let (p, authdata) = read_authfile(rootdir, false).unwrap().unwrap(); + assert_eq!(p, user_runtime_dir); + assert_eq!(authdata, "usr_runtime_dir ostree auth"); + + // Now remove the runtime_dir content, unveiling the config_dir content + root.remove_file(&user_runtime_dir)?; + let (p, authdata) = read_authfile(rootdir, false).unwrap().unwrap(); + assert_eq!(p, user_config_dir); + assert_eq!(authdata, "usr_config_dir ostree auth"); + + Ok(()) + } +} diff --git a/ostree-ext/src/ima.rs b/ostree-ext/src/ima.rs new file mode 100644 index 00000000..ca6d8ccd --- /dev/null +++ b/ostree-ext/src/ima.rs @@ -0,0 +1,287 @@ +//! Write IMA signatures to an ostree commit + +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use crate::objgv::*; +use anyhow::{Context, Result}; +use camino::Utf8PathBuf; +use fn_error_context::context; +use gio::glib; +use gio::prelude::*; +use glib::Cast; +use glib::Variant; +use gvariant::aligned_bytes::TryAsAligned; +use gvariant::{gv, Marker, Structure}; +use ostree::gio; +use rustix::fd::BorrowedFd; +use std::collections::{BTreeMap, HashMap}; +use std::ffi::CString; +use std::fs::File; +use std::io::Seek; +use std::os::unix::io::AsRawFd; +use std::process::{Command, Stdio}; + +/// Extended attribute keys used for IMA. +const IMA_XATTR: &str = "security.ima"; + +/// Attributes to configure IMA signatures. +#[derive(Debug, Clone)] +pub struct ImaOpts { + /// Digest algorithm + pub algorithm: String, + + /// Path to IMA key + pub key: Utf8PathBuf, + + /// Replace any existing IMA signatures. + pub overwrite: bool, +} + +/// Convert a GVariant of type `a(ayay)` to a mutable map +fn xattrs_to_map(v: &glib::Variant) -> BTreeMap, Vec> { + let v = v.data_as_bytes(); + let v = v.try_as_aligned().unwrap(); + let v = gv!("a(ayay)").cast(v); + let mut map: BTreeMap, Vec> = BTreeMap::new(); + for e in v.iter() { + let (k, v) = e.to_tuple(); + map.insert(k.into(), v.into()); + } + map +} + +/// Create a new GVariant of type a(ayay). This is used by OSTree's extended attributes. +pub(crate) fn new_variant_a_ayay<'a, T: 'a + AsRef<[u8]>>( + items: impl IntoIterator, +) -> glib::Variant { + let children = items.into_iter().map(|(a, b)| { + let a = a.as_ref(); + let b = b.as_ref(); + Variant::tuple_from_iter([a.to_variant(), b.to_variant()]) + }); + Variant::array_from_iter::<(&[u8], &[u8])>(children) +} + +struct CommitRewriter<'a> { + repo: &'a ostree::Repo, + ima: &'a ImaOpts, + tempdir: tempfile::TempDir, + /// Maps content object sha256 hex string to a signed object sha256 hex string + rewritten_files: HashMap, +} + +#[allow(unsafe_code)] +#[context("Gathering xattr {}", k)] +fn steal_xattr(f: &File, k: &str) -> Result> { + let k = &CString::new(k)?; + unsafe { + let k = k.as_ptr() as *const _; + let r = libc::fgetxattr(f.as_raw_fd(), k, std::ptr::null_mut(), 0); + if r < 0 { + return Err(std::io::Error::last_os_error().into()); + } + let sz: usize = r.try_into()?; + let mut buf = vec![0u8; sz]; + let r = libc::fgetxattr(f.as_raw_fd(), k, buf.as_mut_ptr() as *mut _, sz); + if r < 0 { + return Err(std::io::Error::last_os_error().into()); + } + let r = libc::fremovexattr(f.as_raw_fd(), k); + if r < 0 { + return Err(std::io::Error::last_os_error().into()); + } + Ok(buf) + } +} + +impl<'a> CommitRewriter<'a> { + fn new(repo: &'a ostree::Repo, ima: &'a ImaOpts) -> Result { + Ok(Self { + repo, + ima, + tempdir: tempfile::tempdir_in(format!("/proc/self/fd/{}/tmp", repo.dfd()))?, + rewritten_files: Default::default(), + }) + } + + /// Use `evmctl` to generate an IMA signature on a file, then + /// scrape the xattr value out of it (removing it). + /// + /// evmctl can write a separate file but it picks the name...so + /// we do this hacky dance of `--xattr-user` instead. + #[allow(unsafe_code)] + #[context("IMA signing object")] + fn ima_sign(&self, instream: &gio::InputStream) -> Result, Vec>> { + let mut tempf = tempfile::NamedTempFile::new_in(self.tempdir.path())?; + // If we're operating on a bare repo, we can clone the file (copy_file_range) directly. + if let Ok(instream) = instream.clone().downcast::() { + use cap_std_ext::cap_std::io_lifetimes::AsFilelike; + // View the fd as a File + let instream_fd = unsafe { BorrowedFd::borrow_raw(instream.as_raw_fd()) }; + let instream_fd = instream_fd.as_filelike_view::(); + std::io::copy(&mut (&*instream_fd), tempf.as_file_mut())?; + } else { + // If we're operating on an archive repo, then we need to uncompress + // and recompress... + let mut instream = instream.clone().into_read(); + let _n = std::io::copy(&mut instream, tempf.as_file_mut())?; + } + tempf.seek(std::io::SeekFrom::Start(0))?; + + let mut proc = Command::new("evmctl"); + proc.current_dir(self.tempdir.path()) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .args(["ima_sign", "--xattr-user", "--key", self.ima.key.as_str()]) + .args(["--hashalgo", self.ima.algorithm.as_str()]) + .arg(tempf.path().file_name().unwrap()); + let status = proc.output().context("Spawning evmctl")?; + if !status.status.success() { + return Err(anyhow::anyhow!( + "evmctl failed: {:?}\n{}", + status.status, + String::from_utf8_lossy(&status.stderr), + )); + } + let mut r = HashMap::new(); + let user_k = IMA_XATTR.replace("security.", "user."); + let v = steal_xattr(tempf.as_file(), user_k.as_str())?; + r.insert(Vec::from(IMA_XATTR.as_bytes()), v); + Ok(r) + } + + #[context("Content object {}", checksum)] + fn map_file(&mut self, checksum: &str) -> Result> { + let cancellable = gio::Cancellable::NONE; + let (instream, meta, xattrs) = self.repo.load_file(checksum, cancellable)?; + let instream = if let Some(i) = instream { + i + } else { + return Ok(None); + }; + let mut xattrs = xattrs_to_map(&xattrs); + let existing_sig = xattrs.remove(IMA_XATTR.as_bytes()); + if existing_sig.is_some() && !self.ima.overwrite { + return Ok(None); + } + + // Now inject the IMA xattr + let xattrs = { + let signed = self.ima_sign(&instream)?; + xattrs.extend(signed); + new_variant_a_ayay(&xattrs) + }; + // Now reload the input stream + let (instream, _, _) = self.repo.load_file(checksum, cancellable)?; + let instream = instream.unwrap(); + let (ostream, size) = + ostree::raw_file_to_content_stream(&instream, &meta, Some(&xattrs), cancellable)?; + let new_checksum = self + .repo + .write_content(None, &ostream, size, cancellable)? + .to_hex(); + + Ok(Some(new_checksum)) + } + + /// Write a dirtree object. + fn map_dirtree(&mut self, checksum: &str) -> Result { + let src = &self + .repo + .load_variant(ostree::ObjectType::DirTree, checksum)?; + let src = src.data_as_bytes(); + let src = src.try_as_aligned()?; + let src = gv_dirtree!().cast(src); + let (files, dirs) = src.to_tuple(); + + // A reusable buffer to avoid heap allocating these + let mut hexbuf = [0u8; 64]; + + let mut new_files = Vec::new(); + for file in files { + let (name, csum) = file.to_tuple(); + let name = name.to_str(); + hex::encode_to_slice(csum, &mut hexbuf)?; + let checksum = std::str::from_utf8(&hexbuf)?; + if let Some(mapped) = self.rewritten_files.get(checksum) { + new_files.push((name, hex::decode(mapped)?)); + } else if let Some(mapped) = self.map_file(checksum)? { + let mapped_bytes = hex::decode(&mapped)?; + self.rewritten_files.insert(checksum.into(), mapped); + new_files.push((name, mapped_bytes)); + } else { + new_files.push((name, Vec::from(csum))); + } + } + + let mut new_dirs = Vec::new(); + for item in dirs { + let (name, contents_csum, meta_csum_bytes) = item.to_tuple(); + let name = name.to_str(); + hex::encode_to_slice(contents_csum, &mut hexbuf)?; + let contents_csum = std::str::from_utf8(&hexbuf)?; + let mapped = self.map_dirtree(contents_csum)?; + let mapped = hex::decode(mapped)?; + new_dirs.push((name, mapped, meta_csum_bytes)); + } + + let new_dirtree = (new_files, new_dirs).to_variant(); + + let mapped = self + .repo + .write_metadata( + ostree::ObjectType::DirTree, + None, + &new_dirtree, + gio::Cancellable::NONE, + )? + .to_hex(); + + Ok(mapped) + } + + /// Write a commit object. + #[context("Mapping {}", rev)] + fn map_commit(&mut self, rev: &str) -> Result { + let checksum = self.repo.require_rev(rev)?; + let cancellable = gio::Cancellable::NONE; + let (commit_v, _) = self.repo.load_commit(&checksum)?; + let commit_v = &commit_v; + + let commit_bytes = commit_v.data_as_bytes(); + let commit_bytes = commit_bytes.try_as_aligned()?; + let commit = gv_commit!().cast(commit_bytes); + let commit = commit.to_tuple(); + let contents = &hex::encode(commit.6); + + let new_dt = self.map_dirtree(contents)?; + + let n_parts = 8; + let mut parts = Vec::with_capacity(n_parts); + for i in 0..n_parts { + parts.push(commit_v.child_value(i)); + } + let new_dt = hex::decode(new_dt)?; + parts[6] = new_dt.to_variant(); + let new_commit = Variant::tuple_from_iter(&parts); + + let new_commit_checksum = self + .repo + .write_metadata(ostree::ObjectType::Commit, None, &new_commit, cancellable)? + .to_hex(); + + Ok(new_commit_checksum) + } +} + +/// Given an OSTree commit and an IMA configuration, generate a new commit object with IMA signatures. +/// +/// The generated commit object will inherit all metadata from the existing commit object +/// such as version, etc. +/// +/// This function does not create an ostree transaction; it's recommended to use outside the call +/// to this function. +pub fn ima_sign(repo: &ostree::Repo, ostree_ref: &str, opts: &ImaOpts) -> Result { + let writer = &mut CommitRewriter::new(repo, opts)?; + writer.map_commit(ostree_ref) +} diff --git a/ostree-ext/src/integrationtest.rs b/ostree-ext/src/integrationtest.rs new file mode 100644 index 00000000..d8166ab2 --- /dev/null +++ b/ostree-ext/src/integrationtest.rs @@ -0,0 +1,243 @@ +//! Module used for integration tests; should not be public. + +use std::path::Path; + +use crate::container_utils::is_ostree_container; +use anyhow::{anyhow, Context, Result}; +use camino::Utf8Path; +use cap_std::fs::Dir; +use cap_std_ext::cap_std; +use containers_image_proxy::oci_spec; +use fn_error_context::context; +use gio::prelude::*; +use oci_spec::image as oci_image; +use ocidir::{ + oci_spec::image::{Arch, Platform}, + GzipLayerWriter, +}; +use ostree::gio; +use xshell::cmd; + +pub(crate) fn detectenv() -> Result<&'static str> { + let r = if is_ostree_container()? { + "ostree-container" + } else if Path::new("/run/ostree-booted").exists() { + "ostree" + } else if crate::container_utils::running_in_container() { + "container" + } else { + "none" + }; + Ok(r) +} + +/// Using `src` as a base, take append `dir` into OCI image. +/// Should only be enabled for testing. +#[context("Generating derived oci")] +pub fn generate_derived_oci( + src: impl AsRef, + dir: impl AsRef, + tag: Option<&str>, +) -> Result<()> { + generate_derived_oci_from_tar( + src, + move |w| { + let dir = dir.as_ref(); + let mut layer_tar = tar::Builder::new(w); + layer_tar.append_dir_all("./", dir)?; + layer_tar.finish()?; + Ok(()) + }, + tag, + None, + ) +} + +/// Using `src` as a base, take append `dir` into OCI image. +/// Should only be enabled for testing. +#[context("Generating derived oci")] +pub fn generate_derived_oci_from_tar( + src: impl AsRef, + f: F, + tag: Option<&str>, + arch: Option, +) -> Result<()> +where + F: FnOnce(&mut GzipLayerWriter) -> Result<()>, +{ + let src = src.as_ref(); + let src = Dir::open_ambient_dir(src, cap_std::ambient_authority())?; + let src = ocidir::OciDir::open(&src)?; + + let idx = src + .read_index()? + .ok_or(anyhow!("Reading image index from source"))?; + let manifest_descriptor = idx + .manifests() + .first() + .ok_or(anyhow!("No manifests in index"))?; + let mut manifest: oci_image::ImageManifest = src + .read_json_blob(manifest_descriptor) + .context("Reading manifest json blob")?; + let mut config: oci_image::ImageConfiguration = src.read_json_blob(manifest.config())?; + + if let Some(arch) = arch.as_ref() { + config.set_architecture(arch.clone()); + } + + let mut bw = src.create_gzip_layer(None)?; + f(&mut bw)?; + let new_layer = bw.complete()?; + + manifest.layers_mut().push( + new_layer + .blob + .descriptor() + .media_type(oci_spec::image::MediaType::ImageLayerGzip) + .build() + .unwrap(), + ); + config.history_mut().push( + oci_spec::image::HistoryBuilder::default() + .created_by("generate_derived_oci") + .build() + .unwrap(), + ); + config + .rootfs_mut() + .diff_ids_mut() + .push(new_layer.uncompressed_sha256.digest().to_string()); + let new_config_desc = src.write_config(config)?; + manifest.set_config(new_config_desc); + + let mut platform = Platform::default(); + if let Some(arch) = arch.as_ref() { + platform.set_architecture(arch.clone()); + } + + if let Some(tag) = tag { + src.insert_manifest(manifest, Some(tag), platform)?; + } else { + src.replace_with_single_manifest(manifest, platform)?; + } + Ok(()) +} + +fn test_proxy_auth() -> Result<()> { + use containers_image_proxy::ImageProxyConfig; + let merge = crate::container::merge_default_container_proxy_opts; + let mut c = ImageProxyConfig::default(); + merge(&mut c)?; + assert_eq!(c.authfile, None); + std::fs::create_dir_all("/etc/ostree")?; + let authpath = Path::new("/etc/ostree/auth.json"); + std::fs::write(authpath, "{}")?; + let mut c = ImageProxyConfig::default(); + merge(&mut c)?; + if rustix::process::getuid().is_root() { + assert!(c.auth_data.is_some()); + } else { + assert_eq!(c.authfile.unwrap().as_path(), authpath,); + } + let c = ImageProxyConfig { + auth_anonymous: true, + ..Default::default() + }; + assert_eq!(c.authfile, None); + std::fs::remove_file(authpath)?; + let mut c = ImageProxyConfig::default(); + merge(&mut c)?; + assert_eq!(c.authfile, None); + Ok(()) +} + +/// Create a test fixture in the same way our unit tests does, and print +/// the location of the temporary directory. Also export a chunked image. +/// Useful for debugging things interactively. +pub(crate) async fn create_fixture() -> Result<()> { + let fixture = crate::fixture::Fixture::new_v1()?; + let imgref = fixture.export_container().await?.0; + println!("Wrote: {:?}", imgref); + let path = fixture.into_tempdir().into_path(); + println!("Wrote: {:?}", path); + Ok(()) +} + +pub(crate) fn test_ima() -> Result<()> { + use gvariant::aligned_bytes::TryAsAligned; + use gvariant::{gv, Marker, Structure}; + + let cancellable = gio::Cancellable::NONE; + let fixture = crate::fixture::Fixture::new_v1()?; + + let config = indoc::indoc! { r#" + [ req ] + default_bits = 3048 + distinguished_name = req_distinguished_name + prompt = no + string_mask = utf8only + x509_extensions = myexts + [ req_distinguished_name ] + O = Test + CN = Test key + emailAddress = example@example.com + [ myexts ] + basicConstraints=critical,CA:FALSE + keyUsage=digitalSignature + subjectKeyIdentifier=hash + authorityKeyIdentifier=keyid + "#}; + std::fs::write(fixture.path.join("genkey.config"), config)?; + let sh = xshell::Shell::new()?; + sh.change_dir(&fixture.path); + cmd!( + sh, + "openssl req -new -nodes -utf8 -sha256 -days 36500 -batch -x509 -config genkey.config -outform DER -out ima.der -keyout privkey_ima.pem" + ) + .ignore_stderr() + .ignore_stdout() + .run()?; + + let imaopts = crate::ima::ImaOpts { + algorithm: "sha256".into(), + key: fixture.path.join("privkey_ima.pem"), + overwrite: false, + }; + let rewritten_commit = + crate::ima::ima_sign(fixture.srcrepo(), fixture.testref(), &imaopts).unwrap(); + + let root = fixture + .srcrepo() + .read_commit(&rewritten_commit, cancellable)? + .0; + let bash = root.resolve_relative_path("/usr/bin/bash"); + let bash = bash.downcast_ref::().unwrap(); + let xattrs = bash.xattrs(cancellable).unwrap(); + let v = xattrs.data_as_bytes(); + let v = v.try_as_aligned().unwrap(); + let v = gv!("a(ayay)").cast(v); + let mut found_ima = false; + for xattr in v.iter() { + let k = xattr.to_tuple().0; + if k != b"security.ima" { + continue; + } + found_ima = true; + break; + } + if !found_ima { + anyhow::bail!("Failed to find IMA xattr"); + } + println!("ok IMA"); + Ok(()) +} + +#[cfg(feature = "internal-testing-api")] +#[context("Running integration tests")] +pub(crate) fn run_tests() -> Result<()> { + crate::container_utils::require_ostree_container()?; + // When there's a new integration test to run, add it here. + test_proxy_auth()?; + println!("integration tests succeeded."); + Ok(()) +} diff --git a/ostree-ext/src/isolation.rs b/ostree-ext/src/isolation.rs new file mode 100644 index 00000000..a0896e35 --- /dev/null +++ b/ostree-ext/src/isolation.rs @@ -0,0 +1,49 @@ +use std::process::Command; + +use once_cell::sync::Lazy; + +pub(crate) const DEFAULT_UNPRIVILEGED_USER: &str = "nobody"; + +/// Checks if the current process is (apparently at least) +/// running under systemd. We use this in various places +/// to e.g. log to the journal instead of printing to stdout. +pub(crate) fn running_in_systemd() -> bool { + static RUNNING_IN_SYSTEMD: Lazy = Lazy::new(|| { + // See https://www.freedesktop.org/software/systemd/man/systemd.exec.html#%24INVOCATION_ID + std::env::var_os("INVOCATION_ID") + .filter(|s| !s.is_empty()) + .is_some() + }); + + *RUNNING_IN_SYSTEMD +} + +/// Return a prepared subprocess configuration that will run as an unprivileged user if possible. +/// +/// This currently only drops privileges when run under systemd with DynamicUser. +pub(crate) fn unprivileged_subprocess(binary: &str, user: &str) -> Command { + // TODO: if we detect we're running in a container as uid 0, perhaps at least switch to the + // "bin" user if we can? + if !running_in_systemd() { + return Command::new(binary); + } + let mut cmd = Command::new("setpriv"); + // Clear some strategic environment variables that may cause the containers/image stack + // to look in the wrong places for things. + cmd.env_remove("HOME"); + cmd.env_remove("XDG_DATA_DIR"); + cmd.env_remove("USER"); + cmd.args([ + "--no-new-privs", + "--init-groups", + "--reuid", + user, + "--bounding-set", + "-all", + "--pdeathsig", + "TERM", + "--", + binary, + ]); + cmd +} diff --git a/ostree-ext/src/keyfileext.rs b/ostree-ext/src/keyfileext.rs new file mode 100644 index 00000000..8d6e3a6e --- /dev/null +++ b/ostree-ext/src/keyfileext.rs @@ -0,0 +1,61 @@ +//! Helper methods for [`glib::KeyFile`]. + +use glib::GString; +use ostree::glib; + +/// Helper methods for [`glib::KeyFile`]. +pub trait KeyFileExt { + /// Get a string value, but return `None` if the key does not exist. + fn optional_string(&self, group: &str, key: &str) -> Result, glib::Error>; + /// Get a boolean value, but return `None` if the key does not exist. + fn optional_bool(&self, group: &str, key: &str) -> Result, glib::Error>; +} + +/// Consume a keyfile error, mapping the case where group or key is not found to `Ok(None)`. +pub fn map_keyfile_optional(res: Result) -> Result, glib::Error> { + match res { + Ok(v) => Ok(Some(v)), + Err(e) => { + if let Some(t) = e.kind::() { + match t { + glib::KeyFileError::GroupNotFound | glib::KeyFileError::KeyNotFound => Ok(None), + _ => Err(e), + } + } else { + Err(e) + } + } + } +} + +impl KeyFileExt for glib::KeyFile { + fn optional_string(&self, group: &str, key: &str) -> Result, glib::Error> { + map_keyfile_optional(self.string(group, key)) + } + + fn optional_bool(&self, group: &str, key: &str) -> Result, glib::Error> { + map_keyfile_optional(self.boolean(group, key)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_optional() { + let kf = glib::KeyFile::new(); + assert_eq!(kf.optional_string("foo", "bar").unwrap(), None); + kf.set_string("foo", "baz", "someval"); + assert_eq!(kf.optional_string("foo", "bar").unwrap(), None); + assert_eq!( + kf.optional_string("foo", "baz").unwrap().unwrap(), + "someval" + ); + + assert!(kf.optional_bool("foo", "baz").is_err()); + assert_eq!(kf.optional_bool("foo", "bar").unwrap(), None); + kf.set_boolean("foo", "somebool", false); + assert_eq!(kf.optional_bool("foo", "somebool").unwrap(), Some(false)); + } +} diff --git a/ostree-ext/src/lib.rs b/ostree-ext/src/lib.rs new file mode 100644 index 00000000..b962c8d6 --- /dev/null +++ b/ostree-ext/src/lib.rs @@ -0,0 +1,78 @@ +//! # Extension APIs for ostree +//! +//! This crate builds on top of the core ostree C library +//! and the Rust bindings to it, adding new functionality +//! written in Rust. + +// See https://doc.rust-lang.org/rustc/lints/listing/allowed-by-default.html +#![deny(missing_docs)] +#![deny(missing_debug_implementations)] +#![forbid(unused_must_use)] +#![deny(unsafe_code)] +#![cfg_attr(feature = "dox", feature(doc_cfg))] +#![deny(clippy::dbg_macro)] +#![deny(clippy::todo)] + +// Re-export our dependencies. See https://gtk-rs.org/blog/2021/06/22/new-release.html +// "Dependencies are re-exported". Users will need e.g. `gio::File`, so this avoids +// them needing to update matching versions. +pub use containers_image_proxy; +pub use containers_image_proxy::oci_spec; +pub use ostree; +pub use ostree::gio; +pub use ostree::gio::glib; + +/// Our generic catchall fatal error, expected to be converted +/// to a string to output to a terminal or logs. +type Result = anyhow::Result; + +// Import global functions. +pub mod globals; + +mod isolation; + +pub mod bootabletree; +pub mod cli; +pub mod container; +pub mod container_utils; +pub mod diff; +pub mod ima; +pub mod keyfileext; +pub(crate) mod logging; +pub mod mountutil; +pub mod ostree_prepareroot; +pub mod refescape; +#[doc(hidden)] +pub mod repair; +pub mod sysroot; +pub mod tar; +pub mod tokio_util; + +pub mod selinux; + +pub mod chunking; +pub mod commit; +pub mod objectsource; +pub(crate) mod objgv; +#[cfg(feature = "internal-testing-api")] +pub mod ostree_manual; +#[cfg(not(feature = "internal-testing-api"))] +pub(crate) mod ostree_manual; + +pub(crate) mod statistics; + +mod utils; + +#[cfg(feature = "docgen")] +mod docgen; + +/// Prelude, intended for glob import. +pub mod prelude { + #[doc(hidden)] + pub use ostree::prelude::*; +} + +#[cfg(feature = "internal-testing-api")] +pub mod fixture; +#[cfg(feature = "internal-testing-api")] +pub mod integrationtest; diff --git a/ostree-ext/src/logging.rs b/ostree-ext/src/logging.rs new file mode 100644 index 00000000..b80f30eb --- /dev/null +++ b/ostree-ext/src/logging.rs @@ -0,0 +1,40 @@ +use std::collections::HashMap; +use std::sync::atomic::{AtomicBool, Ordering}; + +/// Set to true if we failed to write to the journal once +static EMITTED_JOURNAL_ERROR: AtomicBool = AtomicBool::new(false); + +/// Wrapper for systemd structured logging which only emits a message +/// if we're targeting the system repository, and it's booted. +pub(crate) fn system_repo_journal_send( + repo: &ostree::Repo, + priority: libsystemd::logging::Priority, + msg: &str, + vars: impl Iterator, +) where + K: AsRef, + V: AsRef, +{ + if !libsystemd::daemon::booted() { + return; + } + if !repo.is_system() { + return; + } + if let Err(e) = libsystemd::logging::journal_send(priority, msg, vars) { + if !EMITTED_JOURNAL_ERROR.swap(true, Ordering::SeqCst) { + eprintln!("failed to write to journal: {e}"); + } + } +} + +/// Wrapper for systemd structured logging which only emits a message +/// if we're targeting the system repository, and it's booted. +pub(crate) fn system_repo_journal_print( + repo: &ostree::Repo, + priority: libsystemd::logging::Priority, + msg: &str, +) { + let vars: HashMap<&str, &str> = HashMap::new(); + system_repo_journal_send(repo, priority, msg, vars.into_iter()) +} diff --git a/ostree-ext/src/mountutil.rs b/ostree-ext/src/mountutil.rs new file mode 100644 index 00000000..f73cbba2 --- /dev/null +++ b/ostree-ext/src/mountutil.rs @@ -0,0 +1,60 @@ +//! Helpers for interacting with mounts. + +use std::os::fd::AsFd; +use std::path::Path; + +use anyhow::Result; +use cap_std::fs::Dir; +use cap_std_ext::cap_std; + +// Fix musl support +#[cfg(target_env = "gnu")] +use libc::STATX_ATTR_MOUNT_ROOT; +#[cfg(target_env = "musl")] +const STATX_ATTR_MOUNT_ROOT: libc::c_int = 0x2000; + +fn is_mountpoint_impl_statx(root: &Dir, path: &Path) -> Result> { + // https://github.com/systemd/systemd/blob/8fbf0a214e2fe474655b17a4b663122943b55db0/src/basic/mountpoint-util.c#L176 + use rustix::fs::{AtFlags, StatxFlags}; + + // SAFETY(unwrap): We can infallibly convert an i32 into a u64. + let mountroot_flag: u64 = STATX_ATTR_MOUNT_ROOT.try_into().unwrap(); + match rustix::fs::statx( + root.as_fd(), + path, + AtFlags::NO_AUTOMOUNT | AtFlags::SYMLINK_NOFOLLOW, + StatxFlags::empty(), + ) { + Ok(r) => { + let present = (r.stx_attributes_mask & mountroot_flag) > 0; + Ok(present.then_some(r.stx_attributes & mountroot_flag > 0)) + } + Err(e) if e == rustix::io::Errno::NOSYS => Ok(None), + Err(e) => Err(e.into()), + } +} + +/// Try to (heuristically) determine if the provided path is a mount root. +pub fn is_mountpoint(root: &Dir, path: impl AsRef) -> Result> { + is_mountpoint_impl_statx(root, path.as_ref()) +} + +#[cfg(test)] +mod tests { + use super::*; + use cap_std_ext::cap_tempfile; + + #[test] + fn test_is_mountpoint() -> Result<()> { + let root = cap_std::fs::Dir::open_ambient_dir("/", cap_std::ambient_authority())?; + let supported = is_mountpoint(&root, Path::new("/")).unwrap(); + match supported { + Some(r) => assert!(r), + // If the host doesn't support statx, ignore this for now + None => return Ok(()), + } + let tmpdir = cap_tempfile::TempDir::new(cap_std::ambient_authority())?; + assert!(!is_mountpoint(&tmpdir, Path::new(".")).unwrap().unwrap()); + Ok(()) + } +} diff --git a/ostree-ext/src/objectsource.rs b/ostree-ext/src/objectsource.rs new file mode 100644 index 00000000..f32c56ea --- /dev/null +++ b/ostree-ext/src/objectsource.rs @@ -0,0 +1,91 @@ +//! Metadata about the source of an object: a component or package. +//! +//! This is used to help split up containers into distinct layers. + +use indexmap::IndexMap; +use std::borrow::Borrow; +use std::collections::HashSet; +use std::hash::Hash; +use std::rc::Rc; + +use serde::{Deserialize, Serialize, Serializer}; + +mod rcstr_serialize { + use serde::Deserializer; + + use super::*; + + pub(crate) fn serialize(v: &Rc, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_str(v) + } + + pub(crate) fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> + where + D: Deserializer<'de>, + { + let v = String::deserialize(deserializer)?; + Ok(Rc::from(v.into_boxed_str())) + } +} + +/// Identifier for content (e.g. package/layer). Not necessarily human readable. +/// For example in RPMs, this may be a full "NEVRA" i.e. name-epoch:version-release.architecture e.g. kernel-6.2-2.fc38.aarch64 +/// But that's not strictly required as this string should only live in memory and not be persisted. +pub type ContentID = Rc; + +/// Metadata about a component/package. +#[derive(Debug, Eq, Deserialize, Serialize)] +pub struct ObjectSourceMeta { + /// Unique identifier, does not need to be human readable, but can be. + #[serde(with = "rcstr_serialize")] + pub identifier: ContentID, + /// Just the name of the package (no version), needs to be human readable. + #[serde(with = "rcstr_serialize")] + pub name: Rc, + /// Identifier for the *source* of this content; for example, if multiple binary + /// packages derive from a single git repository or source package. + #[serde(with = "rcstr_serialize")] + pub srcid: Rc, + /// Unitless, relative offset of last change time. + /// One suggested way to generate this number is to have it be in units of hours or days + /// since the earliest changed item. + pub change_time_offset: u32, + /// Change frequency + pub change_frequency: u32, +} + +impl PartialEq for ObjectSourceMeta { + fn eq(&self, other: &Self) -> bool { + *self.identifier == *other.identifier + } +} + +impl Hash for ObjectSourceMeta { + fn hash(&self, state: &mut H) { + self.identifier.hash(state); + } +} + +impl Borrow for ObjectSourceMeta { + fn borrow(&self) -> &str { + &self.identifier + } +} + +/// Maps from e.g. "bash" or "kernel" to metadata about that content +pub type ObjectMetaSet = HashSet; + +/// Maps from an ostree content object digest to the `ContentSet` key. +pub type ObjectMetaMap = IndexMap; + +/// Grouping of metadata about an object. +#[derive(Debug, Default)] +pub struct ObjectMeta { + /// The set of object sources with their metadata. + pub set: ObjectMetaSet, + /// Mapping from content object to source. + pub map: ObjectMetaMap, +} diff --git a/ostree-ext/src/objgv.rs b/ostree-ext/src/objgv.rs new file mode 100644 index 00000000..3be5c94c --- /dev/null +++ b/ostree-ext/src/objgv.rs @@ -0,0 +1,31 @@ +/// Type representing an ostree commit object. +macro_rules! gv_commit { + () => { + gvariant::gv!("(a{sv}aya(say)sstayay)") + }; +} +pub(crate) use gv_commit; + +/// Type representing an ostree DIRTREE object. +macro_rules! gv_dirtree { + () => { + gvariant::gv!("(a(say)a(sayay))") + }; +} +pub(crate) use gv_dirtree; + +#[cfg(test)] +mod tests { + use gvariant::aligned_bytes::TryAsAligned; + use gvariant::Marker; + + use super::*; + #[test] + fn test_dirtree() { + // Just a compilation test + let data = b"".try_as_aligned().ok(); + if let Some(data) = data { + let _t = gv_dirtree!().cast(data); + } + } +} diff --git a/ostree-ext/src/ostree_manual.rs b/ostree-ext/src/ostree_manual.rs new file mode 100644 index 00000000..26a12210 --- /dev/null +++ b/ostree-ext/src/ostree_manual.rs @@ -0,0 +1,33 @@ +//! Manual workarounds for ostree bugs + +use std::io::Read; +use std::ptr; + +use ostree::prelude::{Cast, InputStreamExtManual}; +use ostree::{gio, glib}; + +/// Equivalent of `g_file_read()` for ostree::RepoFile to work around https://github.com/ostreedev/ostree/issues/2703 +#[allow(unsafe_code)] +pub fn repo_file_read(f: &ostree::RepoFile) -> Result { + use glib::translate::*; + let stream = unsafe { + let f = f.upcast_ref::(); + let mut error = ptr::null_mut(); + let stream = gio::ffi::g_file_read(f.to_glib_none().0, ptr::null_mut(), &mut error); + if !error.is_null() { + return Err(from_glib_full(error)); + } + // Upcast to GInputStream here + from_glib_full(stream as *mut gio::ffi::GInputStream) + }; + + Ok(stream) +} + +/// Read a repo file to a string. +pub fn repo_file_read_to_string(f: &ostree::RepoFile) -> anyhow::Result { + let mut r = String::new(); + let mut s = repo_file_read(f)?.into_read(); + s.read_to_string(&mut r)?; + Ok(r) +} diff --git a/ostree-ext/src/ostree_prepareroot.rs b/ostree-ext/src/ostree_prepareroot.rs new file mode 100644 index 00000000..de7b84fb --- /dev/null +++ b/ostree-ext/src/ostree_prepareroot.rs @@ -0,0 +1,199 @@ +//! Logic related to parsing ostree-prepare-root.conf. +//! + +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use std::str::FromStr; + +use anyhow::{Context, Result}; +use camino::Utf8Path; +use glib::Cast; +use ostree::prelude::FileExt; +use ostree::{gio, glib}; + +use crate::keyfileext::KeyFileExt; +use crate::ostree_manual; +use crate::utils::ResultExt; + +pub(crate) const CONF_PATH: &str = "ostree/prepare-root.conf"; + +pub(crate) fn load_config(root: &ostree::RepoFile) -> Result> { + let cancellable = gio::Cancellable::NONE; + let kf = glib::KeyFile::new(); + for path in ["etc", "usr/lib"].into_iter().map(Utf8Path::new) { + let path = &path.join(CONF_PATH); + let f = root.resolve_relative_path(path); + if !f.query_exists(cancellable) { + continue; + } + let f = f.downcast_ref::().unwrap(); + let contents = ostree_manual::repo_file_read_to_string(f)?; + kf.load_from_data(&contents, glib::KeyFileFlags::NONE) + .with_context(|| format!("Parsing {path}"))?; + tracing::debug!("Loaded {path}"); + return Ok(Some(kf)); + } + tracing::debug!("No {CONF_PATH} found"); + Ok(None) +} + +/// Query whether the target root has the `root.transient` key +/// which sets up a transient overlayfs. +pub(crate) fn overlayfs_root_enabled(root: &ostree::RepoFile) -> Result { + if let Some(config) = load_config(root)? { + overlayfs_enabled_in_config(&config) + } else { + Ok(false) + } +} + +#[derive(Debug, PartialEq, Eq)] +enum Tristate { + Enabled, + Disabled, + Maybe, +} + +impl FromStr for Tristate { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + let r = match s { + // Keep this in sync with ot_keyfile_get_tristate_with_default from ostree + "yes" | "true" | "1" => Tristate::Enabled, + "no" | "false" | "0" => Tristate::Disabled, + "maybe" => Tristate::Maybe, + o => anyhow::bail!("Invalid tristate value: {o}"), + }; + Ok(r) + } +} + +impl Default for Tristate { + fn default() -> Self { + Self::Disabled + } +} + +impl Tristate { + pub(crate) fn maybe_enabled(&self) -> bool { + match self { + Self::Enabled | Self::Maybe => true, + Self::Disabled => false, + } + } +} + +#[derive(Debug, PartialEq, Eq)] +enum ComposefsState { + Signed, + Tristate(Tristate), +} + +impl Default for ComposefsState { + fn default() -> Self { + Self::Tristate(Tristate::default()) + } +} + +impl FromStr for ComposefsState { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + let r = match s { + "signed" => Self::Signed, + o => Self::Tristate(Tristate::from_str(o)?), + }; + Ok(r) + } +} + +impl ComposefsState { + pub(crate) fn maybe_enabled(&self) -> bool { + match self { + ComposefsState::Signed => true, + ComposefsState::Tristate(t) => t.maybe_enabled(), + } + } +} + +/// Query whether the config uses an overlayfs model (composefs or plain overlayfs). +pub fn overlayfs_enabled_in_config(config: &glib::KeyFile) -> Result { + let root_transient = config + .optional_bool("root", "transient")? + .unwrap_or_default(); + let composefs = config + .optional_string("composefs", "enabled")? + .map(|s| ComposefsState::from_str(s.as_str())) + .transpose() + .log_err_default() + .unwrap_or_default(); + Ok(root_transient || composefs.maybe_enabled()) +} + +#[test] +fn test_tristate() { + for v in ["yes", "true", "1"] { + assert_eq!(Tristate::from_str(v).unwrap(), Tristate::Enabled); + } + assert_eq!(Tristate::from_str("maybe").unwrap(), Tristate::Maybe); + for v in ["no", "false", "0"] { + assert_eq!(Tristate::from_str(v).unwrap(), Tristate::Disabled); + } + for v in ["", "junk", "fal", "tr1"] { + assert!(Tristate::from_str(v).is_err()); + } +} + +#[test] +fn test_composefs_state() { + assert_eq!( + ComposefsState::from_str("signed").unwrap(), + ComposefsState::Signed + ); + for v in ["yes", "true", "1"] { + assert_eq!( + ComposefsState::from_str(v).unwrap(), + ComposefsState::Tristate(Tristate::Enabled) + ); + } + assert_eq!(Tristate::from_str("maybe").unwrap(), Tristate::Maybe); + for v in ["no", "false", "0"] { + assert_eq!( + ComposefsState::from_str(v).unwrap(), + ComposefsState::Tristate(Tristate::Disabled) + ); + } +} + +#[test] +fn test_overlayfs_enabled() { + let d0 = indoc::indoc! { r#" +[foo] +bar = baz +[root] +"# }; + let d1 = indoc::indoc! { r#" +[root] +transient = false + "# }; + let d2 = indoc::indoc! { r#" +[composefs] +enabled = false + "# }; + for v in ["", d0, d1, d2] { + let kf = glib::KeyFile::new(); + kf.load_from_data(v, glib::KeyFileFlags::empty()).unwrap(); + assert_eq!(overlayfs_enabled_in_config(&kf).unwrap(), false); + } + + let e0 = format!("{d0}\n[root]\ntransient = true"); + let e1 = format!("{d1}\n[composefs]\nenabled = true\n[other]\nsomekey = someval"); + let e2 = format!("{d1}\n[composefs]\nenabled = yes"); + let e3 = format!("{d1}\n[composefs]\nenabled = signed"); + for v in [e0, e1, e2, e3] { + let kf = glib::KeyFile::new(); + kf.load_from_data(&v, glib::KeyFileFlags::empty()).unwrap(); + assert_eq!(overlayfs_enabled_in_config(&kf).unwrap(), true); + } +} diff --git a/ostree-ext/src/refescape.rs b/ostree-ext/src/refescape.rs new file mode 100644 index 00000000..cb53aa62 --- /dev/null +++ b/ostree-ext/src/refescape.rs @@ -0,0 +1,198 @@ +//! Escape strings for use in ostree refs. +//! +//! It can be desirable to map arbitrary identifiers, such as RPM/dpkg +//! package names or container image references (e.g. `docker://quay.io/examplecorp/os:latest`) +//! into ostree refs (branch names) which have a quite restricted set +//! of valid characters; basically alphanumeric, plus `/`, `-`, `_`. +//! +//! This escaping scheme uses `_` in a similar way as a `\` character is +//! used in Rust unicode escaped values. For example, `:` is `_3A_` (hexadecimal). +//! Because the empty path is not valid, `//` is escaped as `/_2F_` (i.e. the second `/` is escaped). + +use anyhow::Result; +use std::fmt::Write; + +/// Escape a single string; this is a backend of [`prefix_escape_for_ref`]. +fn escape_for_ref(s: &str) -> Result { + if s.is_empty() { + return Err(anyhow::anyhow!("Invalid empty string for ref")); + } + fn escape_c(r: &mut String, c: char) { + write!(r, "_{:02X}_", c as u32).unwrap() + } + let mut r = String::new(); + let mut it = s + .chars() + .map(|c| { + if c == '\0' { + Err(anyhow::anyhow!( + "Invalid embedded NUL in string for ostree ref" + )) + } else { + Ok(c) + } + }) + .peekable(); + + let mut previous_alphanumeric = false; + while let Some(c) = it.next() { + let has_next = it.peek().is_some(); + let c = c?; + let current_alphanumeric = c.is_ascii_alphanumeric(); + match c { + c if current_alphanumeric => r.push(c), + '/' if previous_alphanumeric && has_next => r.push(c), + // Pass through `-` unconditionally + '-' => r.push(c), + // The underscore `_` quotes itself `__`. + '_' => r.push_str("__"), + o => escape_c(&mut r, o), + } + previous_alphanumeric = current_alphanumeric; + } + Ok(r) +} + +/// Compute a string suitable for use as an OSTree ref, where `s` can be a (nearly) +/// arbitrary UTF-8 string. This requires a non-empty prefix. +/// +/// The restrictions on `s` are: +/// - The empty string is not supported +/// - There may not be embedded `NUL` (`\0`) characters. +/// +/// The intention behind requiring a prefix is that a common need is to use e.g. +/// [`ostree::Repo::list_refs`] to find refs of a certain "type". +/// +/// # Examples: +/// +/// ```rust +/// # fn test() -> anyhow::Result<()> { +/// use ostree_ext::refescape; +/// let s = "registry:quay.io/coreos/fedora:latest"; +/// assert_eq!(refescape::prefix_escape_for_ref("container", s)?, +/// "container/registry_3A_quay_2E_io/coreos/fedora_3A_latest"); +/// # Ok(()) +/// # } +/// ``` +pub fn prefix_escape_for_ref(prefix: &str, s: &str) -> Result { + Ok(format!("{}/{}", prefix, escape_for_ref(s)?)) +} + +/// Reverse the effect of [`escape_for_ref()`]. +fn unescape_for_ref(s: &str) -> Result { + let mut r = String::new(); + let mut it = s.chars(); + let mut buf = String::new(); + while let Some(c) = it.next() { + match c { + c if c.is_ascii_alphanumeric() => { + r.push(c); + } + '-' | '/' => r.push(c), + '_' => { + let next = it.next(); + if let Some('_') = next { + r.push('_') + } else if let Some(c) = next { + buf.clear(); + buf.push(c); + for c in &mut it { + if c == '_' { + break; + } + buf.push(c); + } + let v = u32::from_str_radix(&buf, 16)?; + let c: char = v.try_into()?; + r.push(c); + } + } + o => anyhow::bail!("Invalid character {}", o), + } + } + Ok(r) +} + +/// Remove a prefix from an ostree ref, and return the unescaped remainder. +/// +/// # Examples: +/// +/// ```rust +/// # fn test() -> anyhow::Result<()> { +/// use ostree_ext::refescape; +/// let s = "registry:quay.io/coreos/fedora:latest"; +/// assert_eq!(refescape::unprefix_unescape_ref("container", "container/registry_3A_quay_2E_io/coreos/fedora_3A_latest")?, s); +/// # Ok(()) +/// # } +/// ``` +pub fn unprefix_unescape_ref(prefix: &str, ostree_ref: &str) -> Result { + let rest = ostree_ref + .strip_prefix(prefix) + .and_then(|s| s.strip_prefix('/')) + .ok_or_else(|| { + anyhow::anyhow!( + "ref does not match expected prefix {}/: {}", + ostree_ref, + prefix + ) + })?; + unescape_for_ref(rest) +} + +#[cfg(test)] +mod test { + use super::*; + use quickcheck::{quickcheck, TestResult}; + + const TESTPREFIX: &str = "testprefix/blah"; + + const UNCHANGED: &[&str] = &["foo", "foo/bar/baz-blah/foo"]; + const ROUNDTRIP: &[&str] = &[ + "localhost:5000/foo:latest", + "fedora/x86_64/coreos", + "/foo/bar/foo.oci-archive", + "/foo/bar/foo.docker-archive", + "docker://quay.io/exampleos/blah:latest", + "oci-archive:/path/to/foo.ociarchive", + "docker-archive:/path/to/foo.dockerarchive", + ]; + const CORNERCASES: &[&str] = &["/", "blah/", "/foo/"]; + + #[test] + fn escape() { + // These strings shouldn't change + for &v in UNCHANGED { + let escaped = &escape_for_ref(v).unwrap(); + ostree::validate_rev(escaped).unwrap(); + assert_eq!(escaped.as_str(), v); + } + // Roundtrip cases, plus unchanged cases + for &v in UNCHANGED.iter().chain(ROUNDTRIP).chain(CORNERCASES) { + let escaped = &prefix_escape_for_ref(TESTPREFIX, v).unwrap(); + ostree::validate_rev(escaped).unwrap(); + let unescaped = unprefix_unescape_ref(TESTPREFIX, escaped).unwrap(); + assert_eq!(v, unescaped); + } + // Explicit test + assert_eq!( + escape_for_ref(ROUNDTRIP[0]).unwrap(), + "localhost_3A_5000/foo_3A_latest" + ); + } + + fn roundtrip(s: String) -> TestResult { + // Ensure we only try strings which match the predicates. + let r = prefix_escape_for_ref(TESTPREFIX, &s); + let escaped = match r { + Ok(v) => v, + Err(_) => return TestResult::discard(), + }; + let unescaped = unprefix_unescape_ref(TESTPREFIX, &escaped).unwrap(); + TestResult::from_bool(unescaped == s) + } + + #[test] + fn qcheck() { + quickcheck(roundtrip as fn(String) -> TestResult); + } +} diff --git a/ostree-ext/src/repair.rs b/ostree-ext/src/repair.rs new file mode 100644 index 00000000..d15bdcef --- /dev/null +++ b/ostree-ext/src/repair.rs @@ -0,0 +1,261 @@ +//! System repair functionality + +use std::collections::{BTreeMap, BTreeSet}; +use std::fmt::Display; + +use anyhow::{anyhow, Context, Result}; +use cap_std::fs::{Dir, MetadataExt}; +use cap_std_ext::cap_std; +use fn_error_context::context; +use serde::{Deserialize, Serialize}; + +use crate::sysroot::SysrootLock; + +// Find the inode numbers for objects +fn gather_inodes( + prefix: &str, + dir: &Dir, + little_inodes: &mut BTreeMap, + big_inodes: &mut BTreeMap, +) -> Result<()> { + for child in dir.entries()? { + let child = child?; + let metadata = child.metadata()?; + if !(metadata.is_file() || metadata.is_symlink()) { + continue; + } + let name = child.file_name(); + let name = name + .to_str() + .ok_or_else(|| anyhow::anyhow!("Invalid {name:?}"))?; + let object_rest = name + .split_once('.') + .ok_or_else(|| anyhow!("Invalid object {name}"))? + .0; + let checksum = format!("{prefix}{object_rest}"); + let inode = metadata.ino(); + if let Ok(little) = u32::try_from(inode) { + little_inodes.insert(little, checksum); + } else { + big_inodes.insert(inode, checksum); + } + } + Ok(()) +} + +#[derive(Default, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub struct RepairResult { + /// Result of inode checking + pub inodes: InodeCheck, + // Whether we detected a likely corrupted merge commit + pub likely_corrupted_container_image_merges: Vec, + // Whether the booted deployment is likely corrupted + pub booted_is_likely_corrupted: bool, + // Whether the staged deployment is likely corrupted + pub staged_is_likely_corrupted: bool, +} + +#[derive(Default, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub struct InodeCheck { + // Number of >32 bit inodes found + pub inode64: u64, + // Number of <= 32 bit inodes found + pub inode32: u64, + // Number of collisions found (when 64 bit inode is truncated to 32 bit) + pub collisions: BTreeSet, +} + +impl Display for InodeCheck { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "ostree inode check:\n 64bit inodes: {}\n 32 bit inodes: {}\n collisions: {}\n", + self.inode64, + self.inode32, + self.collisions.len() + ) + } +} + +impl InodeCheck { + pub fn is_ok(&self) -> bool { + self.collisions.is_empty() + } +} + +#[context("Checking inodes")] +#[doc(hidden)] +/// Detect if any commits are potentially incorrect due to inode truncations. +pub fn check_inode_collision(repo: &ostree::Repo, verbose: bool) -> Result { + let repo_dir = Dir::reopen_dir(&repo.dfd_borrow())?; + let objects = repo_dir.open_dir("objects")?; + + println!( + r#"Attempting analysis of ostree state for files that may be incorrectly linked. +For more information, see https://github.com/ostreedev/ostree/pull/2874/commits/de6fddc6adee09a93901243dc7074090828a1912 +"# + ); + + println!("Gathering inodes for ostree objects..."); + let mut little_inodes = BTreeMap::new(); + let mut big_inodes = BTreeMap::new(); + + for child in objects.entries()? { + let child = child?; + if !child.file_type()?.is_dir() { + continue; + } + let name = child.file_name(); + if name.len() != 2 { + continue; + } + let name = name + .to_str() + .ok_or_else(|| anyhow::anyhow!("Invalid {name:?}"))?; + let objdir = child.open_dir()?; + gather_inodes(name, &objdir, &mut little_inodes, &mut big_inodes) + .with_context(|| format!("Processing {name:?}"))?; + } + + let mut colliding_inodes = BTreeMap::new(); + for (big_inum, big_inum_checksum) in big_inodes.iter() { + let truncated = *big_inum as u32; + if let Some(small_inum_object) = little_inodes.get(&truncated) { + // Don't output each collision unless verbose mode is enabled. It's actually + // quite interesting to see data, but only for development and deep introspection + // use cases. + if verbose { + eprintln!( + r#"collision: + inode (>32 bit): {big_inum} + object: {big_inum_checksum} + inode (truncated): {truncated} + object: {small_inum_object} +"# + ); + } + colliding_inodes.insert(big_inum, big_inum_checksum); + } + } + + // From here let's just track the possibly-colliding 64 bit inode, not also + // the checksum. + let collisions = colliding_inodes + .keys() + .map(|&&v| v) + .collect::>(); + + let inode32 = little_inodes.len() as u64; + let inode64 = big_inodes.len() as u64; + Ok(InodeCheck { + inode32, + inode64, + collisions, + }) +} + +/// Attempt to automatically repair any corruption from inode collisions. +#[doc(hidden)] +pub fn analyze_for_repair(sysroot: &SysrootLock, verbose: bool) -> Result { + use crate::container::store as container_store; + let repo = &sysroot.repo(); + + // Query booted and pending state + let booted_deployment = sysroot.booted_deployment(); + let booted_checksum = booted_deployment.as_ref().map(|b| b.csum()); + let booted_checksum = booted_checksum.as_ref().map(|s| s.as_str()); + let staged_deployment = sysroot.staged_deployment(); + let staged_checksum = staged_deployment.as_ref().map(|b| b.csum()); + let staged_checksum = staged_checksum.as_ref().map(|s| s.as_str()); + + let inodes = check_inode_collision(repo, verbose)?; + println!("{}", inodes); + if inodes.is_ok() { + println!("OK no colliding inodes found"); + return Ok(RepairResult { + inodes, + ..Default::default() + }); + } + + let all_images = container_store::list_images(repo)?; + let all_images = all_images + .into_iter() + .map(|img| crate::container::ImageReference::try_from(img.as_str())) + .collect::>>()?; + println!("Verifying ostree-container images: {}", all_images.len()); + let mut likely_corrupted_container_image_merges = Vec::new(); + let mut booted_is_likely_corrupted = false; + let mut staged_is_likely_corrupted = false; + for imgref in all_images { + if let Some(state) = container_store::query_image(repo, &imgref)? { + if !container_store::verify_container_image( + sysroot, + &imgref, + &state, + &inodes.collisions, + verbose, + )? { + eprintln!("warning: Corrupted image {imgref}"); + likely_corrupted_container_image_merges.push(imgref.to_string()); + let merge_commit = state.merge_commit.as_str(); + if booted_checksum == Some(merge_commit) { + booted_is_likely_corrupted = true; + eprintln!("warning: booted deployment is likely corrupted"); + } else if staged_checksum == Some(merge_commit) { + staged_is_likely_corrupted = true; + eprintln!("warning: staged deployment is likely corrupted"); + } + } + } else { + // This really shouldn't happen + eprintln!("warning: Image was removed from underneath us: {imgref}"); + std::thread::sleep(std::time::Duration::from_secs(1)); + } + } + Ok(RepairResult { + inodes, + likely_corrupted_container_image_merges, + booted_is_likely_corrupted, + staged_is_likely_corrupted, + }) +} + +impl RepairResult { + pub fn check(&self) -> anyhow::Result<()> { + if self.booted_is_likely_corrupted { + eprintln!("warning: booted deployment is likely corrupted"); + } + if self.booted_is_likely_corrupted { + eprintln!("warning: staged deployment is likely corrupted"); + } + match self.likely_corrupted_container_image_merges.len() { + 0 => { + println!("OK no corruption found"); + Ok(()) + } + n => { + anyhow::bail!("Found corruption in images: {n}") + } + } + } + + #[context("Repairing")] + pub fn repair(self, sysroot: &SysrootLock) -> Result<()> { + let repo = &sysroot.repo(); + for imgref in self.likely_corrupted_container_image_merges { + let imgref = crate::container::ImageReference::try_from(imgref.as_str())?; + eprintln!("Flushing cached state for corrupted merged image: {imgref}"); + crate::container::store::remove_images(repo, [&imgref])?; + } + if self.booted_is_likely_corrupted { + anyhow::bail!("TODO redeploy and reboot for booted deployment corruption"); + } + if self.staged_is_likely_corrupted { + anyhow::bail!("TODO undeploy for staged deployment corruption"); + } + Ok(()) + } +} diff --git a/ostree-ext/src/selinux.rs b/ostree-ext/src/selinux.rs new file mode 100644 index 00000000..35acb750 --- /dev/null +++ b/ostree-ext/src/selinux.rs @@ -0,0 +1,39 @@ +//! SELinux-related helper APIs. + +use anyhow::Result; +use fn_error_context::context; +use std::path::Path; + +/// The well-known selinuxfs mount point +const SELINUX_MNT: &str = "/sys/fs/selinux"; +/// Hardcoded value for SELinux domain capable of setting unknown contexts. +const INSTALL_T: &str = "install_t"; + +/// Query for whether or not SELinux is enabled. +pub fn is_selinux_enabled() -> bool { + Path::new(SELINUX_MNT).join("access").exists() +} + +/// Return an error If the current process is not running in the `install_t` domain. +#[context("Verifying self is install_t SELinux domain")] +pub fn verify_install_domain() -> Result<()> { + // If it doesn't look like SELinux is enabled, then nothing to do. + if !is_selinux_enabled() { + return Ok(()); + } + + // If we're not root, there's no need to try to warn because we can only + // do read-only operations anyways. + if !rustix::process::getuid().is_root() { + return Ok(()); + } + + let self_domain = std::fs::read_to_string("/proc/self/attr/current")?; + let is_install_t = self_domain.split(':').any(|x| x == INSTALL_T); + if !is_install_t { + anyhow::bail!( + "Detected SELinux enabled system, but the executing binary is not labeled install_exec_t" + ); + } + Ok(()) +} diff --git a/ostree-ext/src/statistics.rs b/ostree-ext/src/statistics.rs new file mode 100644 index 00000000..7b0102fb --- /dev/null +++ b/ostree-ext/src/statistics.rs @@ -0,0 +1,109 @@ +//! This module holds implementations of some basic statistical properties, such as mean and standard deviation. + +pub(crate) fn mean(data: &[u64]) -> Option { + if data.is_empty() { + None + } else { + Some(data.iter().sum::() as f64 / data.len() as f64) + } +} + +pub(crate) fn std_deviation(data: &[u64]) -> Option { + match (mean(data), data.len()) { + (Some(data_mean), count) if count > 0 => { + let variance = data + .iter() + .map(|value| { + let diff = data_mean - (*value as f64); + diff * diff + }) + .sum::() + / count as f64; + Some(variance.sqrt()) + } + _ => None, + } +} + +//Assumed sorted +pub(crate) fn median_absolute_deviation(data: &mut [u64]) -> Option<(f64, f64)> { + if data.is_empty() { + None + } else { + //Sort data + //data.sort_by(|a, b| a.partial_cmp(b).unwrap()); + + //Find median of data + let median_data: f64 = match data.len() % 2 { + 1 => data[data.len() / 2] as f64, + _ => 0.5 * (data[data.len() / 2 - 1] + data[data.len() / 2]) as f64, + }; + + //Absolute deviations + let mut absolute_deviations = Vec::new(); + for size in data { + absolute_deviations.push(f64::abs(*size as f64 - median_data)) + } + + absolute_deviations.sort_by(|a, b| a.partial_cmp(b).unwrap()); + let l = absolute_deviations.len(); + let mad: f64 = match l % 2 { + 1 => absolute_deviations[l / 2], + _ => 0.5 * (absolute_deviations[l / 2 - 1] + absolute_deviations[l / 2]), + }; + + Some((median_data, mad)) + } +} + +#[test] +fn test_mean() { + assert_eq!(mean(&[]), None); + for v in [0u64, 1, 5, 100] { + assert_eq!(mean(&[v]), Some(v as f64)); + } + assert_eq!(mean(&[0, 1]), Some(0.5)); + assert_eq!(mean(&[0, 5, 100]), Some(35.0)); + assert_eq!(mean(&[7, 4, 30, 14]), Some(13.75)); +} + +#[test] +fn test_std_deviation() { + assert_eq!(std_deviation(&[]), None); + for v in [0u64, 1, 5, 100] { + assert_eq!(std_deviation(&[v]), Some(0 as f64)); + } + assert_eq!(std_deviation(&[1, 4]), Some(1.5)); + assert_eq!(std_deviation(&[2, 2, 2, 2]), Some(0.0)); + assert_eq!( + std_deviation(&[1, 20, 300, 4000, 50000, 600000, 7000000, 80000000]), + Some(26193874.56387471) + ); +} + +#[test] +fn test_median_absolute_deviation() { + //Assumes sorted + assert_eq!(median_absolute_deviation(&mut []), None); + for v in [0u64, 1, 5, 100] { + assert_eq!(median_absolute_deviation(&mut [v]), Some((v as f64, 0.0))); + } + assert_eq!(median_absolute_deviation(&mut [1, 4]), Some((2.5, 1.5))); + assert_eq!( + median_absolute_deviation(&mut [2, 2, 2, 2]), + Some((2.0, 0.0)) + ); + assert_eq!( + median_absolute_deviation(&mut [ + 1, 2, 3, 3, 4, 4, 4, 5, 5, 6, 6, 6, 7, 7, 7, 8, 9, 12, 52, 90 + ]), + Some((6.0, 2.0)) + ); + + //if more than half of the data has the same value, MAD = 0, thus any + //value different from the residual median is classified as an outlier + assert_eq!( + median_absolute_deviation(&mut [0, 1, 1, 1, 1, 1, 1, 1, 0]), + Some((1.0, 0.0)) + ); +} diff --git a/ostree-ext/src/sysroot.rs b/ostree-ext/src/sysroot.rs new file mode 100644 index 00000000..a4f97110 --- /dev/null +++ b/ostree-ext/src/sysroot.rs @@ -0,0 +1,62 @@ +//! Helpers for interacting with sysroots. + +use std::ops::Deref; + +use anyhow::Result; + +/// A locked system root. +#[derive(Debug)] +pub struct SysrootLock { + /// The underlying sysroot value. + pub sysroot: ostree::Sysroot, + /// True if we didn't actually lock + unowned: bool, +} + +impl Drop for SysrootLock { + fn drop(&mut self) { + if self.unowned { + return; + } + self.sysroot.unlock(); + } +} + +impl Deref for SysrootLock { + type Target = ostree::Sysroot; + + fn deref(&self) -> &Self::Target { + &self.sysroot + } +} + +impl SysrootLock { + /// Asynchronously acquire a sysroot lock. If the lock cannot be acquired + /// immediately, a status message will be printed to standard output. + /// The lock will be unlocked when this object is dropped. + pub async fn new_from_sysroot(sysroot: &ostree::Sysroot) -> Result { + let mut printed = false; + loop { + if sysroot.try_lock()? { + return Ok(Self { + sysroot: sysroot.clone(), + unowned: false, + }); + } + if !printed { + println!("Waiting for sysroot lock..."); + printed = true; + } + tokio::time::sleep(std::time::Duration::from_secs(3)).await; + } + } + + /// This function should only be used when you have locked the sysroot + /// externally (e.g. in C/C++ code). This also does not unlock on drop. + pub fn from_assumed_locked(sysroot: &ostree::Sysroot) -> Self { + Self { + sysroot: sysroot.clone(), + unowned: true, + } + } +} diff --git a/ostree-ext/src/tar/export.rs b/ostree-ext/src/tar/export.rs new file mode 100644 index 00000000..170b3c44 --- /dev/null +++ b/ostree-ext/src/tar/export.rs @@ -0,0 +1,772 @@ +//! APIs for creating container images from OSTree commits + +use crate::chunking; +use crate::objgv::*; +use anyhow::{anyhow, ensure, Context, Result}; +use camino::{Utf8Path, Utf8PathBuf}; +use fn_error_context::context; +use gio::glib; +use gio::prelude::*; +use gvariant::aligned_bytes::TryAsAligned; +use gvariant::{Marker, Structure}; +use ostree::gio; +use std::borrow::Borrow; +use std::borrow::Cow; +use std::collections::HashSet; +use std::io::BufReader; + +/// The repository mode generated by a tar export stream. +pub const BARE_SPLIT_XATTRS_MODE: &str = "bare-split-xattrs"; + +// This is both special in the tar stream *and* it's in the ostree commit. +const SYSROOT: &str = "sysroot"; +// This way the default ostree -> sysroot/ostree symlink works. +const OSTREEDIR: &str = "sysroot/ostree"; +// The ref added (under ostree/) in the exported OSTree repo pointing at the commit. +#[allow(dead_code)] +const OSTREEREF: &str = "encapsulated"; + +/// In v0 format, we use this relative path prefix. I think I chose this by looking +/// at the current Fedora base image tar stream. However, several others don't do +/// this and have paths be relative by simply omitting `./`, i.e. the tar stream +/// contains `usr/bin/bash` and not `./usr/bin/bash`. The former looks cleaner +/// to me, so in v1 we drop it. +const TAR_PATH_PREFIX_V0: &str = "./"; + +/// The base repository configuration that identifies this is a tar export. +// See https://github.com/ostreedev/ostree/issues/2499 +const REPO_CONFIG: &str = r#"[core] +repo_version=1 +mode=bare-split-xattrs +"#; + +/// A decently large buffer, as used by e.g. coreutils `cat`. +/// System calls are expensive. +const BUF_CAPACITY: usize = 131072; + +/// Convert /usr/etc back to /etc +fn map_path(p: &Utf8Path) -> std::borrow::Cow { + match p.strip_prefix("./usr/etc") { + Ok(r) => Cow::Owned(Utf8Path::new("./etc").join(r)), + _ => Cow::Borrowed(p), + } +} + +/// Convert usr/etc back to etc for the tar stream. +fn map_path_v1(p: &Utf8Path) -> &Utf8Path { + debug_assert!(!p.starts_with("/") && !p.starts_with(".")); + if p.starts_with("usr/etc") { + p.strip_prefix("usr/").unwrap() + } else { + p + } +} + +struct OstreeTarWriter<'a, W: std::io::Write> { + repo: &'a ostree::Repo, + commit_checksum: &'a str, + commit_object: glib::Variant, + out: &'a mut tar::Builder, + #[allow(dead_code)] + options: ExportOptions, + wrote_initdirs: bool, + /// True if we're only writing directories + structure_only: bool, + wrote_vartmp: bool, // Set if the ostree commit contains /var/tmp + wrote_dirtree: HashSet, + wrote_dirmeta: HashSet, + wrote_content: HashSet, + wrote_xattrs: HashSet, +} + +pub(crate) fn object_path(objtype: ostree::ObjectType, checksum: &str) -> Utf8PathBuf { + let suffix = match objtype { + ostree::ObjectType::Commit => "commit", + ostree::ObjectType::CommitMeta => "commitmeta", + ostree::ObjectType::DirTree => "dirtree", + ostree::ObjectType::DirMeta => "dirmeta", + ostree::ObjectType::File => "file", + o => panic!("Unexpected object type: {:?}", o), + }; + let (first, rest) = checksum.split_at(2); + format!("{}/repo/objects/{}/{}.{}", OSTREEDIR, first, rest, suffix).into() +} + +fn v1_xattrs_object_path(checksum: &str) -> Utf8PathBuf { + let (first, rest) = checksum.split_at(2); + format!("{}/repo/objects/{}/{}.file-xattrs", OSTREEDIR, first, rest).into() +} + +fn v1_xattrs_link_object_path(checksum: &str) -> Utf8PathBuf { + let (first, rest) = checksum.split_at(2); + format!( + "{}/repo/objects/{}/{}.file-xattrs-link", + OSTREEDIR, first, rest + ) + .into() +} + +/// Check for "denormal" symlinks which contain "//" +// See https://github.com/fedora-sysv/chkconfig/pull/67 +// [root@cosa-devsh ~]# rpm -qf /usr/lib/systemd/systemd-sysv-install +// chkconfig-1.13-2.el8.x86_64 +// [root@cosa-devsh ~]# ll /usr/lib/systemd/systemd-sysv-install +// lrwxrwxrwx. 2 root root 24 Nov 29 18:08 /usr/lib/systemd/systemd-sysv-install -> ../../..//sbin/chkconfig +// [root@cosa-devsh ~]# +fn symlink_is_denormal(target: &str) -> bool { + target.contains("//") +} + +pub(crate) fn tar_append_default_data( + out: &mut tar::Builder, + path: &Utf8Path, + buf: &[u8], +) -> Result<()> { + let mut h = tar::Header::new_gnu(); + h.set_entry_type(tar::EntryType::Regular); + h.set_uid(0); + h.set_gid(0); + h.set_mode(0o644); + h.set_size(buf.len() as u64); + out.append_data(&mut h, path, buf).map_err(Into::into) +} + +impl<'a, W: std::io::Write> OstreeTarWriter<'a, W> { + fn new( + repo: &'a ostree::Repo, + commit_checksum: &'a str, + out: &'a mut tar::Builder, + options: ExportOptions, + ) -> Result { + let commit_object = repo.load_commit(commit_checksum)?.0; + let r = Self { + repo, + commit_checksum, + commit_object, + out, + options, + wrote_initdirs: false, + structure_only: false, + wrote_vartmp: false, + wrote_dirmeta: HashSet::new(), + wrote_dirtree: HashSet::new(), + wrote_content: HashSet::new(), + wrote_xattrs: HashSet::new(), + }; + Ok(r) + } + + /// Convert the ostree mode to tar mode. + /// The ostree mode bits include the format, tar does not. + /// Historically in format version 0 we injected them, so we need to keep doing so. + fn filter_mode(&self, mode: u32) -> u32 { + mode & !libc::S_IFMT + } + + /// Add a directory entry with default permissions (root/root 0755) + fn append_default_dir(&mut self, path: &Utf8Path) -> Result<()> { + let mut h = tar::Header::new_gnu(); + h.set_entry_type(tar::EntryType::Directory); + h.set_uid(0); + h.set_gid(0); + h.set_mode(0o755); + h.set_size(0); + self.out.append_data(&mut h, path, &mut std::io::empty())?; + Ok(()) + } + + /// Add a regular file entry with default permissions (root/root 0644) + fn append_default_data(&mut self, path: &Utf8Path, buf: &[u8]) -> Result<()> { + tar_append_default_data(self.out, path, buf) + } + + /// Add an hardlink entry with default permissions (root/root 0644) + fn append_default_hardlink(&mut self, path: &Utf8Path, link_target: &Utf8Path) -> Result<()> { + let mut h = tar::Header::new_gnu(); + h.set_entry_type(tar::EntryType::Link); + h.set_uid(0); + h.set_gid(0); + h.set_mode(0o644); + h.set_size(0); + self.out.append_link(&mut h, path, link_target)?; + Ok(()) + } + + /// Write the initial /sysroot/ostree/repo structure. + fn write_repo_structure(&mut self) -> Result<()> { + if self.wrote_initdirs { + return Ok(()); + } + + let objdir: Utf8PathBuf = format!("{}/repo/objects", OSTREEDIR).into(); + // Add all parent directories + let parent_dirs = { + let mut parts: Vec<_> = objdir.ancestors().collect(); + parts.reverse(); + parts + }; + for path in parent_dirs { + match path.as_str() { + "/" | "" => continue, + _ => {} + } + self.append_default_dir(path)?; + } + // Object subdirectories + for d in 0..=0xFF { + let path: Utf8PathBuf = format!("{}/{:02x}", objdir, d).into(); + self.append_default_dir(&path)?; + } + // Standard repo subdirectories. + let subdirs = [ + "extensions", + "refs", + "refs/heads", + "refs/mirrors", + "refs/remotes", + "state", + "tmp", + "tmp/cache", + ]; + for d in subdirs { + let path: Utf8PathBuf = format!("{}/repo/{}", OSTREEDIR, d).into(); + self.append_default_dir(&path)?; + } + + // Repository configuration file. + { + let path = format!("{}/repo/config", OSTREEDIR); + self.append_default_data(Utf8Path::new(&path), REPO_CONFIG.as_bytes())?; + } + + self.wrote_initdirs = true; + Ok(()) + } + + /// Recursively serialize a commit object to the target tar stream. + fn write_commit(&mut self) -> Result<()> { + let cancellable = gio::Cancellable::NONE; + + let commit_bytes = self.commit_object.data_as_bytes(); + let commit_bytes = commit_bytes.try_as_aligned()?; + let commit = gv_commit!().cast(commit_bytes); + let commit = commit.to_tuple(); + let contents = hex::encode(commit.6); + let metadata_checksum = &hex::encode(commit.7); + let metadata_v = self + .repo + .load_variant(ostree::ObjectType::DirMeta, metadata_checksum)?; + // Safety: We passed the correct variant type just above + let metadata = &ostree::DirMetaParsed::from_variant(&metadata_v).unwrap(); + let rootpath = Utf8Path::new(TAR_PATH_PREFIX_V0); + + // We need to write the root directory, before we write any objects. This should be the very + // first thing. + self.append_dir(rootpath, metadata)?; + + // Now, we create sysroot/ and everything under it + self.write_repo_structure()?; + + self.append_commit_object()?; + + // The ostree dirmeta object for the root. + self.append(ostree::ObjectType::DirMeta, metadata_checksum, &metadata_v)?; + + // Recurse and write everything else. + self.append_dirtree( + Utf8Path::new(TAR_PATH_PREFIX_V0), + contents, + true, + cancellable, + )?; + + self.append_standard_var(cancellable)?; + + Ok(()) + } + + fn append_commit_object(&mut self) -> Result<()> { + self.append( + ostree::ObjectType::Commit, + self.commit_checksum, + &self.commit_object.clone(), + )?; + if let Some(commitmeta) = self + .repo + .read_commit_detached_metadata(self.commit_checksum, gio::Cancellable::NONE)? + { + self.append( + ostree::ObjectType::CommitMeta, + self.commit_checksum, + &commitmeta, + )?; + } + Ok(()) + } + + fn append( + &mut self, + objtype: ostree::ObjectType, + checksum: &str, + v: &glib::Variant, + ) -> Result<()> { + let set = match objtype { + ostree::ObjectType::Commit | ostree::ObjectType::CommitMeta => None, + ostree::ObjectType::DirTree => Some(&mut self.wrote_dirtree), + ostree::ObjectType::DirMeta => Some(&mut self.wrote_dirmeta), + o => panic!("Unexpected object type: {:?}", o), + }; + if let Some(set) = set { + if set.contains(checksum) { + return Ok(()); + } + let inserted = set.insert(checksum.to_string()); + debug_assert!(inserted); + } + + let data = v.data_as_bytes(); + let data = data.as_ref(); + self.append_default_data(&object_path(objtype, checksum), data) + .with_context(|| format!("Writing object {checksum}"))?; + Ok(()) + } + + /// Export xattrs to the tar stream, return whether content was written. + #[context("Writing xattrs")] + fn append_xattrs(&mut self, checksum: &str, xattrs: &glib::Variant) -> Result { + let xattrs_data = xattrs.data_as_bytes(); + let xattrs_data = xattrs_data.as_ref(); + + let xattrs_checksum = { + let digest = openssl::hash::hash(openssl::hash::MessageDigest::sha256(), xattrs_data)?; + hex::encode(digest) + }; + + let path = v1_xattrs_object_path(&xattrs_checksum); + // Write xattrs content into a separate `.file-xattrs` object. + if !self.wrote_xattrs.contains(&xattrs_checksum) { + let inserted = self.wrote_xattrs.insert(xattrs_checksum); + debug_assert!(inserted); + self.append_default_data(&path, xattrs_data)?; + } + // Write a `.file-xattrs-link` which links the file object to + // the corresponding detached xattrs. + { + let link_obj_path = v1_xattrs_link_object_path(checksum); + self.append_default_hardlink(&link_obj_path, &path)?; + } + + Ok(true) + } + + /// Write a content object, returning the path/header that should be used + /// as a hard link to it in the target path. This matches how ostree checkouts work. + fn append_content(&mut self, checksum: &str) -> Result<(Utf8PathBuf, tar::Header)> { + let path = object_path(ostree::ObjectType::File, checksum); + + let (instream, meta, xattrs) = self.repo.load_file(checksum, gio::Cancellable::NONE)?; + + let mut h = tar::Header::new_gnu(); + h.set_uid(meta.attribute_uint32("unix::uid") as u64); + h.set_gid(meta.attribute_uint32("unix::gid") as u64); + let mode = meta.attribute_uint32("unix::mode"); + h.set_mode(self.filter_mode(mode)); + if instream.is_some() { + h.set_size(meta.size() as u64); + } + if !self.wrote_content.contains(checksum) { + let inserted = self.wrote_content.insert(checksum.to_string()); + debug_assert!(inserted); + + // The xattrs objects need to be exported before the regular object they + // refer to. Otherwise the importing logic won't have the xattrs available + // when importing file content. + self.append_xattrs(checksum, &xattrs)?; + + if let Some(instream) = instream { + ensure!(meta.file_type() == gio::FileType::Regular); + + h.set_entry_type(tar::EntryType::Regular); + h.set_size(meta.size() as u64); + let mut instream = BufReader::with_capacity(BUF_CAPACITY, instream.into_read()); + self.out + .append_data(&mut h, &path, &mut instream) + .with_context(|| format!("Writing regfile {}", checksum))?; + } else { + ensure!(meta.file_type() == gio::FileType::SymbolicLink); + + let target = meta + .symlink_target() + .ok_or_else(|| anyhow!("Missing symlink target"))?; + let target = target + .to_str() + .ok_or_else(|| anyhow!("Invalid UTF-8 symlink target: {target:?}"))?; + let context = || format!("Writing content symlink: {}", checksum); + h.set_entry_type(tar::EntryType::Symlink); + h.set_size(0); + // Handle //chkconfig, see above + if symlink_is_denormal(target) { + h.set_link_name_literal(target).with_context(context)?; + self.out + .append_data(&mut h, &path, &mut std::io::empty()) + .with_context(context)?; + } else { + self.out + .append_link(&mut h, &path, target) + .with_context(context)?; + } + } + } + + Ok((path, h)) + } + + /// Write a directory using the provided metadata. + fn append_dir(&mut self, dirpath: &Utf8Path, meta: &ostree::DirMetaParsed) -> Result<()> { + let mut header = tar::Header::new_gnu(); + header.set_entry_type(tar::EntryType::Directory); + header.set_size(0); + header.set_uid(meta.uid as u64); + header.set_gid(meta.gid as u64); + header.set_mode(self.filter_mode(meta.mode)); + self.out + .append_data(&mut header, dirpath, std::io::empty())?; + Ok(()) + } + + /// Given a source object (in e.g. ostree/repo/objects/...), write a hardlink to it + /// in its expected target path (e.g. `usr/bin/bash`). + fn append_content_hardlink( + &mut self, + srcpath: &Utf8Path, + mut h: tar::Header, + dest: &Utf8Path, + ) -> Result<()> { + // Query the original size first + let size = h.size().context("Querying size for hardlink append")?; + // Don't create hardlinks to zero-sized files, it's much more likely + // to result in generated tar streams from container builds resulting + // in a modified linked-to file in /sysroot, which we don't currently handle. + // And in the case where the input is *not* zero sized, we still output + // a hardlink of size zero, as this is what is normal. + h.set_size(0); + if h.entry_type() == tar::EntryType::Regular && size == 0 { + self.out.append_data(&mut h, dest, &mut std::io::empty())?; + } else { + h.set_entry_type(tar::EntryType::Link); + h.set_link_name(srcpath)?; + self.out.append_data(&mut h, dest, &mut std::io::empty())?; + } + Ok(()) + } + + /// Write a dirtree object. + fn append_dirtree>( + &mut self, + dirpath: &Utf8Path, + checksum: String, + is_root: bool, + cancellable: Option<&C>, + ) -> Result<()> { + let v = &self + .repo + .load_variant(ostree::ObjectType::DirTree, &checksum)?; + self.append(ostree::ObjectType::DirTree, &checksum, v)?; + drop(checksum); + let v = v.data_as_bytes(); + let v = v.try_as_aligned()?; + let v = gv_dirtree!().cast(v); + let (files, dirs) = v.to_tuple(); + + if let Some(c) = cancellable { + c.set_error_if_cancelled()?; + } + + if !self.structure_only { + for file in files { + let (name, csum) = file.to_tuple(); + let name = name.to_str(); + let checksum = &hex::encode(csum); + let (objpath, h) = self.append_content(checksum)?; + let subpath = &dirpath.join(name); + let subpath = map_path(subpath); + self.append_content_hardlink(&objpath, h, &subpath)?; + } + } + + // Record if the ostree commit includes /var/tmp; if so we don't need to synthesize + // it in `append_standard_var()`. + if dirpath == "var/tmp" { + self.wrote_vartmp = true; + } + + for item in dirs { + let (name, contents_csum, meta_csum) = item.to_tuple(); + let name = name.to_str(); + let metadata = { + let meta_csum = &hex::encode(meta_csum); + let meta_v = &self + .repo + .load_variant(ostree::ObjectType::DirMeta, meta_csum)?; + self.append(ostree::ObjectType::DirMeta, meta_csum, meta_v)?; + // Safety: We passed the correct variant type just above + ostree::DirMetaParsed::from_variant(meta_v).unwrap() + }; + // Special hack because tar stream for containers can't have duplicates. + if is_root && name == SYSROOT { + continue; + } + let dirtree_csum = hex::encode(contents_csum); + let subpath = &dirpath.join(name); + let subpath = map_path(subpath); + self.append_dir(&subpath, &metadata)?; + self.append_dirtree(&subpath, dirtree_csum, false, cancellable)?; + } + + Ok(()) + } + + /// Generate e.g. `/var/tmp`. + /// + /// In the OSTree model we expect `/var` to start out empty, and be populated via + /// e.g. `systemd-tmpfiles`. But, systemd doesn't run in Docker-style containers by default. + /// + /// So, this function creates a few critical directories in `/var` by default. + fn append_standard_var(&mut self, cancellable: Option<&gio::Cancellable>) -> Result<()> { + // If the commit included /var/tmp, then it's already in the tar stream. + if self.wrote_vartmp { + return Ok(()); + } + if let Some(c) = cancellable { + c.set_error_if_cancelled()?; + } + let mut header = tar::Header::new_gnu(); + header.set_entry_type(tar::EntryType::Directory); + header.set_size(0); + header.set_uid(0); + header.set_gid(0); + header.set_mode(self.filter_mode(libc::S_IFDIR | 0o1777)); + self.out + .append_data(&mut header, "var/tmp", std::io::empty())?; + Ok(()) + } +} + +/// Recursively walk an OSTree commit and generate data into a `[tar::Builder]` +/// which contains all of the metadata objects, as well as a hardlinked +/// stream that looks like a checkout. Extended attributes are stored specially out +/// of band of tar so that they can be reliably retrieved. +fn impl_export( + repo: &ostree::Repo, + commit_checksum: &str, + out: &mut tar::Builder, + options: ExportOptions, +) -> Result<()> { + let writer = &mut OstreeTarWriter::new(repo, commit_checksum, out, options)?; + writer.write_commit()?; + Ok(()) +} + +/// Configuration for tar export. +#[derive(Debug, PartialEq, Eq, Default)] +pub struct ExportOptions; + +/// Export an ostree commit to an (uncompressed) tar archive stream. +#[context("Exporting commit")] +pub fn export_commit( + repo: &ostree::Repo, + rev: &str, + out: impl std::io::Write, + options: Option, +) -> Result<()> { + let commit = repo.require_rev(rev)?; + let mut tar = tar::Builder::new(out); + let options = options.unwrap_or_default(); + impl_export(repo, commit.as_str(), &mut tar, options)?; + tar.finish()?; + Ok(()) +} + +/// Chunked (or version 1) tar streams don't have a leading `./`. +fn path_for_tar_v1(p: &Utf8Path) -> &Utf8Path { + debug_assert!(!p.starts_with(".")); + map_path_v1(p.strip_prefix("/").unwrap_or(p)) +} + +/// Implementation of chunk writing, assumes that the preliminary structure +/// has been written to the tar stream. +fn write_chunk( + writer: &mut OstreeTarWriter, + chunk: chunking::ChunkMapping, +) -> Result<()> { + for (checksum, (_size, paths)) in chunk.into_iter() { + let (objpath, h) = writer.append_content(checksum.borrow())?; + for path in paths.iter() { + let path = path_for_tar_v1(path); + let h = h.clone(); + writer.append_content_hardlink(&objpath, h, path)?; + } + } + Ok(()) +} + +/// Output a chunk to a tar stream. +pub(crate) fn export_chunk( + repo: &ostree::Repo, + commit: &str, + chunk: chunking::ChunkMapping, + out: &mut tar::Builder, +) -> Result<()> { + // For chunking, we default to format version 1 + #[allow(clippy::needless_update)] + let opts = ExportOptions; + let writer = &mut OstreeTarWriter::new(repo, commit, out, opts)?; + writer.write_repo_structure()?; + write_chunk(writer, chunk) +} + +/// Output the last chunk in a chunking. +#[context("Exporting final chunk")] +pub(crate) fn export_final_chunk( + repo: &ostree::Repo, + commit_checksum: &str, + remainder: chunking::Chunk, + out: &mut tar::Builder, +) -> Result<()> { + let options = ExportOptions; + let writer = &mut OstreeTarWriter::new(repo, commit_checksum, out, options)?; + // For the final chunk, output the commit object, plus all ostree metadata objects along with + // the containing directories. + writer.structure_only = true; + writer.write_commit()?; + writer.structure_only = false; + write_chunk(writer, remainder.content) +} + +/// Process an exported tar stream, and update the detached metadata. +#[allow(clippy::while_let_on_iterator)] +#[context("Replacing detached metadata")] +pub(crate) fn reinject_detached_metadata>( + src: &mut tar::Archive, + dest: &mut tar::Builder, + detached_buf: Option<&[u8]>, + cancellable: Option<&C>, +) -> Result<()> { + let mut entries = src.entries()?; + let mut commit_ent = None; + // Loop through the tar stream until we find the commit object; copy all prior entries + // such as the baseline directory structure. + while let Some(entry) = entries.next() { + if let Some(c) = cancellable { + c.set_error_if_cancelled()?; + } + let entry = entry?; + let header = entry.header(); + let path = entry.path()?; + let path: &Utf8Path = (&*path).try_into()?; + if !(header.entry_type() == tar::EntryType::Regular && path.as_str().ends_with(".commit")) { + crate::tar::write::copy_entry(entry, dest, None)?; + } else { + commit_ent = Some(entry); + break; + } + } + let commit_ent = commit_ent.ok_or_else(|| anyhow!("Missing commit object"))?; + let commit_path = commit_ent.path()?; + let commit_path = Utf8Path::from_path(&commit_path) + .ok_or_else(|| anyhow!("Invalid non-utf8 path {:?}", commit_path))?; + let (checksum, objtype) = crate::tar::import::Importer::parse_metadata_entry(commit_path)?; + assert_eq!(objtype, ostree::ObjectType::Commit); // Should have been verified above + crate::tar::write::copy_entry(commit_ent, dest, None)?; + + // If provided, inject our new detached metadata object + if let Some(detached_buf) = detached_buf { + let detached_path = object_path(ostree::ObjectType::CommitMeta, &checksum); + tar_append_default_data(dest, &detached_path, detached_buf)?; + } + + // If the next entry is detached metadata, then drop it since we wrote a new one + let next_ent = entries + .next() + .ok_or_else(|| anyhow!("Expected metadata object after commit"))??; + let next_ent_path = next_ent.path()?; + let next_ent_path: &Utf8Path = (&*next_ent_path).try_into()?; + let objtype = crate::tar::import::Importer::parse_metadata_entry(next_ent_path)?.1; + if objtype != ostree::ObjectType::CommitMeta { + crate::tar::write::copy_entry(next_ent, dest, None)?; + } + + // Finally, copy all remaining entries. + while let Some(entry) = entries.next() { + if let Some(c) = cancellable { + c.set_error_if_cancelled()?; + } + crate::tar::write::copy_entry(entry?, dest, None)?; + } + + Ok(()) +} + +/// Replace the detached metadata in an tar stream which is an export of an OSTree commit. +pub fn update_detached_metadata>( + src: impl std::io::Read, + dest: D, + detached_buf: Option<&[u8]>, + cancellable: Option<&C>, +) -> Result { + let mut src = tar::Archive::new(src); + let mut dest = tar::Builder::new(dest); + reinject_detached_metadata(&mut src, &mut dest, detached_buf, cancellable)?; + dest.into_inner().map_err(Into::into) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_map_path() { + assert_eq!(map_path("/".into()), Utf8Path::new("/")); + assert_eq!( + map_path("./usr/etc/blah".into()), + Utf8Path::new("./etc/blah") + ); + for unchanged in ["boot", "usr/bin", "usr/lib/foo"].iter().map(Utf8Path::new) { + assert_eq!(unchanged, map_path_v1(unchanged)); + } + + assert_eq!(Utf8Path::new("etc"), map_path_v1(Utf8Path::new("usr/etc"))); + assert_eq!( + Utf8Path::new("etc/foo"), + map_path_v1(Utf8Path::new("usr/etc/foo")) + ); + } + + #[test] + fn test_denormal_symlink() { + let normal = ["/", "/usr", "../usr/bin/blah"]; + let denormal = ["../../usr/sbin//chkconfig", "foo//bar/baz"]; + for path in normal { + assert!(!symlink_is_denormal(path)); + } + for path in denormal { + assert!(symlink_is_denormal(path)); + } + } + + #[test] + fn test_v1_xattrs_object_path() { + let checksum = "b8627e3ef0f255a322d2bd9610cfaaacc8f122b7f8d17c0e7e3caafa160f9fc7"; + let expected = "sysroot/ostree/repo/objects/b8/627e3ef0f255a322d2bd9610cfaaacc8f122b7f8d17c0e7e3caafa160f9fc7.file-xattrs"; + let output = v1_xattrs_object_path(checksum); + assert_eq!(&output, expected); + } + + #[test] + fn test_v1_xattrs_link_object_path() { + let checksum = "b8627e3ef0f255a322d2bd9610cfaaacc8f122b7f8d17c0e7e3caafa160f9fc7"; + let expected = "sysroot/ostree/repo/objects/b8/627e3ef0f255a322d2bd9610cfaaacc8f122b7f8d17c0e7e3caafa160f9fc7.file-xattrs-link"; + let output = v1_xattrs_link_object_path(checksum); + assert_eq!(&output, expected); + } +} diff --git a/ostree-ext/src/tar/import.rs b/ostree-ext/src/tar/import.rs new file mode 100644 index 00000000..a251937d --- /dev/null +++ b/ostree-ext/src/tar/import.rs @@ -0,0 +1,932 @@ +//! APIs for extracting OSTree commits from container images + +use crate::Result; +use anyhow::{anyhow, bail, ensure, Context}; +use camino::Utf8Path; +use camino::Utf8PathBuf; +use fn_error_context::context; +use gio::glib; +use gio::prelude::*; +use glib::Variant; +use ostree::gio; +use std::collections::BTreeSet; +use std::collections::HashMap; +use std::io::prelude::*; +use tracing::{event, instrument, Level}; + +/// Arbitrary limit on xattrs to avoid RAM exhaustion attacks. The actual filesystem limits are often much smaller. +// See https://en.wikipedia.org/wiki/Extended_file_attributes +// For example, XFS limits to 614 KiB. +const MAX_XATTR_SIZE: u32 = 1024 * 1024; +/// Limit on metadata objects (dirtree/dirmeta); this is copied +/// from ostree-core.h. TODO: Bind this in introspection +const MAX_METADATA_SIZE: u32 = 10 * 1024 * 1024; + +/// Upper size limit for "small" regular files. +// https://stackoverflow.com/questions/258091/when-should-i-use-mmap-for-file-access +pub(crate) const SMALL_REGFILE_SIZE: usize = 127 * 1024; + +// The prefix for filenames that contain content we actually look at. +pub(crate) const REPO_PREFIX: &str = "sysroot/ostree/repo/"; +/// Statistics from import. +#[derive(Debug, Default)] +struct ImportStats { + dirtree: u32, + dirmeta: u32, + regfile_small: u32, + regfile_large: u32, + symlinks: u32, +} + +enum ImporterMode { + Commit(Option), + ObjectSet(BTreeSet), +} + +/// Importer machine. +pub(crate) struct Importer { + repo: ostree::Repo, + remote: Option, + // Cache of xattrs, keyed by their content checksum. + xattrs: HashMap, + // Reusable buffer for xattrs references. It maps a file checksum (.0) + // to an xattrs checksum (.1) in the `xattrs` cache above. + next_xattrs: Option<(String, String)>, + + // Reusable buffer for reads. See also https://github.com/rust-lang/rust/issues/78485 + buf: Vec, + + stats: ImportStats, + + /// Additional state depending on whether we're importing an object set or a commit. + data: ImporterMode, +} + +/// Validate size/type of a tar header for OSTree metadata object. +fn validate_metadata_header(header: &tar::Header, desc: &str) -> Result { + if header.entry_type() != tar::EntryType::Regular { + return Err(anyhow!("Invalid non-regular metadata object {}", desc)); + } + let size = header.size()?; + let max_size = MAX_METADATA_SIZE as u64; + if size > max_size { + return Err(anyhow!( + "object of size {} exceeds {} bytes", + size, + max_size + )); + } + Ok(size as usize) +} + +fn header_attrs(header: &tar::Header) -> Result<(u32, u32, u32)> { + let uid: u32 = header.uid()?.try_into()?; + let gid: u32 = header.gid()?.try_into()?; + let mode: u32 = header.mode()?; + Ok((uid, gid, mode)) +} + +// The C function ostree_object_type_from_string aborts on +// unknown strings, so we have a safe version here. +fn objtype_from_string(t: &str) -> Option { + Some(match t { + "commit" => ostree::ObjectType::Commit, + "commitmeta" => ostree::ObjectType::CommitMeta, + "dirtree" => ostree::ObjectType::DirTree, + "dirmeta" => ostree::ObjectType::DirMeta, + "file" => ostree::ObjectType::File, + _ => return None, + }) +} + +/// Given a tar entry, read it all into a GVariant +fn entry_to_variant( + mut entry: tar::Entry, + desc: &str, +) -> Result { + let header = entry.header(); + let size = validate_metadata_header(header, desc)?; + + let mut buf: Vec = Vec::with_capacity(size); + let n = std::io::copy(&mut entry, &mut buf)?; + assert_eq!(n as usize, size); + let v = glib::Bytes::from_owned(buf); + let v = Variant::from_bytes::(&v); + Ok(v.normal_form()) +} + +/// Parse an object path into (parent, rest, objtype). +/// +/// Normal ostree object paths look like 00/1234.commit. +/// In the tar format, we may also see 00/1234.file.xattrs. +fn parse_object_entry_path(path: &Utf8Path) -> Result<(&str, &Utf8Path, &str)> { + // The "sharded" commit directory. + let parentname = path + .parent() + .and_then(|p| p.file_name()) + .ok_or_else(|| anyhow!("Invalid path (no parent) {}", path))?; + if parentname.len() != 2 { + return Err(anyhow!("Invalid checksum parent {}", parentname)); + } + let name = path + .file_name() + .map(Utf8Path::new) + .ok_or_else(|| anyhow!("Invalid path (dir) {}", path))?; + let objtype = name + .extension() + .ok_or_else(|| anyhow!("Invalid objpath {}", path))?; + + Ok((parentname, name, objtype)) +} + +fn parse_checksum(parent: &str, name: &Utf8Path) -> Result { + let checksum_rest = name + .file_stem() + .ok_or_else(|| anyhow!("Invalid object path part {}", name))?; + // Also take care of the double extension on `.file.xattrs`. + let checksum_rest = checksum_rest.trim_end_matches(".file"); + + if checksum_rest.len() != 62 { + return Err(anyhow!("Invalid checksum part {}", checksum_rest)); + } + let reassembled = format!("{}{}", parent, checksum_rest); + validate_sha256(reassembled) +} + +/// Parse a `.file-xattrs-link` link target into the corresponding checksum. +fn parse_xattrs_link_target(path: &Utf8Path) -> Result { + let (parent, rest, _objtype) = parse_object_entry_path(path)?; + parse_checksum(parent, rest) +} + +impl Importer { + /// Create an importer which will import an OSTree commit object. + pub(crate) fn new_for_commit(repo: &ostree::Repo, remote: Option) -> Self { + Self { + repo: repo.clone(), + remote, + buf: vec![0u8; 16384], + xattrs: Default::default(), + next_xattrs: None, + stats: Default::default(), + data: ImporterMode::Commit(None), + } + } + + /// Create an importer to write an "object set"; a chunk of objects which is + /// usually streamed from a separate storage system, such as an OCI container image layer. + pub(crate) fn new_for_object_set(repo: &ostree::Repo) -> Self { + Self { + repo: repo.clone(), + remote: None, + buf: vec![0u8; 16384], + xattrs: Default::default(), + next_xattrs: None, + stats: Default::default(), + data: ImporterMode::ObjectSet(Default::default()), + } + } + + // Given a tar entry, filter it out if it doesn't look like an object file in + // `/sysroot/ostree`. + // It is an error if the filename is invalid UTF-8. If it is valid UTF-8, return + // an owned copy of the path. + fn filter_entry( + e: tar::Entry, + ) -> Result, Utf8PathBuf)>> { + if e.header().entry_type() == tar::EntryType::Directory { + return Ok(None); + } + let orig_path = e.path()?; + let path = Utf8Path::from_path(&orig_path) + .ok_or_else(|| anyhow!("Invalid non-utf8 path {:?}", orig_path))?; + // Ignore the regular non-object file hardlinks we inject + if let Ok(path) = path.strip_prefix(REPO_PREFIX) { + // Filter out the repo config file and refs dir + if path.file_name() == Some("config") || path.starts_with("refs") { + return Ok(None); + } + let path = path.into(); + Ok(Some((e, path))) + } else { + Ok(None) + } + } + + pub(crate) fn parse_metadata_entry(path: &Utf8Path) -> Result<(String, ostree::ObjectType)> { + let (parentname, name, objtype) = parse_object_entry_path(path)?; + let checksum = parse_checksum(parentname, name)?; + let objtype = objtype_from_string(objtype) + .ok_or_else(|| anyhow!("Invalid object type {}", objtype))?; + Ok((checksum, objtype)) + } + + /// Import a metadata object. + #[context("Importing metadata object")] + fn import_metadata( + &mut self, + entry: tar::Entry, + checksum: &str, + objtype: ostree::ObjectType, + ) -> Result<()> { + let v = match objtype { + ostree::ObjectType::DirTree => { + self.stats.dirtree += 1; + entry_to_variant::<_, ostree::TreeVariantType>(entry, checksum)? + } + ostree::ObjectType::DirMeta => { + self.stats.dirmeta += 1; + entry_to_variant::<_, ostree::DirmetaVariantType>(entry, checksum)? + } + o => return Err(anyhow!("Invalid metadata object type; {:?}", o)), + }; + // FIXME validate here that this checksum was in the set we expected. + // https://github.com/ostreedev/ostree-rs-ext/issues/1 + let actual = + self.repo + .write_metadata(objtype, Some(checksum), &v, gio::Cancellable::NONE)?; + assert_eq!(actual.to_hex(), checksum); + Ok(()) + } + + /// Import a content object, large regular file flavour. + #[context("Importing regfile")] + fn import_large_regfile_object( + &mut self, + mut entry: tar::Entry, + size: usize, + checksum: &str, + xattrs: glib::Variant, + cancellable: Option<&gio::Cancellable>, + ) -> Result<()> { + let (uid, gid, mode) = header_attrs(entry.header())?; + let w = self.repo.write_regfile( + Some(checksum), + uid, + gid, + libc::S_IFREG | mode, + size as u64, + Some(&xattrs), + )?; + { + let w = w.clone().upcast::(); + loop { + let n = entry + .read(&mut self.buf[..]) + .context("Reading large regfile")?; + if n == 0 { + break; + } + w.write(&self.buf[0..n], cancellable) + .context("Writing large regfile")?; + } + } + let c = w.finish(cancellable)?; + debug_assert_eq!(c, checksum); + self.stats.regfile_large += 1; + Ok(()) + } + + /// Import a content object, small regular file flavour. + #[context("Importing regfile small")] + fn import_small_regfile_object( + &mut self, + mut entry: tar::Entry, + size: usize, + checksum: &str, + xattrs: glib::Variant, + cancellable: Option<&gio::Cancellable>, + ) -> Result<()> { + let (uid, gid, mode) = header_attrs(entry.header())?; + assert!(size <= SMALL_REGFILE_SIZE); + let mut buf = vec![0u8; size]; + entry.read_exact(&mut buf[..])?; + let c = self.repo.write_regfile_inline( + Some(checksum), + uid, + gid, + libc::S_IFREG | mode, + Some(&xattrs), + &buf, + cancellable, + )?; + debug_assert_eq!(c.as_str(), checksum); + self.stats.regfile_small += 1; + Ok(()) + } + + /// Import a content object, symlink flavour. + #[context("Importing symlink")] + fn import_symlink_object( + &mut self, + entry: tar::Entry, + checksum: &str, + xattrs: glib::Variant, + ) -> Result<()> { + let (uid, gid, _) = header_attrs(entry.header())?; + let target = entry + .link_name()? + .ok_or_else(|| anyhow!("Invalid symlink"))?; + let target = target + .as_os_str() + .to_str() + .ok_or_else(|| anyhow!("Non-utf8 symlink"))?; + let c = self.repo.write_symlink( + Some(checksum), + uid, + gid, + Some(&xattrs), + target, + gio::Cancellable::NONE, + )?; + debug_assert_eq!(c.as_str(), checksum); + self.stats.symlinks += 1; + Ok(()) + } + + /// Import a content object. + #[context("Processing content object {}", checksum)] + fn import_content_object( + &mut self, + entry: tar::Entry, + checksum: &str, + cancellable: Option<&gio::Cancellable>, + ) -> Result<()> { + let size: usize = entry.header().size()?.try_into()?; + + // Pop the queued xattrs reference. + let (file_csum, xattrs_csum) = self + .next_xattrs + .take() + .ok_or_else(|| anyhow!("Missing xattrs reference"))?; + if checksum != file_csum { + return Err(anyhow!("Object mismatch, found xattrs for {}", file_csum)); + } + + if self + .repo + .has_object(ostree::ObjectType::File, checksum, cancellable)? + { + return Ok(()); + } + + // Retrieve xattrs content from the cache. + let xattrs = self + .xattrs + .get(&xattrs_csum) + .cloned() + .ok_or_else(|| anyhow!("Failed to find xattrs content {}", xattrs_csum,))?; + + match entry.header().entry_type() { + tar::EntryType::Regular => { + if size > SMALL_REGFILE_SIZE { + self.import_large_regfile_object(entry, size, checksum, xattrs, cancellable) + } else { + self.import_small_regfile_object(entry, size, checksum, xattrs, cancellable) + } + } + tar::EntryType::Symlink => self.import_symlink_object(entry, checksum, xattrs), + o => Err(anyhow!("Invalid tar entry of type {:?}", o)), + } + } + + /// Given a tar entry that looks like an object (its path is under ostree/repo/objects/), + /// determine its type and import it. + #[context("Importing object {}", path)] + fn import_object( + &mut self, + entry: tar::Entry<'_, R>, + path: &Utf8Path, + cancellable: Option<&gio::Cancellable>, + ) -> Result<()> { + let (parentname, name, suffix) = parse_object_entry_path(path)?; + let checksum = parse_checksum(parentname, name)?; + + match suffix { + "commit" => Err(anyhow!("Found multiple commit objects")), + "file" => { + self.import_content_object(entry, &checksum, cancellable)?; + // Track the objects we wrote + match &mut self.data { + ImporterMode::ObjectSet(imported) => { + if let Some(p) = imported.replace(checksum) { + anyhow::bail!("Duplicate object: {}", p); + } + } + ImporterMode::Commit(_) => {} + } + Ok(()) + } + "file-xattrs" => self.process_file_xattrs(entry, checksum), + "file-xattrs-link" => self.process_file_xattrs_link(entry, checksum), + "xattrs" => self.process_xattr_ref(entry, checksum), + kind => { + let objtype = objtype_from_string(kind) + .ok_or_else(|| anyhow!("Invalid object type {}", kind))?; + match &mut self.data { + ImporterMode::ObjectSet(_) => { + anyhow::bail!( + "Found metadata object {}.{} in object set mode", + checksum, + objtype + ); + } + ImporterMode::Commit(_) => {} + } + self.import_metadata(entry, &checksum, objtype) + } + } + } + + /// Process a `.file-xattrs` object (v1). + #[context("Processing file xattrs")] + fn process_file_xattrs( + &mut self, + entry: tar::Entry, + checksum: String, + ) -> Result<()> { + self.cache_xattrs_content(entry, Some(checksum))?; + Ok(()) + } + + /// Process a `.file-xattrs-link` object (v1). + /// + /// This is an hardlink that contains extended attributes for a content object. + /// When the max hardlink count is reached, this object may also be encoded as + /// a regular file instead. + #[context("Processing xattrs link")] + fn process_file_xattrs_link( + &mut self, + entry: tar::Entry, + checksum: String, + ) -> Result<()> { + use tar::EntryType::{Link, Regular}; + if let Some(prev) = &self.next_xattrs { + bail!( + "Found previous dangling xattrs for file object '{}'", + prev.0 + ); + } + + // Extract the xattrs checksum from the link target or from the content (v1). + // Later, it will be used as the key for a lookup into the `self.xattrs` cache. + let xattrs_checksum = match entry.header().entry_type() { + Link => { + let link_target = entry + .link_name()? + .ok_or_else(|| anyhow!("No xattrs link content for {}", checksum))?; + let xattr_target = Utf8Path::from_path(&link_target) + .ok_or_else(|| anyhow!("Invalid non-UTF8 xattrs link {}", checksum))?; + parse_xattrs_link_target(xattr_target)? + } + Regular => self.cache_xattrs_content(entry, None)?, + x => bail!("Unexpected xattrs type '{:?}' found for {}", x, checksum), + }; + + // Now xattrs are properly cached for the next content object in the stream, + // which should match `checksum`. + self.next_xattrs = Some((checksum, xattrs_checksum)); + + Ok(()) + } + + /// Process a `.file.xattrs` entry (v0). + /// + /// This is an hardlink that contains extended attributes for a content object. + #[context("Processing xattrs reference")] + fn process_xattr_ref( + &mut self, + entry: tar::Entry, + target: String, + ) -> Result<()> { + if let Some(prev) = &self.next_xattrs { + bail!( + "Found previous dangling xattrs for file object '{}'", + prev.0 + ); + } + + // Parse the xattrs checksum from the link target (v0). + // Later, it will be used as the key for a lookup into the `self.xattrs` cache. + let header = entry.header(); + if header.entry_type() != tar::EntryType::Link { + bail!("Non-hardlink xattrs reference found for {}", target); + } + let xattr_target = entry + .link_name()? + .ok_or_else(|| anyhow!("No xattrs link content for {}", target))?; + let xattr_target = Utf8Path::from_path(&xattr_target) + .ok_or_else(|| anyhow!("Invalid non-UTF8 xattrs link {}", target))?; + let xattr_target = xattr_target + .file_name() + .ok_or_else(|| anyhow!("Invalid xattrs link {}", target))? + .to_string(); + let xattrs_checksum = validate_sha256(xattr_target)?; + + // Now xattrs are properly cached for the next content object in the stream, + // which should match `checksum`. + self.next_xattrs = Some((target, xattrs_checksum)); + + Ok(()) + } + + /// Process a special /xattrs/ entry, with checksum of xattrs content (v0). + fn process_split_xattrs_content( + &mut self, + entry: tar::Entry, + ) -> Result<()> { + let checksum = { + let path = entry.path()?; + let name = path + .file_name() + .ok_or_else(|| anyhow!("Invalid xattrs dir: {:?}", path))?; + let name = name + .to_str() + .ok_or_else(|| anyhow!("Invalid non-UTF8 xattrs name: {:?}", name))?; + validate_sha256(name.to_string())? + }; + self.cache_xattrs_content(entry, Some(checksum))?; + Ok(()) + } + + /// Read an xattrs entry and cache its content, optionally validating its checksum. + /// + /// This returns the computed checksum for the successfully cached content. + fn cache_xattrs_content( + &mut self, + mut entry: tar::Entry, + expected_checksum: Option, + ) -> Result { + let header = entry.header(); + if header.entry_type() != tar::EntryType::Regular { + return Err(anyhow!( + "Invalid xattr entry of type {:?}", + header.entry_type() + )); + } + let n = header.size()?; + if n > MAX_XATTR_SIZE as u64 { + return Err(anyhow!("Invalid xattr size {}", n)); + } + + let mut contents = vec![0u8; n as usize]; + entry.read_exact(contents.as_mut_slice())?; + let data: glib::Bytes = contents.as_slice().into(); + let xattrs_checksum = { + let digest = openssl::hash::hash(openssl::hash::MessageDigest::sha256(), &data)?; + hex::encode(digest) + }; + if let Some(input) = expected_checksum { + ensure!( + input == xattrs_checksum, + "Checksum mismatch, expected '{}' but computed '{}'", + input, + xattrs_checksum + ); + } + + let contents = Variant::from_bytes::<&[(&[u8], &[u8])]>(&data); + self.xattrs.insert(xattrs_checksum.clone(), contents); + Ok(xattrs_checksum) + } + + fn import_objects_impl<'a>( + &mut self, + ents: impl Iterator, Utf8PathBuf)>>, + cancellable: Option<&gio::Cancellable>, + ) -> Result<()> { + for entry in ents { + let (entry, path) = entry?; + if let Ok(p) = path.strip_prefix("objects/") { + self.import_object(entry, p, cancellable)?; + } else if path.strip_prefix("xattrs/").is_ok() { + self.process_split_xattrs_content(entry)?; + } + } + Ok(()) + } + + #[context("Importing objects")] + pub(crate) fn import_objects( + &mut self, + archive: &mut tar::Archive, + cancellable: Option<&gio::Cancellable>, + ) -> Result<()> { + let ents = archive.entries()?.filter_map(|e| match e { + Ok(e) => Self::filter_entry(e).transpose(), + Err(e) => Some(Err(anyhow::Error::msg(e))), + }); + self.import_objects_impl(ents, cancellable) + } + + #[context("Importing commit")] + pub(crate) fn import_commit( + &mut self, + archive: &mut tar::Archive, + cancellable: Option<&gio::Cancellable>, + ) -> Result<()> { + // This can only be invoked once + assert!(matches!(self.data, ImporterMode::Commit(None))); + // Create an iterator that skips over directories; we just care about the file names. + let mut ents = archive.entries()?.filter_map(|e| match e { + Ok(e) => Self::filter_entry(e).transpose(), + Err(e) => Some(Err(anyhow::Error::msg(e))), + }); + // Read the commit object. + let (commit_ent, commit_path) = ents + .next() + .ok_or_else(|| anyhow!("Commit object not found"))??; + + if commit_ent.header().entry_type() != tar::EntryType::Regular { + return Err(anyhow!( + "Expected regular file for commit object, not {:?}", + commit_ent.header().entry_type() + )); + } + let (checksum, objtype) = Self::parse_metadata_entry(&commit_path)?; + if objtype != ostree::ObjectType::Commit { + return Err(anyhow!("Expected commit object, not {:?}", objtype)); + } + let commit = entry_to_variant::<_, ostree::CommitVariantType>(commit_ent, &checksum)?; + + let (next_ent, nextent_path) = ents + .next() + .ok_or_else(|| anyhow!("End of stream after commit object"))??; + let (next_checksum, next_objtype) = Self::parse_metadata_entry(&nextent_path)?; + + if let Some(remote) = self.remote.as_deref() { + if next_objtype != ostree::ObjectType::CommitMeta { + return Err(anyhow!( + "Using remote {} for verification; Expected commitmeta object, not {:?}", + remote, + next_objtype + )); + } + if next_checksum != checksum { + return Err(anyhow!( + "Expected commitmeta checksum {}, found {}", + checksum, + next_checksum + )); + } + let commitmeta = entry_to_variant::<_, std::collections::HashMap>( + next_ent, + &next_checksum, + )?; + + // Now that we have both the commit and detached metadata in memory, verify that + // the signatures in the detached metadata correctly sign the commit. + self.repo + .signature_verify_commit_data( + remote, + &commit.data_as_bytes(), + &commitmeta.data_as_bytes(), + ostree::RepoVerifyFlags::empty(), + ) + .context("Verifying ostree commit in tar stream")?; + + self.repo.mark_commit_partial(&checksum, true)?; + + // Write the commit object, which also verifies its checksum. + let actual_checksum = + self.repo + .write_metadata(objtype, Some(&checksum), &commit, cancellable)?; + assert_eq!(actual_checksum.to_hex(), checksum); + event!(Level::DEBUG, "Imported {}.commit", checksum); + + // Finally, write the detached metadata. + self.repo + .write_commit_detached_metadata(&checksum, Some(&commitmeta), cancellable)?; + } else { + self.repo.mark_commit_partial(&checksum, true)?; + + // We're not doing any validation of the commit, so go ahead and write it. + let actual_checksum = + self.repo + .write_metadata(objtype, Some(&checksum), &commit, cancellable)?; + assert_eq!(actual_checksum.to_hex(), checksum); + event!(Level::DEBUG, "Imported {}.commit", checksum); + + // Write the next object, whether it's commit metadata or not. + let (meta_checksum, meta_objtype) = Self::parse_metadata_entry(&nextent_path)?; + match meta_objtype { + ostree::ObjectType::CommitMeta => { + let commitmeta = entry_to_variant::< + _, + std::collections::HashMap, + >(next_ent, &meta_checksum)?; + self.repo.write_commit_detached_metadata( + &checksum, + Some(&commitmeta), + gio::Cancellable::NONE, + )?; + } + _ => { + self.import_object(next_ent, &nextent_path, cancellable)?; + } + } + } + match &mut self.data { + ImporterMode::Commit(c) => { + c.replace(checksum); + } + ImporterMode::ObjectSet(_) => unreachable!(), + } + + self.import_objects_impl(ents, cancellable)?; + + Ok(()) + } + + pub(crate) fn finish_import_commit(self) -> String { + tracing::debug!("Import stats: {:?}", self.stats); + match self.data { + ImporterMode::Commit(c) => c.unwrap(), + ImporterMode::ObjectSet(_) => unreachable!(), + } + } + + pub(crate) fn default_dirmeta() -> glib::Variant { + let finfo = gio::FileInfo::new(); + finfo.set_attribute_uint32("unix::uid", 0); + finfo.set_attribute_uint32("unix::gid", 0); + finfo.set_attribute_uint32("unix::mode", libc::S_IFDIR | 0o755); + // SAFETY: TODO: This is not a nullable return, fix it in ostree + ostree::create_directory_metadata(&finfo, None) + } + + pub(crate) fn finish_import_object_set(self) -> Result { + let objset = match self.data { + ImporterMode::Commit(_) => unreachable!(), + ImporterMode::ObjectSet(s) => s, + }; + tracing::debug!("Imported {} content objects", objset.len()); + let mtree = ostree::MutableTree::new(); + for checksum in objset.into_iter() { + mtree.replace_file(&checksum, &checksum)?; + } + let dirmeta = self.repo.write_metadata( + ostree::ObjectType::DirMeta, + None, + &Self::default_dirmeta(), + gio::Cancellable::NONE, + )?; + mtree.set_metadata_checksum(&dirmeta.to_hex()); + let tree = self.repo.write_mtree(&mtree, gio::Cancellable::NONE)?; + let commit = self.repo.write_commit_with_time( + None, + None, + None, + None, + tree.downcast_ref().unwrap(), + 0, + gio::Cancellable::NONE, + )?; + Ok(commit.to_string()) + } +} + +fn validate_sha256(input: String) -> Result { + if input.len() != 64 { + return Err(anyhow!("Invalid sha256 checksum (len) {}", input)); + } + if !input.chars().all(|c| matches!(c, '0'..='9' | 'a'..='f')) { + return Err(anyhow!("Invalid sha256 checksum {}", input)); + } + Ok(input) +} + +/// Configuration for tar import. +#[derive(Debug, Default)] +#[non_exhaustive] +pub struct TarImportOptions { + /// Name of the remote to use for signature verification. + pub remote: Option, +} + +/// Read the contents of a tarball and import the ostree commit inside. +/// Returns the sha256 of the imported commit. +#[instrument(level = "debug", skip_all)] +pub async fn import_tar( + repo: &ostree::Repo, + src: impl tokio::io::AsyncRead + Send + Unpin + 'static, + options: Option, +) -> Result { + let options = options.unwrap_or_default(); + let src = tokio_util::io::SyncIoBridge::new(src); + let repo = repo.clone(); + // The tar code we use today is blocking, so we spawn a thread. + crate::tokio_util::spawn_blocking_cancellable_flatten(move |cancellable| { + let mut archive = tar::Archive::new(src); + let txn = repo.auto_transaction(Some(cancellable))?; + let mut importer = Importer::new_for_commit(&repo, options.remote); + importer.import_commit(&mut archive, Some(cancellable))?; + let checksum = importer.finish_import_commit(); + txn.commit(Some(cancellable))?; + repo.mark_commit_partial(&checksum, false)?; + Ok::<_, anyhow::Error>(checksum) + }) + .await +} + +/// Read the contents of a tarball and import the content objects inside. +/// Generates a synthetic commit object referencing them. +#[instrument(level = "debug", skip_all)] +pub async fn import_tar_objects( + repo: &ostree::Repo, + src: impl tokio::io::AsyncRead + Send + Unpin + 'static, +) -> Result { + let src = tokio_util::io::SyncIoBridge::new(src); + let repo = repo.clone(); + // The tar code we use today is blocking, so we spawn a thread. + crate::tokio_util::spawn_blocking_cancellable_flatten(move |cancellable| { + let mut archive = tar::Archive::new(src); + let mut importer = Importer::new_for_object_set(&repo); + let txn = repo.auto_transaction(Some(cancellable))?; + importer.import_objects(&mut archive, Some(cancellable))?; + let r = importer.finish_import_object_set()?; + txn.commit(Some(cancellable))?; + Ok::<_, anyhow::Error>(r) + }) + .await +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_metadata_entry() { + let c = "a8/6d80a3e9ff77c2e3144c787b7769b300f91ffd770221aac27bab854960b964"; + let invalid = format!("{}.blah", c); + for &k in &["", "42", c, &invalid] { + assert!(Importer::parse_metadata_entry(k.into()).is_err()) + } + let valid = format!("{}.commit", c); + let r = Importer::parse_metadata_entry(valid.as_str().into()).unwrap(); + assert_eq!(r.0, c.replace('/', "")); + assert_eq!(r.1, ostree::ObjectType::Commit); + } + + #[test] + fn test_validate_sha256() { + let err_cases = &[ + "a86d80a3e9ff77c2e3144c787b7769b300f91ffd770221aac27bab854960b9644", + "a86d80a3E9ff77c2e3144c787b7769b300f91ffd770221aac27bab854960b964", + ]; + for input in err_cases { + validate_sha256(input.to_string()).unwrap_err(); + } + + validate_sha256( + "a86d80a3e9ff77c2e3144c787b7769b300f91ffd770221aac27bab854960b964".to_string(), + ) + .unwrap(); + } + + #[test] + fn test_parse_object_entry_path() { + let path = + "sysroot/ostree/repo/objects/b8/627e3ef0f255a322d2bd9610cfaaacc8f122b7f8d17c0e7e3caafa160f9fc7.file.xattrs"; + let input = Utf8PathBuf::from(path); + let expected_parent = "b8"; + let expected_rest = + "627e3ef0f255a322d2bd9610cfaaacc8f122b7f8d17c0e7e3caafa160f9fc7.file.xattrs"; + let expected_objtype = "xattrs"; + let output = parse_object_entry_path(&input).unwrap(); + assert_eq!(output.0, expected_parent); + assert_eq!(output.1, expected_rest); + assert_eq!(output.2, expected_objtype); + } + + #[test] + fn test_parse_checksum() { + let parent = "b8"; + let name = "627e3ef0f255a322d2bd9610cfaaacc8f122b7f8d17c0e7e3caafa160f9fc7.file.xattrs"; + let expected = "b8627e3ef0f255a322d2bd9610cfaaacc8f122b7f8d17c0e7e3caafa160f9fc7"; + let output = parse_checksum(parent, &Utf8PathBuf::from(name)).unwrap(); + assert_eq!(output, expected); + } + + #[test] + fn test_parse_xattrs_link_target() { + let err_cases = &[ + "", + "b8627e3ef0f255a322d2bd9610cfaaacc8f122b7f8d17c0e7e3caafa160f9fc7.file-xattrs", + "../b8/62.file-xattrs", + ]; + for input in err_cases { + parse_xattrs_link_target(Utf8Path::new(input)).unwrap_err(); + } + + let ok_cases = &[ + "../b8/627e3ef0f255a322d2bd9610cfaaacc8f122b7f8d17c0e7e3caafa160f9fc7.file-xattrs", + "sysroot/ostree/repo/objects/b8/627e3ef0f255a322d2bd9610cfaaacc8f122b7f8d17c0e7e3caafa160f9fc7.file-xattrs", + ]; + let expected = "b8627e3ef0f255a322d2bd9610cfaaacc8f122b7f8d17c0e7e3caafa160f9fc7"; + for input in ok_cases { + let output = parse_xattrs_link_target(Utf8Path::new(input)).unwrap(); + assert_eq!(output, expected); + } + } +} diff --git a/ostree-ext/src/tar/mod.rs b/ostree-ext/src/tar/mod.rs new file mode 100644 index 00000000..2e1bbc72 --- /dev/null +++ b/ostree-ext/src/tar/mod.rs @@ -0,0 +1,51 @@ +//! # Losslessly export and import ostree commits as tar archives +//! +//! Convert an ostree commit into a tarball stream, and import it again, including +//! support for OSTree signature verification. +//! +//! In the current libostree C library, while it supports export to tar, this +//! process is lossy - commit metadata is discarded. Further, re-importing +//! requires recalculating all of the object checksums, and tying these +//! together, it does not support verifying ostree level cryptographic signatures +//! such as GPG/ed25519. +//! +//! # Tar stream layout +//! +//! In order to solve these problems, this new tar serialization format effectively +//! combines *both* a `/sysroot/ostree/repo/objects` directory and a checkout in `/usr`, +//! where the latter are hardlinks to the former. +//! +//! The exported stream will have the ostree metadata first; in particular the commit object. +//! Following the commit object is the `.commitmeta` object, which contains any cryptographic +//! signatures. +//! +//! This library then supports verifying the pair of (commit, commitmeta) using an ostree +//! remote, in the same way that `ostree pull` will do. +//! +//! The remainder of the stream is a breadth-first traversal of dirtree/dirmeta objects and the +//! content objects they reference. +//! +//! # `bare-split-xattrs` repository mode +//! +//! In format version 1, the tar stream embeds a proper ostree repository using a tailored +//! `bare-split-xattrs` mode. +//! +//! This is because extended attributes (xattrs) are a complex subject for tar, which has +//! many variants. +//! Further, when exporting bootable ostree commits to container images, it is not actually +//! desired to have the container runtime try to unpack and apply those. +//! +//! For these reasons, extended attributes (xattrs) get serialized into detached objects +//! which are associated with the relevant content objects. +//! +//! At a low level, two dedicated object types are used: +//! * `file-xattrs` as regular files storing (and de-duplicating) xattrs content. +//! * `file-xattrs-link` as hardlinks which associate a `file` object to its corresponding +//! `file-xattrs` object. + +mod import; +pub use import::*; +mod export; +pub use export::*; +mod write; +pub use write::*; diff --git a/ostree-ext/src/tar/write.rs b/ostree-ext/src/tar/write.rs new file mode 100644 index 00000000..ae311831 --- /dev/null +++ b/ostree-ext/src/tar/write.rs @@ -0,0 +1,623 @@ +//! APIs to write a tarball stream into an OSTree commit. +//! +//! This functionality already exists in libostree mostly, +//! this API adds a higher level, more ergonomic Rust frontend +//! to it. +//! +//! In the future, this may also evolve into parsing the tar +//! stream in Rust, not in C. + +use crate::Result; +use anyhow::{anyhow, Context}; +use camino::{Utf8Component, Utf8Path, Utf8PathBuf}; + +use cap_std::io_lifetimes; +use cap_std_ext::cap_std::fs::Dir; +use cap_std_ext::cmdext::CapStdExtCommandExt; +use cap_std_ext::{cap_std, cap_tempfile}; +use containers_image_proxy::oci_spec::image as oci_image; +use fn_error_context::context; +use ostree::gio; +use ostree::prelude::FileExt; +use std::borrow::Cow; +use std::collections::{BTreeMap, HashMap}; +use std::io::{BufWriter, Seek, Write}; +use std::path::Path; +use std::process::Stdio; +use std::sync::Arc; +use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite}; +use tracing::instrument; + +// Exclude things in https://www.freedesktop.org/wiki/Software/systemd/APIFileSystems/ +// from being placed in the rootfs. +const EXCLUDED_TOPLEVEL_PATHS: &[&str] = &["run", "tmp", "proc", "sys", "dev"]; + +/// Copy a tar entry to a new tar archive, optionally using a different filesystem path. +#[context("Copying entry")] +pub(crate) fn copy_entry( + mut entry: tar::Entry, + dest: &mut tar::Builder, + path: Option<&Path>, +) -> Result<()> { + // Make copies of both the header and path, since that's required for the append APIs + let path = if let Some(path) = path { + path.to_owned() + } else { + (*entry.path()?).to_owned() + }; + let mut header = entry.header().clone(); + if let Some(headers) = entry.pax_extensions()? { + let extensions = headers + .map(|ext| { + let ext = ext?; + Ok((ext.key()?, ext.value_bytes())) + }) + .collect::>>()?; + dest.append_pax_extensions(extensions.as_slice().iter().copied())?; + } + + // Need to use the entry.link_name() not the header.link_name() + // api as the header api does not handle long paths: + // https://github.com/alexcrichton/tar-rs/issues/192 + match entry.header().entry_type() { + tar::EntryType::Symlink => { + let target = entry.link_name()?.ok_or_else(|| anyhow!("Invalid link"))?; + // Sanity check UTF-8 here too. + let target: &Utf8Path = (&*target).try_into()?; + dest.append_link(&mut header, path, target) + } + tar::EntryType::Link => { + let target = entry.link_name()?.ok_or_else(|| anyhow!("Invalid link"))?; + let target: &Utf8Path = (&*target).try_into()?; + // We need to also normalize the target in order to handle hardlinked files in /etc + // where we remap /etc to /usr/etc. + let target = remap_etc_path(target); + dest.append_link(&mut header, path, &*target) + } + _ => dest.append_data(&mut header, path, entry), + } + .map_err(Into::into) +} + +/// Configuration for tar layer commits. +#[derive(Debug, Default)] +#[non_exhaustive] +pub struct WriteTarOptions { + /// Base ostree commit hash + pub base: Option, + /// Enable SELinux labeling from the base commit + /// Requires the `base` option. + pub selinux: bool, + /// Allow content not in /usr; this should be paired with ostree rootfs.transient = true + pub allow_nonusr: bool, + /// If true, do not move content in /var to /usr/share/factory/var. This should be used + /// with ostree v2024.3 or newer. + pub retain_var: bool, +} + +/// The result of writing a tar stream. +/// +/// This includes some basic data on the number of files that were filtered +/// out because they were not in `/usr`. +#[derive(Debug, Default)] +pub struct WriteTarResult { + /// The resulting OSTree commit SHA-256. + pub commit: String, + /// Number of paths in a prefix (e.g. `/var` or `/boot`) which were discarded. + pub filtered: BTreeMap, +} + +// Copy of logic from https://github.com/ostreedev/ostree/pull/2447 +// to avoid waiting for backport + releases +fn sepolicy_from_base(repo: &ostree::Repo, base: &str) -> Result { + let cancellable = gio::Cancellable::NONE; + let policypath = "usr/etc/selinux"; + let tempdir = tempfile::tempdir()?; + let (root, _) = repo.read_commit(base, cancellable)?; + let policyroot = root.resolve_relative_path(policypath); + if policyroot.query_exists(cancellable) { + let policydest = tempdir.path().join(policypath); + std::fs::create_dir_all(policydest.parent().unwrap())?; + let opts = ostree::RepoCheckoutAtOptions { + mode: ostree::RepoCheckoutMode::User, + subpath: Some(Path::new(policypath).to_owned()), + ..Default::default() + }; + repo.checkout_at(Some(&opts), ostree::AT_FDCWD, policydest, base, cancellable)?; + } + Ok(tempdir) +} + +#[derive(Debug, PartialEq, Eq)] +enum NormalizedPathResult<'a> { + Filtered(&'a str), + Normal(Utf8PathBuf), +} + +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub(crate) struct TarImportConfig { + allow_nonusr: bool, + remap_factory_var: bool, +} + +// If a path starts with /etc or ./etc or etc, remap it to be usr/etc. +fn remap_etc_path(path: &Utf8Path) -> Cow { + let mut components = path.components(); + let Some(prefix) = components.next() else { + return Cow::Borrowed(path); + }; + let (prefix, first) = if matches!(prefix, Utf8Component::CurDir | Utf8Component::RootDir) { + let Some(next) = components.next() else { + return Cow::Borrowed(path); + }; + (Some(prefix), next) + } else { + (None, prefix) + }; + if first.as_str() == "etc" { + let usr = Utf8Component::Normal("usr"); + Cow::Owned( + prefix + .into_iter() + .chain([usr, first]) + .chain(components) + .collect(), + ) + } else { + Cow::Borrowed(path) + } +} + +fn normalize_validate_path<'a>( + path: &'a Utf8Path, + config: &'_ TarImportConfig, +) -> Result> { + // This converts e.g. `foo//bar/./baz` into `foo/bar/baz`. + let mut components = path + .components() + .map(|part| { + match part { + // Convert absolute paths to relative + camino::Utf8Component::RootDir => Ok(camino::Utf8Component::CurDir), + // Allow ./ and regular parts + camino::Utf8Component::Normal(_) | camino::Utf8Component::CurDir => Ok(part), + // Barf on Windows paths as well as Unix path uplinks `..` + _ => Err(anyhow!("Invalid path: {}", path)), + } + }) + .peekable(); + let mut ret = Utf8PathBuf::new(); + // Insert a leading `./` if not present + if let Some(Ok(camino::Utf8Component::Normal(_))) = components.peek() { + ret.push(camino::Utf8Component::CurDir); + } + let mut found_first = false; + let mut excluded = false; + for part in components { + let part = part?; + if excluded { + return Ok(NormalizedPathResult::Filtered(part.as_str())); + } + if !found_first { + if let Utf8Component::Normal(part) = part { + found_first = true; + match part { + // We expect all the OS content to live in usr in general + "usr" => ret.push(part), + // ostree has special support for /etc + "etc" => { + ret.push("usr/etc"); + } + "var" => { + // Content in /var will get copied by a systemd tmpfiles.d unit + if config.remap_factory_var { + ret.push("usr/share/factory/var"); + } else { + ret.push(part) + } + } + o if EXCLUDED_TOPLEVEL_PATHS.contains(&o) => { + // We don't want to actually drop the toplevel, but mark + // *children* of it as excluded. + excluded = true; + ret.push(part) + } + _ if config.allow_nonusr => ret.push(part), + _ => { + return Ok(NormalizedPathResult::Filtered(part)); + } + } + } else { + ret.push(part); + } + } else { + ret.push(part); + } + } + + Ok(NormalizedPathResult::Normal(ret)) +} + +/// Perform various filtering on imported tar archives. +/// - Move /etc to /usr/etc +/// - Entirely drop files not in /usr +/// +/// This also acts as a Rust "pre-parser" of the tar archive, hopefully +/// catching anything corrupt that might be exploitable from the C libarchive side. +/// Remember that we're parsing this while we're downloading it, and in order +/// to verify integrity we rely on the total sha256 of the blob, so all content +/// written before then must be considered untrusted. +pub(crate) fn filter_tar( + src: impl std::io::Read, + dest: impl std::io::Write, + config: &TarImportConfig, + tmpdir: &Dir, +) -> Result> { + let src = std::io::BufReader::new(src); + let mut src = tar::Archive::new(src); + let dest = BufWriter::new(dest); + let mut dest = tar::Builder::new(dest); + let mut filtered = BTreeMap::new(); + + let ents = src.entries()?; + + tracing::debug!("Filtering tar; config={config:?}"); + + // Lookaside data for dealing with hardlinked files into /sysroot; see below. + let mut changed_sysroot_objects = HashMap::new(); + let mut new_sysroot_link_targets = HashMap::::new(); + + for entry in ents { + let mut entry = entry?; + let header = entry.header(); + let path = entry.path()?; + let path: &Utf8Path = (&*path).try_into()?; + // Force all paths to relative + let path = path.strip_prefix("/").unwrap_or(path); + + let is_modified = header.mtime().unwrap_or_default() > 0; + let is_regular = header.entry_type() == tar::EntryType::Regular; + if path.strip_prefix(crate::tar::REPO_PREFIX).is_ok() { + // If it's a modified file in /sysroot, it may be a target for future hardlinks. + // In that case, we copy the data off to a temporary file. Then the first hardlink + // to it becomes instead the real file, and any *further* hardlinks refer to that + // file instead. + if is_modified && is_regular { + tracing::debug!("Processing modified sysroot file {path}"); + // Create an O_TMPFILE (anonymous file) to use as a temporary store for the file data + let mut tmpf = cap_tempfile::TempFile::new_anonymous(tmpdir) + .map(BufWriter::new) + .context("Creating tmpfile")?; + let path = path.to_owned(); + let header = header.clone(); + std::io::copy(&mut entry, &mut tmpf) + .map_err(anyhow::Error::msg) + .context("Copying")?; + let mut tmpf = tmpf.into_inner()?; + tmpf.seek(std::io::SeekFrom::Start(0))?; + // Cache this data, indexed by the file path + changed_sysroot_objects.insert(path, (header, tmpf)); + continue; + } + } else if header.entry_type() == tar::EntryType::Link && is_modified { + let target = header + .link_name()? + .ok_or_else(|| anyhow!("Invalid empty hardlink"))?; + let target: &Utf8Path = (&*target).try_into()?; + // Canonicalize to a relative path + let target = path.strip_prefix("/").unwrap_or(target); + // If this is a hardlink into /sysroot... + if target.strip_prefix(crate::tar::REPO_PREFIX).is_ok() { + // And we found a previously processed modified file there + if let Some((mut header, data)) = changed_sysroot_objects.remove(target) { + tracing::debug!("Making {path} canonical for sysroot link {target}"); + // Make *this* entry the canonical one, consuming the temporary file data + dest.append_data(&mut header, path, data)?; + // And cache this file path as the new link target + new_sysroot_link_targets.insert(target.to_owned(), path.to_owned()); + } else if let Some(real_target) = new_sysroot_link_targets.get(target) { + tracing::debug!("Relinking {path} to {real_target}"); + // We found a 2nd (or 3rd, etc.) link into /sysroot; rewrite the link + // target to be the first file outside of /sysroot we found. + let mut header = header.clone(); + dest.append_link(&mut header, path, real_target)?; + } else { + tracing::debug!("Found unhandled modified link from {path} to {target}"); + } + continue; + } + } + + let normalized = match normalize_validate_path(path, config)? { + NormalizedPathResult::Filtered(path) => { + tracing::trace!("Filtered: {path}"); + if let Some(v) = filtered.get_mut(path) { + *v += 1; + } else { + filtered.insert(path.to_string(), 1); + } + continue; + } + NormalizedPathResult::Normal(path) => path, + }; + + copy_entry(entry, &mut dest, Some(normalized.as_std_path()))?; + } + dest.into_inner()?.flush()?; + Ok(filtered) +} + +/// Asynchronous wrapper for filter_tar() +#[context("Filtering tar stream")] +async fn filter_tar_async( + src: impl AsyncRead + Send + 'static, + media_type: oci_image::MediaType, + mut dest: impl AsyncWrite + Send + Unpin, + config: &TarImportConfig, + repo_tmpdir: Dir, +) -> Result> { + let (tx_buf, mut rx_buf) = tokio::io::duplex(8192); + // The source must be moved to the heap so we know it is stable for passing to the worker thread + let src = Box::pin(src); + let config = config.clone(); + let tar_transformer = crate::tokio_util::spawn_blocking_flatten(move || { + let src = tokio_util::io::SyncIoBridge::new(src); + let mut src = crate::container::decompressor(&media_type, src)?; + let dest = tokio_util::io::SyncIoBridge::new(tx_buf); + + let r = filter_tar(&mut src, dest, &config, &repo_tmpdir); + // Pass ownership of the input stream back to the caller - see below. + Ok((r, src)) + }); + let copier = tokio::io::copy(&mut rx_buf, &mut dest); + let (r, v) = tokio::join!(tar_transformer, copier); + let _v: u64 = v?; + let (r, src) = r?; + // Note that the worker thread took temporary ownership of the input stream; we only close + // it at this point, after we're sure we've done all processing of the input. The reason + // for this is that both the skopeo process *or* us could encounter an error (see join_fetch). + // By ensuring we hold the stream open as long as possible, it ensures that we're going to + // see a remote error first, instead of the remote skopeo process seeing us close the pipe + // because we found an error. + drop(src); + // And pass back the result + r +} + +/// Write the contents of a tarball as an ostree commit. +#[allow(unsafe_code)] // For raw fd bits +#[instrument(level = "debug", skip_all)] +pub async fn write_tar( + repo: &ostree::Repo, + src: impl tokio::io::AsyncRead + Send + Unpin + 'static, + media_type: oci_image::MediaType, + refname: &str, + options: Option, +) -> Result { + let repo = repo.clone(); + let options = options.unwrap_or_default(); + let sepolicy = if options.selinux { + if let Some(base) = options.base { + Some(sepolicy_from_base(&repo, &base).context("tar: Preparing sepolicy")?) + } else { + None + } + } else { + None + }; + let mut c = std::process::Command::new("ostree"); + let repofd = repo.dfd_as_file()?; + let repofd: Arc = Arc::new(repofd.into()); + { + let c = c + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .args(["commit"]); + c.take_fd_n(repofd.clone(), 3); + c.arg("--repo=/proc/self/fd/3"); + if let Some(sepolicy) = sepolicy.as_ref() { + c.arg("--selinux-policy"); + c.arg(sepolicy.path()); + } + c.arg(format!( + "--add-metadata-string=ostree.importer.version={}", + env!("CARGO_PKG_VERSION") + )); + c.args([ + "--no-bindings", + "--tar-autocreate-parents", + "--tree=tar=/proc/self/fd/0", + "--branch", + refname, + ]); + } + let mut c = tokio::process::Command::from(c); + c.kill_on_drop(true); + let mut r = c.spawn()?; + tracing::trace!("Spawned ostree child process"); + // Safety: We passed piped() for all of these + let child_stdin = r.stdin.take().unwrap(); + let mut child_stdout = r.stdout.take().unwrap(); + let mut child_stderr = r.stderr.take().unwrap(); + // Copy the filtered tar stream to child stdin + let import_config = TarImportConfig { + allow_nonusr: options.allow_nonusr, + remap_factory_var: !options.retain_var, + }; + let repo_tmpdir = Dir::reopen_dir(&repo.dfd_borrow())? + .open_dir("tmp") + .context("Getting repo tmpdir")?; + let filtered_result = + filter_tar_async(src, media_type, child_stdin, &import_config, repo_tmpdir); + let output_copier = async move { + // Gather stdout/stderr to buffers + let mut child_stdout_buf = String::new(); + let mut child_stderr_buf = String::new(); + let (_a, _b) = tokio::try_join!( + child_stdout.read_to_string(&mut child_stdout_buf), + child_stderr.read_to_string(&mut child_stderr_buf) + )?; + Ok::<_, anyhow::Error>((child_stdout_buf, child_stderr_buf)) + }; + + // We must convert the child exit status here to an error to + // ensure we break out of the try_join! below. + let status = async move { + let status = r.wait().await?; + if !status.success() { + return Err(anyhow!("Failed to commit tar: {:?}", status)); + } + anyhow::Ok(()) + }; + tracing::debug!("Waiting on child process"); + let (filtered_result, child_stdout) = + match tokio::try_join!(status, filtered_result).context("Processing tar") { + Ok(((), filtered_result)) => { + let (child_stdout, _) = output_copier.await.context("Copying child output")?; + (filtered_result, child_stdout) + } + Err(e) => { + if let Ok((_, child_stderr)) = output_copier.await { + // Avoid trailing newline + let child_stderr = child_stderr.trim(); + Err(e.context(child_stderr.to_string()))? + } else { + Err(e)? + } + } + }; + drop(sepolicy); + + tracing::trace!("tar written successfully"); + // TODO: trim string in place + let s = child_stdout.trim(); + Ok(WriteTarResult { + commit: s.to_string(), + filtered: filtered_result, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Cursor; + + #[test] + fn test_remap_etc() { + // These shouldn't change. Test etcc to verify we're not doing string matching. + let unchanged = ["", "foo", "/etcc/foo", "../etc/baz"]; + for x in unchanged { + similar_asserts::assert_eq!(x, remap_etc_path(x.into()).as_str()); + } + // Verify all 3 forms of "./etc", "/etc" and "etc", and also test usage of + // ".."" (should be unchanged) and "//" (will be normalized). + for (p, expected) in [ + ("/etc/foo/../bar/baz", "/usr/etc/foo/../bar/baz"), + ("etc/foo//bar", "usr/etc/foo/bar"), + ("./etc/foo", "./usr/etc/foo"), + ("etc", "usr/etc"), + ] { + similar_asserts::assert_eq!(remap_etc_path(p.into()).as_str(), expected); + } + } + + #[test] + fn test_normalize_path() { + let imp_default = &TarImportConfig { + allow_nonusr: false, + remap_factory_var: true, + }; + let allow_nonusr = &TarImportConfig { + allow_nonusr: true, + remap_factory_var: true, + }; + let composefs_and_new_ostree = &TarImportConfig { + allow_nonusr: true, + remap_factory_var: false, + }; + let valid_all = &[ + ("/usr/bin/blah", "./usr/bin/blah"), + ("usr/bin/blah", "./usr/bin/blah"), + ("usr///share/.//blah", "./usr/share/blah"), + ("var/lib/blah", "./usr/share/factory/var/lib/blah"), + ("./var/lib/blah", "./usr/share/factory/var/lib/blah"), + ("dev", "./dev"), + ("/proc", "./proc"), + ("./", "."), + ]; + let valid_nonusr = &[("boot", "./boot"), ("opt/puppet/blah", "./opt/puppet/blah")]; + for &(k, v) in valid_all { + let r = normalize_validate_path(k.into(), imp_default).unwrap(); + let r2 = normalize_validate_path(k.into(), allow_nonusr).unwrap(); + assert_eq!(r, r2); + match r { + NormalizedPathResult::Normal(r) => assert_eq!(r, v), + NormalizedPathResult::Filtered(o) => panic!("Should not have filtered {o}"), + } + } + for &(k, v) in valid_nonusr { + let strict = normalize_validate_path(k.into(), imp_default).unwrap(); + assert!( + matches!(strict, NormalizedPathResult::Filtered(_)), + "Incorrect filter for {k}" + ); + let nonusr = normalize_validate_path(k.into(), allow_nonusr).unwrap(); + match nonusr { + NormalizedPathResult::Normal(r) => assert_eq!(r, v), + NormalizedPathResult::Filtered(o) => panic!("Should not have filtered {o}"), + } + } + let filtered = &["/run/blah", "/sys/foo", "/dev/somedev"]; + for &k in filtered { + match normalize_validate_path(k.into(), imp_default).unwrap() { + NormalizedPathResult::Filtered(_) => {} + NormalizedPathResult::Normal(_) => { + panic!("{} should be filtered", k) + } + } + } + let errs = &["usr/foo/../../bar"]; + for &k in errs { + assert!(normalize_validate_path(k.into(), allow_nonusr).is_err()); + assert!(normalize_validate_path(k.into(), imp_default).is_err()); + } + assert!(matches!( + normalize_validate_path("var/lib/foo".into(), composefs_and_new_ostree).unwrap(), + NormalizedPathResult::Normal(_) + )); + } + + #[tokio::test] + async fn tar_filter() -> Result<()> { + let tempd = tempfile::tempdir()?; + let rootfs = &tempd.path().join("rootfs"); + + std::fs::create_dir_all(rootfs.join("etc/systemd/system"))?; + std::fs::write(rootfs.join("etc/systemd/system/foo.service"), "fooservice")?; + std::fs::write(rootfs.join("blah"), "blah")?; + let rootfs_tar_path = &tempd.path().join("rootfs.tar"); + let rootfs_tar = std::fs::File::create(rootfs_tar_path)?; + let mut rootfs_tar = tar::Builder::new(rootfs_tar); + rootfs_tar.append_dir_all(".", rootfs)?; + let _ = rootfs_tar.into_inner()?; + let mut dest = Vec::new(); + let src = tokio::io::BufReader::new(tokio::fs::File::open(rootfs_tar_path).await?); + let cap_tmpdir = Dir::open_ambient_dir(&tempd, cap_std::ambient_authority())?; + filter_tar_async( + src, + oci_image::MediaType::ImageLayer, + &mut dest, + &Default::default(), + cap_tmpdir, + ) + .await?; + let dest = dest.as_slice(); + let mut final_tar = tar::Archive::new(Cursor::new(dest)); + let destdir = &tempd.path().join("destdir"); + final_tar.unpack(destdir)?; + assert!(destdir.join("usr/etc/systemd/system/foo.service").exists()); + assert!(!destdir.join("blah").exists()); + Ok(()) + } +} diff --git a/ostree-ext/src/tokio_util.rs b/ostree-ext/src/tokio_util.rs new file mode 100644 index 00000000..e21b142c --- /dev/null +++ b/ostree-ext/src/tokio_util.rs @@ -0,0 +1,106 @@ +//! Helpers for bridging GLib async/mainloop with Tokio. + +use anyhow::Result; +use core::fmt::{Debug, Display}; +use futures_util::{Future, FutureExt}; +use ostree::gio; +use ostree::prelude::{CancellableExt, CancellableExtManual}; + +/// Call a faillible future, while monitoring `cancellable` and return an error if cancelled. +pub async fn run_with_cancellable(f: F, cancellable: &gio::Cancellable) -> Result +where + F: Future>, +{ + // Bridge GCancellable to a tokio notification + let notify = std::sync::Arc::new(tokio::sync::Notify::new()); + let notify2 = notify.clone(); + cancellable.connect_cancelled(move |_| notify2.notify_one()); + cancellable.set_error_if_cancelled()?; + // See https://blog.yoshuawuyts.com/futures-concurrency-3/ on why + // `select!` is a trap in general, but I believe this case is safe. + tokio::select! { + r = f => r, + _ = notify.notified() => { + Err(anyhow::anyhow!("Operation was cancelled")) + } + } +} + +struct CancelOnDrop(gio::Cancellable); + +impl Drop for CancelOnDrop { + fn drop(&mut self) { + self.0.cancel(); + } +} + +/// Wrapper for [`tokio::task::spawn_blocking`] which provides a [`gio::Cancellable`] that will be triggered on drop. +/// +/// This function should be used in a Rust/tokio native `async fn`, but that want to invoke +/// GLib style blocking APIs that use `GCancellable`. The cancellable will be triggered when this +/// future is dropped, which helps bound thread usage. +/// +/// This is in a sense the inverse of [`run_with_cancellable`]. +pub fn spawn_blocking_cancellable(f: F) -> tokio::task::JoinHandle +where + F: FnOnce(&gio::Cancellable) -> R + Send + 'static, + R: Send + 'static, +{ + tokio::task::spawn_blocking(move || { + let dropper = CancelOnDrop(gio::Cancellable::new()); + f(&dropper.0) + }) +} + +/// Flatten a nested Result>, defaulting to converting the error type to an `anyhow::Error`. +/// See https://doc.rust-lang.org/std/result/enum.Result.html#method.flatten +pub(crate) fn flatten_anyhow(r: std::result::Result, E>) -> Result +where + E: Display + Debug + Send + Sync + 'static, +{ + match r { + Ok(x) => x, + Err(e) => Err(anyhow::anyhow!(e)), + } +} + +/// A wrapper around [`spawn_blocking_cancellable`] that flattens nested results. +pub fn spawn_blocking_cancellable_flatten(f: F) -> impl Future> +where + F: FnOnce(&gio::Cancellable) -> Result + Send + 'static, + T: Send + 'static, +{ + spawn_blocking_cancellable(f).map(flatten_anyhow) +} + +/// A wrapper around [`tokio::task::spawn_blocking`] that flattens nested results. +pub fn spawn_blocking_flatten(f: F) -> impl Future> +where + F: FnOnce() -> Result + Send + 'static, + T: Send + 'static, +{ + tokio::task::spawn_blocking(f).map(flatten_anyhow) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn test_cancellable() { + let cancellable = ostree::gio::Cancellable::new(); + + let cancellable_copy = cancellable.clone(); + let s = async move { + tokio::time::sleep(std::time::Duration::from_millis(200)).await; + cancellable_copy.cancel(); + }; + let r = async move { + tokio::time::sleep(std::time::Duration::from_secs(200)).await; + Ok(()) + }; + let r = run_with_cancellable(r, &cancellable); + let (_, r) = tokio::join!(s, r); + assert!(r.is_err()); + } +} diff --git a/ostree-ext/src/utils.rs b/ostree-ext/src/utils.rs new file mode 100644 index 00000000..c7821df2 --- /dev/null +++ b/ostree-ext/src/utils.rs @@ -0,0 +1,34 @@ +pub(crate) trait ResultExt { + /// Return the Ok value unchanged. In the err case, log it, and call the closure to compute the default + fn log_err_or_else(self, default: F) -> T + where + F: FnOnce() -> T; + /// Return the Ok value unchanged. In the err case, log it, and return the default value + fn log_err_default(self) -> T + where + T: Default; +} + +impl ResultExt for Result { + #[track_caller] + fn log_err_or_else(self, default: F) -> T + where + F: FnOnce() -> T, + { + match self { + Ok(r) => r, + Err(e) => { + tracing::debug!("{e}"); + default() + } + } + } + + #[track_caller] + fn log_err_default(self) -> T + where + T: Default, + { + self.log_err_or_else(|| Default::default()) + } +} diff --git a/ostree-ext/tests/it/fixtures/hlinks.tar.gz b/ostree-ext/tests/it/fixtures/hlinks.tar.gz new file mode 100644 index 00000000..0bbc06d4 Binary files /dev/null and b/ostree-ext/tests/it/fixtures/hlinks.tar.gz differ diff --git a/ostree-ext/tests/it/fixtures/manifest1.json b/ostree-ext/tests/it/fixtures/manifest1.json new file mode 100644 index 00000000..52f09f28 --- /dev/null +++ b/ostree-ext/tests/it/fixtures/manifest1.json @@ -0,0 +1 @@ +{"schemaVersion":2,"config":{"mediaType":"application/vnd.oci.image.config.v1+json","digest":"sha256:f3b50d0849a19894aa27ca2346a78efdacf2c56bdc2a3493672d2a819990fedf","size":9301},"layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:75f4abe8518ec55cb8bf0d358a737084f38e2c030a28651d698c0b7569d680a6","size":1387849},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:777cb841d2803f775a36fba62bcbfe84b2a1e0abc27cf995961b63c3d218a410","size":48676116},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:1179dc1e2994ec0466787ec43967db9016b4b93c602bb9675d7fe4c0993366ba","size":124705297},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:74555b3730c4c0f77529ead433db58e038070666b93a5cc0da262d7b8debff0e","size":38743650},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:0ff8b1fdd38e5cfb6390024de23ba4b947cd872055f62e70f2c21dad5c928925","size":77161948},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:76b83eea62b7b93200a056b5e0201ef486c67f1eeebcf2c7678ced4d614cece2","size":21970157},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:d85c742f69904cb8dbf98abca4724d364d91792fcf8b5f5634ab36dda162bfc4","size":59797135},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:167e5df36d0fcbed876ca90c1ed1e6c79b5e2bdaba5eae74ab86444654b19eff","size":49410348},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:b34384ba76fa1e335cc8d75522508d977854f2b423f8aceb50ca6dfc2f609a99","size":21714783},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:7bf2d65ebf222ee10115284abf6909b1a3da0f3bd6d8d849e30723636b7145cb","size":15264848},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:a75bbf55d8de4dbd54e429e16fbd46688717faf4ea823c94676529cc2525fd5f","size":14373701},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:cf728677fa8c84bfcfd71e17953062421538d492d7fbfdd0dbce8eb1e5f6eec3","size":8400473},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:caff60c1ef085fb500c94230ccab9338e531578635070230b1413b439fd53f8f","size":6914489},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:65ca8f9bddaa720b74c5a7401bf273e93eba6b3b855a62422a8258373e0b1ae0","size":8294965},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:387bab4fcb713e9691617a645b6af2b7ad29fe5e009b0b0d3215645ef315481c","size":6600369},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:f63dcde5a664dad3eb3321bbcf2913d9644d16561a67c86ab61d814c1462583d","size":16869027},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:8bcd90242651342fbd2ed5ca3e60d03de90fdd28c3a9f634329f6e1c21c79718","size":5735283},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:cb65c21a0659b5b826881280556995a7ca4818c2b9b7a89e31d816a996fa8640","size":4528663},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:5187f51b62f4a2e82198a75afcc623a0323d4804fa7848e2e0acb30d77b8d9ab","size":5266030},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:bfef79d6d35378fba9093083ff6bd7b5ed9f443f87517785e6ff134dc8d08c6a","size":4316135},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:1cf332fd50b382af7941d6416994f270c894e9d60fb5c6cecf25de887673bbcb","size":3914655},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:e0d80be6e71bfae398f06f7a7e3b224290f3dde7544c8413f922934abeb1f599","size":2441858},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:48ff87e7a7af41d7139c5230e2e939aa97cafb1f62a114825bda5f5904e04a0e","size":3818782},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:8bcc652ccaa27638bd5bd2d7188053f1736586afbae87b3952e9211c773e3563","size":3885971},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:d83d9388b8c8c1e7c97b6b18f5107b74354700ebce9da161ccb73156a2c54a2e","size":3442642},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:efc465ae44a18ee395e542eb97c8d1fc21bf9d5fb49244ba4738e9bf48bfd3dc","size":3066348},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:c5c471cce08aa9cc7d96884a9e1981b7bb67ee43524af47533f50a8ddde7a83d","size":909923},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:8956cd951abc481ba364cf8ef5deca7cc9185b59ed95ae40b52e42afdc271d8e","size":3553645},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:5b0963a6c89d595b5c4786e2f3ce0bc168a262efab74dfce3d7c8d1063482c60","size":1495301},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:bf2df295da2716291f9dd4707158bca218b4a7920965955a4808b824c1bee2b6","size":3063142},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:19b2ea8d63794b8249960d581216ae1ccb80f8cfe518ff8dd1f12d65d19527a5","size":8109718},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:420636df561ccc835ef9665f41d4bc91c5f00614a61dca266af2bcd7bee2cc25","size":3003935},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:5ae67caf0978d82848d47ff932eee83a1e5d2581382c9c47335f69c9d7acc180","size":2468557},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:4f4b8bb8463dc74bb7f32eee78d02b71f61a322967b6d6cbb29829d262376f74","size":2427605},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:69373f86b83e6e5a962de07f40ff780a031b42d2568ffbb8b3c36de42cc90dec","size":2991782},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:2d05c2f993f9761946701da37f45fc573a2db8467f92b3f0d356f5f7adaf229e","size":3085765},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:41925843e5c965165bedc9c8124b96038f08a89c95ba94603a5f782dc813f0a8","size":2724309},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:a8c39f2998073e0e8b55fb88ccd68d2621a0fb6e31a528fd4790a1c90f8508a9","size":2512079},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:b905f801d092faba0c155597dd1303fa8c0540116af59c111ed7744e486ed63b","size":2341122},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:4f46b58b37828fa71fa5d7417a8ca7a62761cc6a72eb1592943572fc2446b054","size":2759344},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:3fbae92ecc64cf253b643a0e75b56514dc694451f163b47fb4e15af373238e10","size":2539288},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:744dd4a3ec521668942661cf1f184eb8f07f44025ce1aa35d5072ad9d72946fe","size":2415870},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:6c74c0a05a36bddabef1fdfae365ff87a9c5dd1ec7345d9e20f7f8ab04b39fc6","size":2145078},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:910ff6f93303ebedde3459f599b06d7b70d8f0674e3fe1d6623e3af809245cc4","size":5098511},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:2752e2f62f38fea3a390f111d673d2529dbf929f6c67ec7ef4359731d1a7edd8","size":1051999},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:5065c3aac5fcc3c1bde50a19d776974353301f269a936dd2933a67711af3b703","size":2713694},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:8bf6993eea50bbd8b448e6fd719f83c82d1d40b623f2c415f7727e766587ea83","size":1686714},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:630221744f0f9632f4f34f74241e65f79e78f938100266a119113af1ce10a1c5","size":2061581},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:e7e2eae322bca0ffa01bb2cae72288507bef1a11ad51f99d0a4faba1b1e000b9","size":2079706},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:bb6374635385b0c2539c284b137d831bd45fbe64b5e49aee8ad92d14c156a41b","size":3142398},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:40493ecd0f9ab499a2bec715415c3a98774ea6d1c9c01eb30a6b56793204a02d","size":69953187}]} \ No newline at end of file diff --git a/ostree-ext/tests/it/fixtures/manifest2.json b/ostree-ext/tests/it/fixtures/manifest2.json new file mode 100644 index 00000000..102c4017 --- /dev/null +++ b/ostree-ext/tests/it/fixtures/manifest2.json @@ -0,0 +1 @@ +{"schemaVersion":2,"config":{"mediaType":"application/vnd.oci.image.config.v1+json","digest":"sha256:ca0f7e342503b45a1110aba49177e386242e9192ab1742a95998b6b99c2a0150","size":9301},"layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:bca674ffe2ebe92b9e952bc807b9f1cd0d559c057e95ac81f3bae12a9b96b53e","size":1387854},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:777cb841d2803f775a36fba62bcbfe84b2a1e0abc27cf995961b63c3d218a410","size":48676116},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:1179dc1e2994ec0466787ec43967db9016b4b93c602bb9675d7fe4c0993366ba","size":124705297},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:74555b3730c4c0f77529ead433db58e038070666b93a5cc0da262d7b8debff0e","size":38743650},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:0b5d930ffc92d444b0a7b39beed322945a3038603fbe2a56415a6d02d598df1f","size":77162517},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:8d12d20c2d1c8f05c533a2a1b27a457f25add8ad38382523660c4093f180887b","size":21970100},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:d85c742f69904cb8dbf98abca4724d364d91792fcf8b5f5634ab36dda162bfc4","size":59797135},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:167e5df36d0fcbed876ca90c1ed1e6c79b5e2bdaba5eae74ab86444654b19eff","size":49410348},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:b34384ba76fa1e335cc8d75522508d977854f2b423f8aceb50ca6dfc2f609a99","size":21714783},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:7bf2d65ebf222ee10115284abf6909b1a3da0f3bd6d8d849e30723636b7145cb","size":15264848},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:a75bbf55d8de4dbd54e429e16fbd46688717faf4ea823c94676529cc2525fd5f","size":14373701},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:cf728677fa8c84bfcfd71e17953062421538d492d7fbfdd0dbce8eb1e5f6eec3","size":8400473},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:caff60c1ef085fb500c94230ccab9338e531578635070230b1413b439fd53f8f","size":6914489},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:65ca8f9bddaa720b74c5a7401bf273e93eba6b3b855a62422a8258373e0b1ae0","size":8294965},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:387bab4fcb713e9691617a645b6af2b7ad29fe5e009b0b0d3215645ef315481c","size":6600369},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:f63dcde5a664dad3eb3321bbcf2913d9644d16561a67c86ab61d814c1462583d","size":16869027},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:8bcd90242651342fbd2ed5ca3e60d03de90fdd28c3a9f634329f6e1c21c79718","size":5735283},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:cb65c21a0659b5b826881280556995a7ca4818c2b9b7a89e31d816a996fa8640","size":4528663},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:5187f51b62f4a2e82198a75afcc623a0323d4804fa7848e2e0acb30d77b8d9ab","size":5266030},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:bfef79d6d35378fba9093083ff6bd7b5ed9f443f87517785e6ff134dc8d08c6a","size":4316135},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:1cf332fd50b382af7941d6416994f270c894e9d60fb5c6cecf25de887673bbcb","size":3914655},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:e0d80be6e71bfae398f06f7a7e3b224290f3dde7544c8413f922934abeb1f599","size":2441858},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:48ff87e7a7af41d7139c5230e2e939aa97cafb1f62a114825bda5f5904e04a0e","size":3818782},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:8bcc652ccaa27638bd5bd2d7188053f1736586afbae87b3952e9211c773e3563","size":3885971},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:d83d9388b8c8c1e7c97b6b18f5107b74354700ebce9da161ccb73156a2c54a2e","size":3442642},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:efc465ae44a18ee395e542eb97c8d1fc21bf9d5fb49244ba4738e9bf48bfd3dc","size":3066348},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:c5c471cce08aa9cc7d96884a9e1981b7bb67ee43524af47533f50a8ddde7a83d","size":909923},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:8956cd951abc481ba364cf8ef5deca7cc9185b59ed95ae40b52e42afdc271d8e","size":3553645},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:5b0963a6c89d595b5c4786e2f3ce0bc168a262efab74dfce3d7c8d1063482c60","size":1495301},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:bf2df295da2716291f9dd4707158bca218b4a7920965955a4808b824c1bee2b6","size":3063142},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:19b2ea8d63794b8249960d581216ae1ccb80f8cfe518ff8dd1f12d65d19527a5","size":8109718},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:420636df561ccc835ef9665f41d4bc91c5f00614a61dca266af2bcd7bee2cc25","size":3003935},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:5ae67caf0978d82848d47ff932eee83a1e5d2581382c9c47335f69c9d7acc180","size":2468557},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:4f4b8bb8463dc74bb7f32eee78d02b71f61a322967b6d6cbb29829d262376f74","size":2427605},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:69373f86b83e6e5a962de07f40ff780a031b42d2568ffbb8b3c36de42cc90dec","size":2991782},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:2d05c2f993f9761946701da37f45fc573a2db8467f92b3f0d356f5f7adaf229e","size":3085765},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:41925843e5c965165bedc9c8124b96038f08a89c95ba94603a5f782dc813f0a8","size":2724309},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:a8c39f2998073e0e8b55fb88ccd68d2621a0fb6e31a528fd4790a1c90f8508a9","size":2512079},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:b905f801d092faba0c155597dd1303fa8c0540116af59c111ed7744e486ed63b","size":2341122},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:4f46b58b37828fa71fa5d7417a8ca7a62761cc6a72eb1592943572fc2446b054","size":2759344},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:3fbae92ecc64cf253b643a0e75b56514dc694451f163b47fb4e15af373238e10","size":2539288},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:744dd4a3ec521668942661cf1f184eb8f07f44025ce1aa35d5072ad9d72946fe","size":2415870},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:6c74c0a05a36bddabef1fdfae365ff87a9c5dd1ec7345d9e20f7f8ab04b39fc6","size":2145078},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:910ff6f93303ebedde3459f599b06d7b70d8f0674e3fe1d6623e3af809245cc4","size":5098511},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:2752e2f62f38fea3a390f111d673d2529dbf929f6c67ec7ef4359731d1a7edd8","size":1051999},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:5065c3aac5fcc3c1bde50a19d776974353301f269a936dd2933a67711af3b703","size":2713694},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:8bf6993eea50bbd8b448e6fd719f83c82d1d40b623f2c415f7727e766587ea83","size":1686714},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:630221744f0f9632f4f34f74241e65f79e78f938100266a119113af1ce10a1c5","size":2061581},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:e7e2eae322bca0ffa01bb2cae72288507bef1a11ad51f99d0a4faba1b1e000b9","size":2079706},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:bb6374635385b0c2539c284b137d831bd45fbe64b5e49aee8ad92d14c156a41b","size":3142398},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:cb9b8a4ac4a8df62df79e6f0348a14b3ec239816d42985631c88e76d4e3ff815","size":69952385}]} \ No newline at end of file diff --git a/ostree-ext/tests/it/main.rs b/ostree-ext/tests/it/main.rs new file mode 100644 index 00000000..e7576409 --- /dev/null +++ b/ostree-ext/tests/it/main.rs @@ -0,0 +1,1872 @@ +use anyhow::{Context, Result}; +use camino::Utf8Path; +use cap_std::fs::{Dir, DirBuilder, DirBuilderExt}; +use cap_std_ext::cap_std; +use containers_image_proxy::oci_spec; +use oci_image::ImageManifest; +use oci_spec::image as oci_image; +use ocidir::oci_spec::image::{Arch, DigestAlgorithm}; +use once_cell::sync::Lazy; +use ostree_ext::chunking::ObjectMetaSized; +use ostree_ext::container::{store, ManifestDiff}; +use ostree_ext::container::{ + Config, ExportOpts, ImageReference, OstreeImageReference, SignatureSource, Transport, +}; +use ostree_ext::prelude::{Cast, FileExt}; +use ostree_ext::tar::TarImportOptions; +use ostree_ext::{fixture, ostree_manual}; +use ostree_ext::{gio, glib}; +use std::borrow::Cow; +use std::collections::{HashMap, HashSet}; +use std::io::{BufReader, BufWriter}; +use std::process::Command; +use std::time::SystemTime; +use xshell::cmd; + +use ostree_ext::fixture::{FileDef, Fixture, CONTENTS_CHECKSUM_V0, LAYERS_V0_LEN, PKGS_V0_LEN}; + +const EXAMPLE_TAR_LAYER: &[u8] = include_bytes!("fixtures/hlinks.tar.gz"); +const TEST_REGISTRY_DEFAULT: &str = "localhost:5000"; + +#[track_caller] +fn assert_err_contains(r: Result, s: impl AsRef) { + let s = s.as_ref(); + let msg = format!("{:#}", r.err().expect("Expecting an error")); + if !msg.contains(s) { + panic!(r#"Error message "{}" did not contain "{}""#, msg, s); + } +} + +static TEST_REGISTRY: Lazy = Lazy::new(|| match std::env::var_os("TEST_REGISTRY") { + Some(t) => t.to_str().unwrap().to_owned(), + None => TEST_REGISTRY_DEFAULT.to_string(), +}); + +// This is mostly just sanity checking these functions are publicly accessible +#[test] +fn test_cli_fns() -> Result<()> { + let fixture = Fixture::new_v1()?; + let srcpath = fixture.path.join("src/repo"); + let srcrepo_parsed = ostree_ext::cli::parse_repo(&srcpath).unwrap(); + assert_eq!(srcrepo_parsed.mode(), fixture.srcrepo().mode()); + + let ir = + ostree_ext::cli::parse_imgref("ostree-unverified-registry:quay.io/examplens/exampleos") + .unwrap(); + assert_eq!(ir.imgref.transport, Transport::Registry); + + let ir = ostree_ext::cli::parse_base_imgref("docker://quay.io/examplens/exampleos").unwrap(); + assert_eq!(ir.transport, Transport::Registry); + Ok(()) +} + +#[tokio::test] +async fn test_tar_import_empty() -> Result<()> { + let fixture = Fixture::new_v1()?; + let r = ostree_ext::tar::import_tar(fixture.destrepo(), tokio::io::empty(), None).await; + assert_err_contains(r, "Commit object not found"); + Ok(()) +} + +#[tokio::test] +async fn test_tar_export_reproducible() -> Result<()> { + let fixture = Fixture::new_v1()?; + let (_, rev) = fixture + .srcrepo() + .read_commit(fixture.testref(), gio::Cancellable::NONE)?; + let export1 = { + let mut h = openssl::hash::Hasher::new(openssl::hash::MessageDigest::sha256())?; + ostree_ext::tar::export_commit(fixture.srcrepo(), rev.as_str(), &mut h, None)?; + h.finish()? + }; + // Artificial delay to flush out mtimes (one second granularity baseline, plus another 100ms for good measure). + std::thread::sleep(std::time::Duration::from_millis(1100)); + let export2 = { + let mut h = openssl::hash::Hasher::new(openssl::hash::MessageDigest::sha256())?; + ostree_ext::tar::export_commit(fixture.srcrepo(), rev.as_str(), &mut h, None)?; + h.finish()? + }; + assert_eq!(*export1, *export2); + Ok(()) +} + +#[tokio::test] +async fn test_tar_import_signed() -> Result<()> { + let fixture = Fixture::new_v1()?; + let sh = fixture.new_shell()?; + let test_tar = fixture.export_tar()?; + + let rev = fixture.srcrepo().require_rev(fixture.testref())?; + let (commitv, _) = fixture.srcrepo().load_commit(rev.as_str())?; + assert_eq!( + ostree::commit_get_content_checksum(&commitv) + .unwrap() + .as_str(), + CONTENTS_CHECKSUM_V0 + ); + + // Verify we fail with an unknown remote. + let src_tar = tokio::fs::File::from_std(fixture.dir.open(test_tar)?.into_std()); + let mut taropts = TarImportOptions::default(); + taropts.remote = Some("nosuchremote".to_string()); + let r = ostree_ext::tar::import_tar(fixture.destrepo(), src_tar, Some(taropts)).await; + assert_err_contains(r, r#"Remote "nosuchremote" not found"#); + + // Test a remote, but without a key + let opts = glib::VariantDict::new(None); + opts.insert("gpg-verify", &true); + opts.insert("custom-backend", &"ostree-rs-ext"); + fixture + .destrepo() + .remote_add("myremote", None, Some(&opts.end()), gio::Cancellable::NONE)?; + let src_tar = tokio::fs::File::from_std(fixture.dir.open(test_tar)?.into_std()); + let mut taropts = TarImportOptions::default(); + taropts.remote = Some("myremote".to_string()); + let r = ostree_ext::tar::import_tar(fixture.destrepo(), src_tar, Some(taropts)).await; + assert_err_contains(r, r#"Can't check signature: public key not found"#); + + // And signed correctly + cmd!( + sh, + "ostree --repo=dest/repo remote gpg-import --stdin myremote" + ) + .stdin(sh.read_file("src/gpghome/key1.asc")?) + .ignore_stdout() + .run()?; + let src_tar = tokio::fs::File::from_std(fixture.dir.open(test_tar)?.into_std()); + let mut taropts = TarImportOptions::default(); + taropts.remote = Some("myremote".to_string()); + let imported = ostree_ext::tar::import_tar(fixture.destrepo(), src_tar, Some(taropts)).await?; + let (commitdata, state) = fixture.destrepo().load_commit(&imported)?; + assert_eq!( + CONTENTS_CHECKSUM_V0, + ostree::commit_get_content_checksum(&commitdata) + .unwrap() + .as_str() + ); + assert_eq!(state, ostree::RepoCommitState::NORMAL); + + // Drop the commit metadata, and verify that import fails + fixture.clear_destrepo()?; + let nometa = "test-no-commitmeta.tar"; + let srcf = fixture.dir.open(test_tar)?; + let destf = fixture.dir.create(nometa)?; + tokio::task::spawn_blocking(move || -> Result<_> { + let src = BufReader::new(srcf); + let f = BufWriter::new(destf); + ostree_ext::tar::update_detached_metadata(src, f, None, gio::Cancellable::NONE).unwrap(); + Ok(()) + }) + .await??; + let src_tar = tokio::fs::File::from_std(fixture.dir.open(nometa)?.into_std()); + let mut taropts = TarImportOptions::default(); + taropts.remote = Some("myremote".to_string()); + let r = ostree_ext::tar::import_tar(fixture.destrepo(), src_tar, Some(taropts)).await; + assert_err_contains(r, "Expected commitmeta object"); + + // Now inject garbage into the commitmeta by flipping some bits in the signature + let rev = fixture.srcrepo().require_rev(fixture.testref())?; + let commitmeta = fixture + .srcrepo() + .read_commit_detached_metadata(&rev, gio::Cancellable::NONE)? + .unwrap(); + let mut commitmeta = Vec::from(&*commitmeta.data_as_bytes()); + let len = commitmeta.len() / 2; + let last = commitmeta.get_mut(len).unwrap(); + (*last) = last.wrapping_add(1); + + let srcf = fixture.dir.open(test_tar)?; + let destf = fixture.dir.create(nometa)?; + tokio::task::spawn_blocking(move || -> Result<_> { + let src = BufReader::new(srcf); + let f = BufWriter::new(destf); + ostree_ext::tar::update_detached_metadata( + src, + f, + Some(&commitmeta), + gio::Cancellable::NONE, + ) + .unwrap(); + Ok(()) + }) + .await??; + let src_tar = tokio::fs::File::from_std(fixture.dir.open(nometa)?.into_std()); + let mut taropts = TarImportOptions::default(); + taropts.remote = Some("myremote".to_string()); + let r = ostree_ext::tar::import_tar(fixture.destrepo(), src_tar, Some(taropts)).await; + assert_err_contains(r, "BAD signature"); + + Ok(()) +} + +#[derive(Debug)] +struct TarExpected { + path: &'static str, + etype: tar::EntryType, + mode: u32, +} + +#[allow(clippy::from_over_into)] +impl Into for (&'static str, tar::EntryType, u32) { + fn into(self) -> TarExpected { + TarExpected { + path: self.0, + etype: self.1, + mode: self.2, + } + } +} + +fn validate_tar_expected( + t: &mut tar::Entries, + expected: impl IntoIterator, +) -> Result<()> { + let mut expected: HashMap<&'static str, TarExpected> = + expected.into_iter().map(|exp| (exp.path, exp)).collect(); + let entries = t.map(|e| e.unwrap()); + let mut seen_paths = HashSet::new(); + // Verify we're injecting directories, fixes the absence of `/tmp` in our + // images for example. + for entry in entries { + if expected.is_empty() { + return Ok(()); + } + let header = entry.header(); + let entry_path = entry.path().unwrap().to_string_lossy().into_owned(); + if seen_paths.contains(&entry_path) { + anyhow::bail!("Duplicate path: {}", entry_path); + } + seen_paths.insert(entry_path.clone()); + if let Some(exp) = expected.remove(entry_path.as_str()) { + assert_eq!(header.entry_type(), exp.etype, "{}", entry_path); + let expected_mode = exp.mode; + let header_mode = header.mode().unwrap(); + assert_eq!( + header_mode, + expected_mode, + "h={header_mode:o} e={expected_mode:o} type: {:?} path: {}", + header.entry_type(), + entry_path + ); + } + } + + assert!( + expected.is_empty(), + "Expected but not found:\n{:?}", + expected + ); + Ok(()) +} + +fn common_tar_structure() -> impl Iterator { + use tar::EntryType::Directory; + [ + ("sysroot/ostree/repo/objects/00", Directory, 0o755), + ("sysroot/ostree/repo/objects/23", Directory, 0o755), + ("sysroot/ostree/repo/objects/77", Directory, 0o755), + ("sysroot/ostree/repo/objects/bc", Directory, 0o755), + ("sysroot/ostree/repo/objects/ff", Directory, 0o755), + ("sysroot/ostree/repo/refs", Directory, 0o755), + ("sysroot/ostree/repo/refs", Directory, 0o755), + ("sysroot/ostree/repo/refs/heads", Directory, 0o755), + ("sysroot/ostree/repo/refs/mirrors", Directory, 0o755), + ("sysroot/ostree/repo/refs/remotes", Directory, 0o755), + ("sysroot/ostree/repo/state", Directory, 0o755), + ("sysroot/ostree/repo/tmp", Directory, 0o755), + ("sysroot/ostree/repo/tmp/cache", Directory, 0o755), + ] + .into_iter() + .map(Into::into) +} + +// Find various expected files +fn common_tar_contents_all() -> impl Iterator { + use tar::EntryType::{Directory, Link, Regular}; + [ + ("boot", Directory, 0o755), + ("usr", Directory, 0o755), + ("usr/lib/emptyfile", Regular, 0o644), + ("usr/lib64/emptyfile2", Regular, 0o644), + ("usr/bin/bash", Link, 0o755), + ("usr/bin/hardlink-a", Link, 0o644), + ("usr/bin/hardlink-b", Link, 0o644), + ("var/tmp", Directory, 0o1777), + ] + .into_iter() + .map(Into::into) +} + +/// Validate metadata (prelude) in a v1 tar. +fn validate_tar_v1_metadata(src: &mut tar::Entries) -> Result<()> { + use tar::EntryType::{Directory, Regular}; + let prelude = [ + ("sysroot/ostree/repo", Directory, 0o755), + ("sysroot/ostree/repo/config", Regular, 0o644), + ] + .into_iter() + .map(Into::into); + + validate_tar_expected(src, common_tar_structure().chain(prelude))?; + + Ok(()) +} + +/// Validate basic structure of the tar export. +#[test] +fn test_tar_export_structure() -> Result<()> { + let fixture = Fixture::new_v1()?; + + let src_tar = fixture.export_tar()?; + let mut src_tar = fixture + .dir + .open(src_tar) + .map(BufReader::new) + .map(tar::Archive::new)?; + let mut src_tar = src_tar.entries()?; + validate_tar_v1_metadata(&mut src_tar).unwrap(); + validate_tar_expected(&mut src_tar, common_tar_contents_all())?; + + Ok(()) +} + +#[tokio::test] +async fn test_tar_import_export() -> Result<()> { + let fixture = Fixture::new_v1()?; + let sh = fixture.new_shell()?; + let p = fixture.export_tar()?; + let src_tar = tokio::fs::File::from_std(fixture.dir.open(p)?.into_std()); + + let imported_commit: String = + ostree_ext::tar::import_tar(fixture.destrepo(), src_tar, None).await?; + let (commitdata, _) = fixture.destrepo().load_commit(&imported_commit)?; + assert_eq!( + CONTENTS_CHECKSUM_V0, + ostree::commit_get_content_checksum(&commitdata) + .unwrap() + .as_str() + ); + cmd!(sh, "ostree --repo=dest/repo ls -R {imported_commit}") + .ignore_stdout() + .run()?; + let val = cmd!(sh, "ostree --repo=dest/repo show --print-detached-metadata-key=my-detached-key {imported_commit}").read()?; + assert_eq!(val.as_str(), "'my-detached-value'"); + + let (root, _) = fixture + .destrepo() + .read_commit(&imported_commit, gio::Cancellable::NONE)?; + let kdir = ostree_ext::bootabletree::find_kernel_dir(&root, gio::Cancellable::NONE)?; + let kdir = kdir.unwrap(); + assert_eq!( + kdir.basename().unwrap().to_str().unwrap(), + "5.10.18-200.x86_64" + ); + + Ok(()) +} + +#[tokio::test] +async fn test_tar_write() -> Result<()> { + let fixture = Fixture::new_v1()?; + let sh = fixture.new_shell()?; + // Test translating /etc to /usr/etc + fixture.dir.create_dir_all("tmproot/etc")?; + let tmproot = &fixture.dir.open_dir("tmproot")?; + let tmpetc = tmproot.open_dir("etc")?; + tmpetc.write("someconfig.conf", b"some config")?; + tmproot.create_dir_all("var/log")?; + let tmpvarlog = tmproot.open_dir("var/log")?; + tmpvarlog.write("foo.log", "foolog")?; + tmpvarlog.write("bar.log", "barlog")?; + tmproot.create_dir("run")?; + tmproot.write("run/somefile", "somestate")?; + let tmptar = "testlayer.tar"; + cmd!(sh, "tar cf {tmptar} -C tmproot .").run()?; + let src = fixture.dir.open(tmptar)?; + fixture.dir.remove_file(tmptar)?; + let src = tokio::fs::File::from_std(src.into_std()); + let r = ostree_ext::tar::write_tar( + fixture.destrepo(), + src, + oci_image::MediaType::ImageLayer, + "layer", + None, + ) + .await?; + let layer_commit = r.commit.as_str(); + cmd!( + sh, + "ostree --repo=dest/repo ls {layer_commit} /usr/etc/someconfig.conf" + ) + .ignore_stdout() + .run()?; + assert_eq!(r.filtered.len(), 1); + assert!(r.filtered.get("var").is_none()); + // TODO: change filter_tar to properly make this run/somefile, but eh...we're + // just going to accept this stuff in the future but ignore it anyways. + assert_eq!(*r.filtered.get("somefile").unwrap(), 1); + + Ok(()) +} + +#[tokio::test] +async fn test_tar_write_tar_layer() -> Result<()> { + let fixture = Fixture::new_v1()?; + let mut v = Vec::new(); + let mut dec = flate2::bufread::GzDecoder::new(std::io::Cursor::new(EXAMPLE_TAR_LAYER)); + let _n = std::io::copy(&mut dec, &mut v)?; + let r = tokio::io::BufReader::new(std::io::Cursor::new(v)); + ostree_ext::tar::write_tar( + fixture.destrepo(), + r, + oci_image::MediaType::ImageLayer, + "test", + None, + ) + .await?; + Ok(()) +} + +fn skopeo_inspect(imgref: &str) -> Result { + let out = Command::new("skopeo") + .args(["inspect", imgref]) + .stdout(std::process::Stdio::piped()) + .output()?; + Ok(String::from_utf8(out.stdout)?) +} + +fn skopeo_inspect_config(imgref: &str) -> Result { + let out = Command::new("skopeo") + .args(["inspect", "--config", imgref]) + .stdout(std::process::Stdio::piped()) + .output()?; + Ok(serde_json::from_slice(&out.stdout)?) +} + +async fn impl_test_container_import_export(chunked: bool) -> Result<()> { + let fixture = Fixture::new_v1()?; + let sh = fixture.new_shell()?; + let testrev = fixture + .srcrepo() + .require_rev(fixture.testref()) + .context("Failed to resolve ref")?; + + let srcoci_path = &fixture.path.join("oci"); + let srcoci_imgref = ImageReference { + transport: Transport::OciDir, + name: srcoci_path.as_str().to_string(), + }; + let config = Config { + labels: Some( + [("foo", "bar"), ("test", "value")] + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(), + ), + ..Default::default() + }; + // If chunking is requested, compute object ownership and size mappings + let contentmeta = chunked + .then(|| { + let meta = fixture.get_object_meta().context("Computing object meta")?; + ObjectMetaSized::compute_sizes(fixture.srcrepo(), meta).context("Computing sizes") + }) + .transpose()?; + let mut opts = ExportOpts::default(); + let container_config = oci_spec::image::ConfigBuilder::default() + .stop_signal("SIGRTMIN+3") + .build() + .unwrap(); + opts.copy_meta_keys = vec!["buildsys.checksum".to_string()]; + opts.copy_meta_opt_keys = vec!["nosuchvalue".to_string()]; + opts.max_layers = std::num::NonZeroU32::new(PKGS_V0_LEN as u32); + opts.contentmeta = contentmeta.as_ref(); + opts.container_config = Some(container_config); + let digest = ostree_ext::container::encapsulate( + fixture.srcrepo(), + fixture.testref(), + &config, + Some(opts), + &srcoci_imgref, + ) + .await + .context("exporting")?; + assert!(srcoci_path.exists()); + + let inspect = skopeo_inspect(&srcoci_imgref.to_string())?; + // Legacy path includes this + assert!(!inspect.contains(r#""version": "42.0""#)); + // Also include the new standard version + assert!(inspect.contains(r#""org.opencontainers.image.version": "42.0""#)); + assert!(inspect.contains(r#""foo": "bar""#)); + assert!(inspect.contains(r#""test": "value""#)); + assert!(inspect.contains( + r#""buildsys.checksum": "41af286dc0b172ed2f1ca934fd2278de4a1192302ffa07087cea2682e7d372e3""# + )); + let cfg = skopeo_inspect_config(&srcoci_imgref.to_string())?; + let creation_time = + chrono::NaiveDateTime::parse_from_str(cfg.created().as_deref().unwrap(), "%+").unwrap(); + assert_eq!(creation_time.and_utc().timestamp(), 872879442); + let found_cfg = cfg.config().as_ref().unwrap(); + // unwrap. Unwrap. UnWrap. UNWRAP!!!!!!! + assert_eq!( + found_cfg + .cmd() + .as_ref() + .unwrap() + .get(0) + .as_ref() + .unwrap() + .as_str(), + "/usr/bin/bash" + ); + assert_eq!(found_cfg.stop_signal().as_deref().unwrap(), "SIGRTMIN+3"); + + let n_chunks = if chunked { LAYERS_V0_LEN } else { 1 }; + assert_eq!(cfg.rootfs().diff_ids().len(), n_chunks); + assert_eq!(cfg.history().len(), n_chunks); + + // Verify exporting to ociarchive + { + let archivepath = &fixture.path.join("export.ociarchive"); + let ociarchive_dest = ImageReference { + transport: Transport::OciArchive, + name: archivepath.as_str().to_string(), + }; + let _: oci_image::Digest = ostree_ext::container::encapsulate( + fixture.srcrepo(), + fixture.testref(), + &config, + None, + &ociarchive_dest, + ) + .await + .context("exporting to ociarchive") + .unwrap(); + assert!(archivepath.is_file()); + } + + let srcoci_unverified = OstreeImageReference { + sigverify: SignatureSource::ContainerPolicyAllowInsecure, + imgref: srcoci_imgref.clone(), + }; + + let (_, pushed_digest) = ostree_ext::container::fetch_manifest(&srcoci_unverified).await?; + assert_eq!(pushed_digest, digest); + + let (_, pushed_digest, _config) = + ostree_ext::container::fetch_manifest_and_config(&srcoci_unverified).await?; + assert_eq!(pushed_digest, digest); + + // No remote matching + let srcoci_unknownremote = OstreeImageReference { + sigverify: SignatureSource::OstreeRemote("unknownremote".to_string()), + imgref: srcoci_imgref.clone(), + }; + let r = ostree_ext::container::unencapsulate(fixture.destrepo(), &srcoci_unknownremote) + .await + .context("importing"); + assert_err_contains(r, r#"Remote "unknownremote" not found"#); + + // Test with a signature + let opts = glib::VariantDict::new(None); + opts.insert("gpg-verify", &true); + opts.insert("custom-backend", &"ostree-rs-ext"); + fixture + .destrepo() + .remote_add("myremote", None, Some(&opts.end()), gio::Cancellable::NONE)?; + cmd!( + sh, + "ostree --repo=dest/repo remote gpg-import --stdin myremote" + ) + .stdin(sh.read_file("src/gpghome/key1.asc")?) + .run()?; + let srcoci_verified = OstreeImageReference { + sigverify: SignatureSource::OstreeRemote("myremote".to_string()), + imgref: srcoci_imgref.clone(), + }; + let import = ostree_ext::container::unencapsulate(fixture.destrepo(), &srcoci_verified) + .await + .context("importing")?; + assert_eq!(import.ostree_commit, testrev.as_str()); + + let temp_unsigned = ImageReference { + transport: Transport::OciDir, + name: fixture.path.join("unsigned.ocidir").to_string(), + }; + let _ = ostree_ext::container::update_detached_metadata(&srcoci_imgref, &temp_unsigned, None) + .await + .unwrap(); + let temp_unsigned = OstreeImageReference { + sigverify: SignatureSource::OstreeRemote("myremote".to_string()), + imgref: temp_unsigned, + }; + fixture.clear_destrepo()?; + let r = ostree_ext::container::unencapsulate(fixture.destrepo(), &temp_unsigned).await; + assert_err_contains(r, "Expected commitmeta object"); + + // Test without signature verification + // Create a new repo + { + let fixture = Fixture::new_v1()?; + let import = ostree_ext::container::unencapsulate(fixture.destrepo(), &srcoci_unverified) + .await + .context("importing")?; + assert_eq!(import.ostree_commit, testrev.as_str()); + } + + Ok(()) +} + +#[tokio::test] +async fn test_export_as_container_nonderived() -> Result<()> { + let fixture = Fixture::new_v1()?; + // Export into an OCI directory + let src_imgref = fixture.export_container().await.unwrap().0; + + let initimport = fixture.must_import(&src_imgref).await?; + let initimport_ls = fixture::ostree_ls(fixture.destrepo(), &initimport.merge_commit).unwrap(); + + let exported_ocidir_name = "exported.ocidir"; + let dest = ImageReference { + transport: Transport::OciDir, + name: format!("{}:exported-test", fixture.path.join(exported_ocidir_name)), + }; + fixture.dir.create_dir(exported_ocidir_name)?; + let ocidir = ocidir::OciDir::ensure(&fixture.dir.open_dir(exported_ocidir_name)?)?; + let exported = store::export(fixture.destrepo(), &src_imgref, &dest, None) + .await + .unwrap(); + + let idx = ocidir.read_index()?.unwrap(); + let desc = idx.manifests().first().unwrap(); + let new_manifest: oci_image::ImageManifest = ocidir.read_json_blob(desc).unwrap(); + + assert_eq!(desc.digest().to_string(), exported.to_string()); + assert_eq!(new_manifest.layers().len(), fixture::LAYERS_V0_LEN); + + // Reset the destrepo + fixture.clear_destrepo()?; + // Clear out the original source + std::fs::remove_dir_all(src_imgref.name.as_str())?; + + let reimported = fixture.must_import(&dest).await?; + let reimport_ls = fixture::ostree_ls(fixture.destrepo(), &reimported.merge_commit).unwrap(); + similar_asserts::assert_eq!(initimport_ls, reimport_ls); + Ok(()) +} + +#[tokio::test] +async fn test_export_as_container_derived() -> Result<()> { + let fixture = Fixture::new_v1()?; + // Export into an OCI directory + let src_imgref = fixture.export_container().await.unwrap().0; + // Add a derived layer + let derived_tag = "derived"; + // Build a derived image + let srcpath = src_imgref.name.as_str(); + fixture.generate_test_derived_oci(srcpath, Some(&derived_tag))?; + let derived_imgref = ImageReference { + transport: src_imgref.transport.clone(), + name: format!("{}:{derived_tag}", src_imgref.name.as_str()), + }; + + // The first import into destrepo of the derived OCI + let initimport = fixture.must_import(&derived_imgref).await?; + let initimport_ls = fixture::ostree_ls(fixture.destrepo(), &initimport.merge_commit).unwrap(); + // Export it + let exported_ocidir_name = "exported.ocidir"; + let dest = ImageReference { + transport: Transport::OciDir, + name: format!("{}:exported-test", fixture.path.join(exported_ocidir_name)), + }; + fixture.dir.create_dir(exported_ocidir_name)?; + let ocidir = ocidir::OciDir::ensure(&fixture.dir.open_dir(exported_ocidir_name)?)?; + let exported = store::export(fixture.destrepo(), &derived_imgref, &dest, None) + .await + .unwrap(); + + let idx = ocidir.read_index()?.unwrap(); + let desc = idx.manifests().first().unwrap(); + let new_manifest: oci_image::ImageManifest = ocidir.read_json_blob(desc).unwrap(); + + assert_eq!(desc.digest().digest(), exported.digest()); + assert_eq!(new_manifest.layers().len(), fixture::LAYERS_V0_LEN + 1); + + // Reset the destrepo + fixture.clear_destrepo()?; + // Clear out the original source + std::fs::remove_dir_all(srcpath)?; + + let reimported = fixture.must_import(&dest).await?; + let reimport_ls = fixture::ostree_ls(fixture.destrepo(), &reimported.merge_commit).unwrap(); + similar_asserts::assert_eq!(initimport_ls, reimport_ls); + + Ok(()) +} + +#[tokio::test] +async fn test_unencapsulate_unbootable() -> Result<()> { + let fixture = { + let mut fixture = Fixture::new_base()?; + fixture.bootable = false; + fixture.commit_filedefs(FileDef::iter_from(ostree_ext::fixture::CONTENTS_V0))?; + fixture + }; + let testrev = fixture + .srcrepo() + .require_rev(fixture.testref()) + .context("Failed to resolve ref")?; + let srcoci_path = &fixture.path.join("oci"); + let srcoci_imgref = ImageReference { + transport: Transport::OciDir, + name: srcoci_path.as_str().to_string(), + }; + let srcoci_unverified = OstreeImageReference { + sigverify: SignatureSource::ContainerPolicyAllowInsecure, + imgref: srcoci_imgref.clone(), + }; + + let config = Config::default(); + let _digest = ostree_ext::container::encapsulate( + fixture.srcrepo(), + fixture.testref(), + &config, + None, + &srcoci_imgref, + ) + .await + .context("exporting")?; + assert!(srcoci_path.exists()); + + assert!(fixture + .destrepo() + .resolve_rev(fixture.testref(), true) + .unwrap() + .is_none()); + + let target = ostree_ext::container::unencapsulate(fixture.destrepo(), &srcoci_unverified) + .await + .unwrap(); + + assert_eq!(target.ostree_commit.as_str(), testrev.as_str()); + + Ok(()) +} + +/// Parse a chunked container image and validate its structure; particularly +fn validate_chunked_structure(oci_path: &Utf8Path) -> Result<()> { + use tar::EntryType::Link; + + let d = Dir::open_ambient_dir(oci_path, cap_std::ambient_authority())?; + let d = ocidir::OciDir::open(&d)?; + let idx = d.read_index()?.unwrap(); + let desc = idx.manifests().first().unwrap(); + let manifest: oci_image::ImageManifest = d.read_json_blob(desc).unwrap(); + + assert_eq!(manifest.layers().len(), LAYERS_V0_LEN); + let ostree_layer = manifest.layers().first().unwrap(); + let mut ostree_layer_blob = d + .read_blob(ostree_layer) + .map(BufReader::new) + .map(flate2::read::GzDecoder::new) + .map(tar::Archive::new)?; + let mut ostree_layer_blob = ostree_layer_blob.entries()?; + validate_tar_v1_metadata(&mut ostree_layer_blob)?; + + // This layer happens to be first + let pkgdb_layer_offset = 1; + let pkgdb_layer = &manifest.layers()[pkgdb_layer_offset]; + let mut pkgdb_blob = d + .read_blob(pkgdb_layer) + .map(BufReader::new) + .map(flate2::read::GzDecoder::new) + .map(tar::Archive::new)?; + + let pkgdb = [ + ("usr/lib/pkgdb/pkgdb", Link, 0o644), + ("usr/lib/sysimage/pkgdb", Link, 0o644), + ] + .into_iter() + .map(Into::into); + + validate_tar_expected(&mut pkgdb_blob.entries()?, pkgdb)?; + + Ok(()) +} + +#[tokio::test] +async fn test_container_arch_mismatch() -> Result<()> { + let fixture = Fixture::new_v1()?; + + let imgref = fixture.export_container().await.unwrap().0; + + // Build a derived image + let derived_path = &fixture.path.join("derived.oci"); + let srcpath = imgref.name.as_str(); + oci_clone(srcpath, derived_path).await.unwrap(); + ostree_ext::integrationtest::generate_derived_oci_from_tar( + derived_path, + |w| { + let mut layer_tar = tar::Builder::new(w); + let mut h = tar::Header::new_gnu(); + h.set_uid(0); + h.set_gid(0); + h.set_size(0); + h.set_mode(0o755); + h.set_entry_type(tar::EntryType::Directory); + layer_tar.append_data( + &mut h.clone(), + "etc/mips-operating-system", + &mut std::io::empty(), + )?; + layer_tar.into_inner()?; + Ok(()) + }, + None, + Some(Arch::Mips64le), + )?; + + let derived_imgref = OstreeImageReference { + sigverify: SignatureSource::ContainerPolicyAllowInsecure, + imgref: ImageReference { + transport: Transport::OciDir, + name: derived_path.to_string(), + }, + }; + let mut imp = + store::ImageImporter::new(fixture.destrepo(), &derived_imgref, Default::default()).await?; + imp.require_bootable(); + imp.set_ostree_version(2023, 11); + let r = imp.prepare().await; + assert_err_contains(r, "Image has architecture mips64le"); + + Ok(()) +} + +#[tokio::test] +async fn test_container_chunked() -> Result<()> { + let nlayers = LAYERS_V0_LEN - 1; + let mut fixture = Fixture::new_v1()?; + + let (imgref, expected_digest) = fixture.export_container().await.unwrap(); + let imgref = OstreeImageReference { + sigverify: SignatureSource::ContainerPolicyAllowInsecure, + imgref, + }; + // Validate the structure of the image + match &imgref.imgref { + ImageReference { + transport: Transport::OciDir, + name, + } => validate_chunked_structure(Utf8Path::new(name)).unwrap(), + _ => unreachable!(), + }; + + let mut imp = + store::ImageImporter::new(fixture.destrepo(), &imgref, Default::default()).await?; + assert!(store::query_image(fixture.destrepo(), &imgref.imgref) + .unwrap() + .is_none()); + let prep = match imp.prepare().await.context("Init prep derived")? { + store::PrepareResult::AlreadyPresent(_) => panic!("should not be already imported"), + store::PrepareResult::Ready(r) => r, + }; + assert!(prep.deprecated_warning().is_none()); + assert_eq!(prep.version(), Some("42.0")); + let digest = prep.manifest_digest.clone(); + assert!(prep.ostree_commit_layer.commit.is_none()); + assert_eq!(prep.ostree_layers.len(), nlayers); + assert_eq!(prep.layers.len(), 0); + for layer in prep.layers.iter() { + assert!(layer.commit.is_none()); + } + assert_eq!(digest, expected_digest); + { + let mut layer_history = prep.layers_with_history(); + assert!(layer_history + .next() + .unwrap()? + .1 + .created_by() + .as_ref() + .unwrap() + .starts_with("ostree export")); + assert_eq!( + layer_history + .next() + .unwrap()? + .1 + .created_by() + .as_ref() + .unwrap(), + "8 components" + ); + } + let import = imp.import(prep).await.context("Init pull derived").unwrap(); + assert_eq!(import.manifest_digest, digest); + + assert_eq!(store::list_images(fixture.destrepo()).unwrap().len(), 1); + + assert!( + store::image_filtered_content_warning(fixture.destrepo(), &imgref.imgref) + .unwrap() + .is_none() + ); + // Verify there are no updates. + let mut imp = + store::ImageImporter::new(fixture.destrepo(), &imgref, Default::default()).await?; + let state = match imp.prepare().await? { + store::PrepareResult::AlreadyPresent(i) => i, + store::PrepareResult::Ready(_) => panic!("should be already imported"), + }; + assert!(state.cached_update.is_none()); + + const ADDITIONS: &str = indoc::indoc! { " +r usr/bin/bash bash-v0 +"}; + fixture + .update(FileDef::iter_from(ADDITIONS), std::iter::empty()) + .context("Failed to update")?; + + let expected_digest = fixture.export_container().await.unwrap().1; + assert_ne!(digest, expected_digest); + + let mut imp = + store::ImageImporter::new(fixture.destrepo(), &imgref, Default::default()).await?; + let prep = match imp.prepare().await.context("Init prep derived")? { + store::PrepareResult::AlreadyPresent(_) => panic!("should not be already imported"), + store::PrepareResult::Ready(r) => r, + }; + // Verify we also serialized the cached update + { + let cached = store::query_image(fixture.destrepo(), &imgref.imgref) + .unwrap() + .unwrap(); + assert_eq!(cached.version(), Some("42.0")); + + let cached_update = cached.cached_update.unwrap(); + assert_eq!(cached_update.manifest_digest, prep.manifest_digest); + assert_eq!(cached_update.version(), Some("42.0")); + } + let to_fetch = prep.layers_to_fetch().collect::>>()?; + assert_eq!(to_fetch.len(), 2); + assert_eq!(expected_digest, prep.manifest_digest); + assert!(prep.ostree_commit_layer.commit.is_none()); + assert_eq!(prep.ostree_layers.len(), nlayers); + let (first, second) = (to_fetch[0], to_fetch[1]); + assert!(first.0.commit.is_none()); + assert!(second.0.commit.is_none()); + assert_eq!( + first.1, + "ostree export of commit fe4ba8bbd8f61a69ae53cde0dd53c637f26dfbc87717b2e71e061415d931361e" + ); + assert_eq!(second.1, "8 components"); + + assert_eq!(store::list_images(fixture.destrepo()).unwrap().len(), 1); + let n = store::count_layer_references(fixture.destrepo())? as i64; + let _import = imp.import(prep).await.unwrap(); + + assert_eq!(store::list_images(fixture.destrepo()).unwrap().len(), 1); + + let n2 = store::count_layer_references(fixture.destrepo())? as i64; + assert_eq!(n, n2); + fixture + .destrepo() + .prune(ostree::RepoPruneFlags::REFS_ONLY, 0, gio::Cancellable::NONE)?; + + // Build a derived image + let srcpath = imgref.imgref.name.as_str(); + let derived_tag = "derived"; + fixture.generate_test_derived_oci(srcpath, Some(&derived_tag))?; + + let derived_imgref = OstreeImageReference { + sigverify: SignatureSource::ContainerPolicyAllowInsecure, + imgref: ImageReference { + transport: Transport::OciDir, + name: format!("{srcpath}:{derived_tag}"), + }, + }; + let mut imp = + store::ImageImporter::new(fixture.destrepo(), &derived_imgref, Default::default()).await?; + let prep = match imp.prepare().await.unwrap() { + store::PrepareResult::AlreadyPresent(_) => panic!("should not be already imported"), + store::PrepareResult::Ready(r) => r, + }; + let to_fetch = prep.layers_to_fetch().collect::>>()?; + assert_eq!(to_fetch.len(), 1); + assert!(prep.ostree_commit_layer.commit.is_some()); + assert_eq!(prep.ostree_layers.len(), nlayers); + + // We want to test explicit layer pruning + imp.disable_gc(); + let _import = imp.import(prep).await.unwrap(); + assert_eq!(store::list_images(fixture.destrepo()).unwrap().len(), 2); + + assert!( + store::image_filtered_content_warning(fixture.destrepo(), &derived_imgref.imgref) + .unwrap() + .is_none() + ); + + // Should only be new layers + let n_removed = store::gc_image_layers(fixture.destrepo())?; + assert_eq!(n_removed, 0); + // Also test idempotence + store::remove_image(fixture.destrepo(), &imgref.imgref).unwrap(); + store::remove_image(fixture.destrepo(), &imgref.imgref).unwrap(); + assert_eq!(store::list_images(fixture.destrepo()).unwrap().len(), 1); + // Still no removed layers after removing the base image + let n_removed = store::gc_image_layers(fixture.destrepo())?; + assert_eq!(n_removed, 0); + store::remove_images(fixture.destrepo(), [&derived_imgref.imgref]).unwrap(); + assert_eq!(store::list_images(fixture.destrepo()).unwrap().len(), 0); + let n_removed = store::gc_image_layers(fixture.destrepo())?; + assert_eq!(n_removed, (LAYERS_V0_LEN + 1) as u32); + + // Repo should be clean now + assert_eq!(store::count_layer_references(fixture.destrepo())?, 0); + assert_eq!( + fixture + .destrepo() + .list_refs(None, gio::Cancellable::NONE) + .unwrap() + .len(), + 0 + ); + + Ok(()) +} + +#[tokio::test] +async fn test_container_var_content() -> Result<()> { + let fixture = Fixture::new_v1()?; + + let imgref = fixture.export_container().await.unwrap().0; + let imgref = OstreeImageReference { + sigverify: SignatureSource::ContainerPolicyAllowInsecure, + imgref, + }; + + // Build a derived image + let derived_path = &fixture.path.join("derived.oci"); + let srcpath = imgref.imgref.name.as_str(); + oci_clone(srcpath, derived_path).await.unwrap(); + let temproot = &fixture.path.join("temproot"); + let junk_var_data = "junk var data"; + || -> Result<_> { + std::fs::create_dir(temproot)?; + let temprootd = Dir::open_ambient_dir(temproot, cap_std::ambient_authority())?; + let mut db = DirBuilder::new(); + db.mode(0o755); + db.recursive(true); + temprootd.create_dir_with("var/lib", &db)?; + temprootd.write("var/lib/foo", junk_var_data)?; + Ok(()) + }() + .context("generating temp content")?; + ostree_ext::integrationtest::generate_derived_oci(derived_path, temproot, None)?; + + let derived_imgref = OstreeImageReference { + sigverify: SignatureSource::ContainerPolicyAllowInsecure, + imgref: ImageReference { + transport: Transport::OciDir, + name: derived_path.to_string(), + }, + }; + let mut imp = + store::ImageImporter::new(fixture.destrepo(), &derived_imgref, Default::default()).await?; + imp.set_ostree_version(2023, 11); + let prep = match imp.prepare().await.unwrap() { + store::PrepareResult::AlreadyPresent(_) => panic!("should not be already imported"), + store::PrepareResult::Ready(r) => r, + }; + let import = imp.import(prep).await.unwrap(); + + let ostree_root = fixture + .destrepo() + .read_commit(&import.merge_commit, gio::Cancellable::NONE)? + .0; + let varfile = ostree_root + .child("usr/share/factory/var/lib/foo") + .downcast::() + .unwrap(); + assert_eq!( + ostree_manual::repo_file_read_to_string(&varfile)?, + junk_var_data + ); + assert!(!ostree_root + .child("var/lib/foo") + .query_exists(gio::Cancellable::NONE)); + + assert!( + store::image_filtered_content_warning(fixture.destrepo(), &derived_imgref.imgref) + .unwrap() + .is_none() + ); + + // Reset things + fixture.clear_destrepo()?; + + let mut imp = + store::ImageImporter::new(fixture.destrepo(), &derived_imgref, Default::default()).await?; + imp.set_ostree_version(2024, 3); + let prep = match imp.prepare().await.unwrap() { + store::PrepareResult::AlreadyPresent(_) => panic!("should not be already imported"), + store::PrepareResult::Ready(r) => r, + }; + let import = imp.import(prep).await.unwrap(); + let ostree_root = fixture + .destrepo() + .read_commit(&import.merge_commit, gio::Cancellable::NONE)? + .0; + let varfile = ostree_root + .child("usr/share/factory/var/lib/foo") + .downcast::() + .unwrap(); + assert!(!varfile.query_exists(gio::Cancellable::NONE)); + assert!(ostree_root + .child("var/lib/foo") + .query_exists(gio::Cancellable::NONE)); + Ok(()) +} + +#[tokio::test] +async fn test_container_etc_hardlinked_absolute() -> Result<()> { + test_container_etc_hardlinked(true).await +} + +#[tokio::test] +async fn test_container_etc_hardlinked_relative() -> Result<()> { + test_container_etc_hardlinked(false).await +} + +async fn test_container_etc_hardlinked(absolute: bool) -> Result<()> { + let fixture = Fixture::new_v1()?; + + let imgref = fixture.export_container().await.unwrap().0; + let imgref = OstreeImageReference { + sigverify: SignatureSource::ContainerPolicyAllowInsecure, + imgref, + }; + + // Build a derived image + let derived_path = &fixture.path.join("derived.oci"); + let srcpath = imgref.imgref.name.as_str(); + oci_clone(srcpath, derived_path).await.unwrap(); + ostree_ext::integrationtest::generate_derived_oci_from_tar( + derived_path, + |w| { + let mut layer_tar = tar::Builder::new(w); + // Create a simple hardlinked file /etc/foo and /etc/bar in the tar stream, which + // needs usr/etc processing. + let mut h = tar::Header::new_gnu(); + h.set_uid(0); + h.set_gid(0); + h.set_size(0); + h.set_mode(0o755); + h.set_entry_type(tar::EntryType::Directory); + layer_tar.append_data(&mut h.clone(), "etc", &mut std::io::empty())?; + let testdata = "hardlinked test data"; + h.set_mode(0o644); + h.set_size(testdata.len().try_into().unwrap()); + h.set_entry_type(tar::EntryType::Regular); + layer_tar.append_data( + &mut h.clone(), + "etc/foo", + std::io::Cursor::new(testdata.as_bytes()), + )?; + h.set_entry_type(tar::EntryType::Link); + h.set_size(0); + layer_tar.append_link(&mut h.clone(), "etc/bar", "etc/foo")?; + + // Another case where we have /etc/dnf.conf and a hardlinked /ostree/repo/objects + // link into it - in this case we should ignore the hardlinked one. + let testdata = "hardlinked into object store"; + let mut h = tar::Header::new_ustar(); + h.set_mode(0o644); + h.set_mtime(42); + h.set_size(testdata.len().try_into().unwrap()); + h.set_entry_type(tar::EntryType::Regular); + layer_tar.append_data( + &mut h.clone(), + "etc/dnf.conf", + std::io::Cursor::new(testdata.as_bytes()), + )?; + h.set_entry_type(tar::EntryType::Link); + h.set_mtime(42); + h.set_size(0); + let path = "sysroot/ostree/repo/objects/45/7279b28b541ca20358bec8487c81baac6a3d5ed3cea019aee675137fab53cb.file"; + let target = "etc/dnf.conf"; + if absolute { + let ustarname = &mut h.as_ustar_mut().unwrap().name; + // The tar crate doesn't let us set absolute paths in tar archives, so we bypass + // it and just write to the path buffer directly. + assert!(path.len() < ustarname.len()); + ustarname[0..path.len()].copy_from_slice(path.as_bytes()); + h.set_link_name(target)?; + h.set_cksum(); + layer_tar.append(&mut h.clone(), std::io::empty())?; + } else { + layer_tar.append_link(&mut h.clone(), path, target)?; + } + layer_tar.finish()?; + Ok(()) + }, + None, + None, + )?; + + let derived_imgref = OstreeImageReference { + sigverify: SignatureSource::ContainerPolicyAllowInsecure, + imgref: ImageReference { + transport: Transport::OciDir, + name: derived_path.to_string(), + }, + }; + let mut imp = + store::ImageImporter::new(fixture.destrepo(), &derived_imgref, Default::default()).await?; + imp.set_ostree_version(2023, 11); + let prep = match imp.prepare().await.unwrap() { + store::PrepareResult::AlreadyPresent(_) => panic!("should not be already imported"), + store::PrepareResult::Ready(r) => r, + }; + let import = imp.import(prep).await.unwrap(); + let r = fixture + .destrepo() + .read_commit(import.get_commit(), gio::Cancellable::NONE)? + .0; + let foo = r.resolve_relative_path("usr/etc/foo"); + let foo = foo.downcast_ref::().unwrap(); + foo.ensure_resolved()?; + let bar = r.resolve_relative_path("usr/etc/bar"); + let bar = bar.downcast_ref::().unwrap(); + bar.ensure_resolved()?; + assert_eq!(foo.checksum(), bar.checksum()); + + let dnfconf = r.resolve_relative_path("usr/etc/dnf.conf"); + let dnfconf: &ostree::RepoFile = dnfconf.downcast_ref::().unwrap(); + dnfconf.ensure_resolved()?; + + Ok(()) +} + +/// Copy an OCI directory. +async fn oci_clone(src: impl AsRef, dest: impl AsRef) -> Result<()> { + let src = src.as_ref(); + let dest = dest.as_ref(); + // For now we just fork off `cp` and rely on reflinks, but we could and should + // explicitly hardlink blobs/sha256 e.g. + let cmd = tokio::process::Command::new("cp") + .args(["-a", "--reflink=auto"]) + .args([src, dest]) + .status() + .await?; + if !cmd.success() { + anyhow::bail!("cp failed"); + } + Ok(()) +} + +#[tokio::test] +async fn test_container_import_export_v1() { + impl_test_container_import_export(false).await.unwrap(); + impl_test_container_import_export(true).await.unwrap(); +} + +/// But layers work via the container::write module. +#[tokio::test] +async fn test_container_write_derive() -> Result<()> { + let cancellable = gio::Cancellable::NONE; + let fixture = Fixture::new_v1()?; + let sh = fixture.new_shell()?; + let base_oci_path = &fixture.path.join("exampleos.oci"); + let _digest = ostree_ext::container::encapsulate( + fixture.srcrepo(), + fixture.testref(), + &Config { + cmd: Some(vec!["/bin/bash".to_string()]), + ..Default::default() + }, + None, + &ImageReference { + transport: Transport::OciDir, + name: base_oci_path.to_string(), + }, + ) + .await + .context("exporting")?; + assert!(base_oci_path.exists()); + + // Build the derived images + let derived_path = &fixture.path.join("derived.oci"); + oci_clone(base_oci_path, derived_path).await?; + let temproot = &fixture.path.join("temproot"); + std::fs::create_dir_all(temproot.join("usr/bin"))?; + let newderivedfile_contents = "newderivedfile v0"; + std::fs::write( + temproot.join("usr/bin/newderivedfile"), + newderivedfile_contents, + )?; + std::fs::write( + temproot.join("usr/bin/newderivedfile3"), + "newderivedfile3 v0", + )?; + // Remove the kernel directory and make a new one + let moddir = temproot.join("usr/lib/modules"); + let oldkernel = "5.10.18-200.x86_64"; + std::fs::create_dir_all(&moddir)?; + let oldkernel_wh = &format!(".wh.{oldkernel}"); + std::fs::write(moddir.join(oldkernel_wh), "")?; + let newkdir = moddir.join("5.12.7-42.x86_64"); + std::fs::create_dir_all(&newkdir)?; + std::fs::write(newkdir.join("vmlinuz"), "a new kernel")?; + ostree_ext::integrationtest::generate_derived_oci(derived_path, temproot, None)?; + // And v2 + let derived2_path = &fixture.path.join("derived2.oci"); + oci_clone(base_oci_path, derived2_path).await?; + std::fs::remove_dir_all(temproot)?; + std::fs::create_dir_all(temproot.join("usr/bin"))?; + std::fs::write(temproot.join("usr/bin/newderivedfile"), "newderivedfile v1")?; + std::fs::write( + temproot.join("usr/bin/newderivedfile2"), + "newderivedfile2 v0", + )?; + ostree_ext::integrationtest::generate_derived_oci(derived2_path, temproot, None)?; + + let derived_ref = OstreeImageReference { + sigverify: SignatureSource::ContainerPolicyAllowInsecure, + imgref: ImageReference { + transport: Transport::OciDir, + name: derived_path.to_string(), + }, + }; + // There shouldn't be any container images stored yet. + let images = store::list_images(fixture.destrepo())?; + assert!(images.is_empty()); + + // Verify importing a derived image fails + let r = ostree_ext::container::unencapsulate(fixture.destrepo(), &derived_ref).await; + assert_err_contains(r, "Image has 1 non-ostree layers"); + + // Pull a derived image - two layers, new base plus one layer. + let mut imp = + store::ImageImporter::new(fixture.destrepo(), &derived_ref, Default::default()).await?; + let prep = match imp.prepare().await.context("Init prep derived")? { + store::PrepareResult::AlreadyPresent(_) => panic!("should not be already imported"), + store::PrepareResult::Ready(r) => r, + }; + let expected_digest = prep.manifest_digest.clone(); + assert!(prep.ostree_commit_layer.commit.is_none()); + assert_eq!(prep.layers.len(), 1); + for layer in prep.layers.iter() { + assert!(layer.commit.is_none()); + } + let import = imp.import(prep).await.context("Init pull derived")?; + // We should have exactly one image stored. + let images = store::list_images(fixture.destrepo())?; + assert_eq!(images.len(), 1); + assert_eq!(images[0], derived_ref.imgref.to_string()); + + let imported_commit = &fixture + .destrepo() + .load_commit(import.merge_commit.as_str())? + .0; + let digest = store::manifest_digest_from_commit(imported_commit)?; + assert_eq!(digest.algorithm(), &DigestAlgorithm::Sha256); + assert_eq!(digest, expected_digest); + + let commit_meta = &imported_commit.child_value(0); + let commit_meta = glib::VariantDict::new(Some(commit_meta)); + let config = commit_meta + .lookup::("ostree.container.image-config")? + .unwrap(); + let config: oci_spec::image::ImageConfiguration = serde_json::from_str(&config)?; + assert_eq!(config.os(), &oci_spec::image::Os::Linux); + + // Parse the commit and verify we pulled the derived content. + let root = fixture + .destrepo() + .read_commit(&import.merge_commit, cancellable)? + .0; + let root = root.downcast_ref::().unwrap(); + { + let derived = root.resolve_relative_path("usr/bin/newderivedfile"); + let derived = derived.downcast_ref::().unwrap(); + let found_newderived_contents = + ostree_ext::ostree_manual::repo_file_read_to_string(derived)?; + assert_eq!(found_newderived_contents, newderivedfile_contents); + + let kver = ostree_ext::bootabletree::find_kernel_dir(root.upcast_ref(), cancellable) + .unwrap() + .unwrap() + .basename() + .unwrap(); + let kver = Utf8Path::from_path(&kver).unwrap(); + assert_eq!(kver, newkdir.file_name().unwrap()); + + let old_kernel_dir = root.resolve_relative_path(format!("usr/lib/modules/{oldkernel}")); + assert!(!old_kernel_dir.query_exists(cancellable)); + } + + // Import again, but there should be no changes. + let mut imp = + store::ImageImporter::new(fixture.destrepo(), &derived_ref, Default::default()).await?; + let already_present = match imp.prepare().await? { + store::PrepareResult::AlreadyPresent(c) => c, + store::PrepareResult::Ready(_) => { + panic!("Should have already imported {}", &derived_ref) + } + }; + assert_eq!(import.merge_commit, already_present.merge_commit); + + // Test upgrades; replace the oci-archive with new content. + std::fs::remove_dir_all(derived_path)?; + std::fs::rename(derived2_path, derived_path)?; + let mut imp = + store::ImageImporter::new(fixture.destrepo(), &derived_ref, Default::default()).await?; + let prep = match imp.prepare().await? { + store::PrepareResult::AlreadyPresent(_) => panic!("should not be already imported"), + store::PrepareResult::Ready(r) => r, + }; + // We *should* already have the base layer. + assert!(prep.ostree_commit_layer.commit.is_some()); + // One new layer + assert_eq!(prep.layers.len(), 1); + for layer in prep.layers.iter() { + assert!(layer.commit.is_none()); + } + let import = imp.import(prep).await?; + // New commit. + assert_ne!(import.merge_commit, already_present.merge_commit); + // We should still have exactly one image stored. + let images = store::list_images(fixture.destrepo())?; + assert_eq!(images[0], derived_ref.imgref.to_string()); + assert_eq!(images.len(), 1); + + // Verify we have the new file and *not* the old one + let merge_commit = import.merge_commit.as_str(); + cmd!( + sh, + "ostree --repo=dest/repo ls {merge_commit} /usr/bin/newderivedfile2" + ) + .ignore_stdout() + .run()?; + let c = cmd!( + sh, + "ostree --repo=dest/repo cat {merge_commit} /usr/bin/newderivedfile" + ) + .read()?; + assert_eq!(c.as_str(), "newderivedfile v1"); + assert!(cmd!( + sh, + "ostree --repo=dest/repo ls {merge_commit} /usr/bin/newderivedfile3" + ) + .ignore_stderr() + .run() + .is_err()); + + // And there should be no changes on upgrade again. + let mut imp = + store::ImageImporter::new(fixture.destrepo(), &derived_ref, Default::default()).await?; + let already_present = match imp.prepare().await? { + store::PrepareResult::AlreadyPresent(c) => c, + store::PrepareResult::Ready(_) => { + panic!("Should have already imported {}", &derived_ref) + } + }; + assert_eq!(import.merge_commit, already_present.merge_commit); + + // Create a new repo, and copy to it + let destrepo2 = ostree::Repo::create_at( + ostree::AT_FDCWD, + fixture.path.join("destrepo2").as_str(), + ostree::RepoMode::Archive, + None, + gio::Cancellable::NONE, + )?; + #[allow(deprecated)] + store::copy( + fixture.destrepo(), + &derived_ref.imgref, + &destrepo2, + &derived_ref.imgref, + ) + .await + .context("Copying")?; + + let images = store::list_images(&destrepo2)?; + assert_eq!(images.len(), 1); + assert_eq!(images[0], derived_ref.imgref.to_string()); + + // And test copy_as + let target_name = "quay.io/exampleos/centos:stream9"; + let registry_ref = ImageReference { + transport: Transport::Registry, + name: target_name.to_string(), + }; + store::copy( + fixture.destrepo(), + &derived_ref.imgref, + &destrepo2, + ®istry_ref, + ) + .await + .context("Copying")?; + + let mut images = store::list_images(&destrepo2)?; + images.sort_unstable(); + assert_eq!(images[0], registry_ref.to_string()); + assert_eq!(images[1], derived_ref.imgref.to_string()); + + Ok(()) +} + +/// Implementation of a test case for non-gzip (i.e. zstd or zstd:chunked) compression +async fn test_non_gzip(format: &str) -> Result<()> { + let fixture = Fixture::new_v1()?; + let baseimg = &fixture.export_container().await?.0; + let basepath = &match baseimg.transport { + Transport::OciDir => fixture.path.join(baseimg.name.as_str()), + _ => unreachable!(), + }; + let baseimg_ref = format!("oci:{basepath}"); + let zstd_image_path = &fixture.path.join("zstd.oci"); + let st = tokio::process::Command::new("skopeo") + .args([ + "copy", + &format!("--dest-compress-format={format}"), + baseimg_ref.as_str(), + &format!("oci:{zstd_image_path}"), + ]) + .status() + .await?; + assert!(st.success()); + + let zstdref = &OstreeImageReference { + sigverify: SignatureSource::ContainerPolicyAllowInsecure, + imgref: ImageReference { + transport: Transport::OciDir, + name: zstd_image_path.to_string(), + }, + }; + let mut imp = + store::ImageImporter::new(fixture.destrepo(), zstdref, Default::default()).await?; + let prep = match imp.prepare().await.context("Init prep derived")? { + store::PrepareResult::AlreadyPresent(_) => panic!("should not be already imported"), + store::PrepareResult::Ready(r) => r, + }; + let _ = imp.import(prep).await.unwrap(); + + Ok(()) +} + +/// Test for zstd +#[tokio::test] +async fn test_container_zstd() -> Result<()> { + test_non_gzip("zstd").await +} + +/// Test for zstd:chunked +#[tokio::test] +async fn test_container_zstd_chunked() -> Result<()> { + test_non_gzip("zstd:chunked").await +} + +/// Test for https://github.com/ostreedev/ostree-rs-ext/issues/405 +/// We need to handle the case of modified hardlinks into /sysroot +#[tokio::test] +async fn test_container_write_derive_sysroot_hardlink() -> Result<()> { + let fixture = Fixture::new_v1()?; + let sh = fixture.new_shell()?; + let baseimg = &fixture.export_container().await?.0; + let basepath = &match baseimg.transport { + Transport::OciDir => fixture.path.join(baseimg.name.as_str()), + _ => unreachable!(), + }; + + // Build a derived image + let derived_path = &fixture.path.join("derived.oci"); + oci_clone(basepath, derived_path).await?; + ostree_ext::integrationtest::generate_derived_oci_from_tar( + derived_path, + |w| { + let mut tar = tar::Builder::new(w); + let objpath = Utf8Path::new("sysroot/ostree/repo/objects/60/feb13e826d2f9b62490ab24cea0f4a2d09615fb57027e55f713c18c59f4796.file"); + let d = objpath.parent().unwrap(); + fn mkparents( + t: &mut tar::Builder, + path: &Utf8Path, + ) -> std::io::Result<()> { + if let Some(parent) = path.parent().filter(|p| !p.as_str().is_empty()) { + mkparents(t, parent)?; + } + let mut h = tar::Header::new_gnu(); + h.set_entry_type(tar::EntryType::Directory); + h.set_uid(0); + h.set_gid(0); + h.set_mode(0o755); + h.set_size(0); + t.append_data(&mut h, path, std::io::empty()) + } + mkparents(&mut tar, d).context("Appending parent")?; + + let now = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH)? + .as_secs(); + let mut h = tar::Header::new_gnu(); + h.set_entry_type(tar::EntryType::Regular); + h.set_uid(0); + h.set_gid(0); + h.set_mode(0o644); + h.set_mtime(now); + let data = b"hello"; + h.set_size(data.len() as u64); + tar.append_data(&mut h, objpath, std::io::Cursor::new(data)) + .context("appending object")?; + for path in ["usr/bin/bash", "usr/bin/bash-hardlinked"] { + let targetpath = Utf8Path::new(path); + h.set_size(0); + h.set_mtime(now); + h.set_entry_type(tar::EntryType::Link); + tar.append_link(&mut h, targetpath, objpath) + .context("appending target")?; + } + Ok::<_, anyhow::Error>(()) + }, + None, + None, + )?; + let derived_ref = &OstreeImageReference { + sigverify: SignatureSource::ContainerPolicyAllowInsecure, + imgref: ImageReference { + transport: Transport::OciDir, + name: derived_path.to_string(), + }, + }; + let mut imp = + store::ImageImporter::new(fixture.destrepo(), derived_ref, Default::default()).await?; + let prep = match imp.prepare().await.context("Init prep derived")? { + store::PrepareResult::AlreadyPresent(_) => panic!("should not be already imported"), + store::PrepareResult::Ready(r) => r, + }; + let import = imp.import(prep).await.unwrap(); + + // Verify we have the new file + let merge_commit = import.merge_commit.as_str(); + cmd!( + sh, + "ostree --repo=dest/repo ls {merge_commit} /usr/bin/bash" + ) + .ignore_stdout() + .run()?; + for path in ["/usr/bin/bash", "/usr/bin/bash-hardlinked"] { + let r = cmd!(sh, "ostree --repo=dest/repo cat {merge_commit} {path}").read()?; + assert_eq!(r.as_str(), "hello"); + } + + Ok(()) +} + +#[tokio::test] +// Today rpm-ostree vendors a stable ostree-rs-ext; this test +// verifies that the old ostree-rs-ext code can parse the containers +// generated by the new ostree code. +async fn test_old_code_parses_new_export() -> Result<()> { + let rpmostree = Utf8Path::new("/usr/bin/rpm-ostree"); + if !rpmostree.exists() { + return Ok(()); + } + let fixture = Fixture::new_v1()?; + let imgref = fixture.export_container().await?.0; + let imgref = OstreeImageReference { + sigverify: SignatureSource::ContainerPolicyAllowInsecure, + imgref, + }; + fixture.clear_destrepo()?; + let destrepo_path = fixture.path.join("dest/repo"); + let s = Command::new("ostree") + .args([ + "container", + "unencapsulate", + "--repo", + destrepo_path.as_str(), + imgref.to_string().as_str(), + ]) + .output()?; + if !s.status.success() { + anyhow::bail!( + "Failed to run ostree: {:?}: {}", + s, + String::from_utf8_lossy(&s.stderr) + ); + } + Ok(()) +} + +/// Test for https://github.com/ostreedev/ostree-rs-ext/issues/655 +#[tokio::test] +async fn test_container_xattr() -> Result<()> { + let fixture = Fixture::new_v1()?; + let sh = fixture.new_shell()?; + let baseimg = &fixture.export_container().await?.0; + let basepath = &match baseimg.transport { + Transport::OciDir => fixture.path.join(baseimg.name.as_str()), + _ => unreachable!(), + }; + + // Build a derived image + let derived_path = &fixture.path.join("derived.oci"); + oci_clone(basepath, derived_path).await?; + ostree_ext::integrationtest::generate_derived_oci_from_tar( + derived_path, + |w| { + let mut tar = tar::Builder::new(w); + let mut h = tar::Header::new_gnu(); + h.set_entry_type(tar::EntryType::Regular); + h.set_uid(0); + h.set_gid(0); + h.set_mode(0o644); + h.set_mtime(0); + let data = b"hello"; + h.set_size(data.len() as u64); + tar.append_pax_extensions([("SCHILY.xattr.user.foo", b"bar".as_slice())]) + .unwrap(); + tar.append_data(&mut h, "usr/bin/testxattr", std::io::Cursor::new(data)) + .unwrap(); + Ok::<_, anyhow::Error>(()) + }, + None, + None, + )?; + let derived_ref = &OstreeImageReference { + sigverify: SignatureSource::ContainerPolicyAllowInsecure, + imgref: ImageReference { + transport: Transport::OciDir, + name: derived_path.to_string(), + }, + }; + let mut imp = + store::ImageImporter::new(fixture.destrepo(), derived_ref, Default::default()).await?; + let prep = match imp.prepare().await.context("Init prep derived")? { + store::PrepareResult::AlreadyPresent(_) => panic!("should not be already imported"), + store::PrepareResult::Ready(r) => r, + }; + let import = imp.import(prep).await.unwrap(); + let merge_commit = import.merge_commit; + + // Yeah we just scrape the output of ostree because it's easy + let out = cmd!( + sh, + "ostree --repo=dest/repo ls -X {merge_commit} /usr/bin/testxattr" + ) + .read()?; + assert!(out.contains("'user.foo', [byte 0x62, 0x61, 0x72]")); + + Ok(()) +} + +#[ignore] +#[tokio::test] +// Verify that we can push and pull to a registry, not just oci-archive:. +// This requires a registry set up externally right now. One can run a HTTP registry via e.g. +// `podman run --rm -ti -p 5000:5000 --name registry docker.io/library/registry:2` +// but that doesn't speak HTTPS and adding that is complex. +// A simple option is setting up e.g. quay.io/$myuser/exampleos and then do: +// Then you can run this test via `env TEST_REGISTRY=quay.io/$myuser cargo test -- --ignored`. +async fn test_container_import_export_registry() -> Result<()> { + let tr = &*TEST_REGISTRY; + let fixture = Fixture::new_v1()?; + let testref = fixture.testref(); + let testrev = fixture + .srcrepo() + .require_rev(testref) + .context("Failed to resolve ref")?; + let src_imgref = ImageReference { + transport: Transport::Registry, + name: format!("{}/exampleos", tr), + }; + let config = Config { + cmd: Some(vec!["/bin/bash".to_string()]), + ..Default::default() + }; + let digest = + ostree_ext::container::encapsulate(fixture.srcrepo(), testref, &config, None, &src_imgref) + .await + .context("exporting to registry")?; + let mut digested_imgref = src_imgref.clone(); + digested_imgref.name = format!("{}@{}", src_imgref.name, digest); + + let import_ref = OstreeImageReference { + sigverify: SignatureSource::ContainerPolicyAllowInsecure, + imgref: digested_imgref, + }; + let import = ostree_ext::container::unencapsulate(fixture.destrepo(), &import_ref) + .await + .context("importing")?; + assert_eq!(import.ostree_commit, testrev.as_str()); + Ok(()) +} + +#[test] +fn test_diff() -> Result<()> { + let mut fixture = Fixture::new_v1()?; + const ADDITIONS: &str = indoc::indoc! { " +r /usr/bin/newbin some-new-binary +d /usr/share +"}; + fixture + .update( + FileDef::iter_from(ADDITIONS), + [Cow::Borrowed("/usr/bin/bash".into())].into_iter(), + ) + .context("Failed to update")?; + let from = &format!("{}^", fixture.testref()); + let repo = fixture.srcrepo(); + let subdir: Option<&str> = None; + let diff = ostree_ext::diff::diff(repo, from, fixture.testref(), subdir)?; + assert!(diff.subdir.is_none()); + assert_eq!(diff.added_dirs.len(), 1); + assert_eq!(diff.added_dirs.iter().next().unwrap(), "/usr/share"); + assert_eq!(diff.added_files.len(), 1); + assert_eq!(diff.added_files.iter().next().unwrap(), "/usr/bin/newbin"); + assert_eq!(diff.removed_files.len(), 1); + assert_eq!(diff.removed_files.iter().next().unwrap(), "/usr/bin/bash"); + let diff = ostree_ext::diff::diff(repo, from, fixture.testref(), Some("/usr"))?; + assert_eq!(diff.subdir.as_ref().unwrap(), "/usr"); + assert_eq!(diff.added_dirs.len(), 1); + assert_eq!(diff.added_dirs.iter().next().unwrap(), "/share"); + assert_eq!(diff.added_files.len(), 1); + assert_eq!(diff.added_files.iter().next().unwrap(), "/bin/newbin"); + assert_eq!(diff.removed_files.len(), 1); + assert_eq!(diff.removed_files.iter().next().unwrap(), "/bin/bash"); + Ok(()) +} + +#[test] +fn test_manifest_diff() { + let a: ImageManifest = serde_json::from_str(include_str!("fixtures/manifest1.json")).unwrap(); + let b: ImageManifest = serde_json::from_str(include_str!("fixtures/manifest2.json")).unwrap(); + + let d = ManifestDiff::new(&a, &b); + assert_eq!(d.from, &a); + assert_eq!(d.to, &b); + assert_eq!(d.added.len(), 4); + assert_eq!( + d.added[0].digest().to_string(), + "sha256:0b5d930ffc92d444b0a7b39beed322945a3038603fbe2a56415a6d02d598df1f" + ); + assert_eq!( + d.added[3].digest().to_string(), + "sha256:cb9b8a4ac4a8df62df79e6f0348a14b3ec239816d42985631c88e76d4e3ff815" + ); + assert_eq!(d.removed.len(), 4); + assert_eq!( + d.removed[0].digest().to_string(), + "sha256:0ff8b1fdd38e5cfb6390024de23ba4b947cd872055f62e70f2c21dad5c928925" + ); + assert_eq!( + d.removed[3].digest().to_string(), + "sha256:76b83eea62b7b93200a056b5e0201ef486c67f1eeebcf2c7678ced4d614cece2" + ); +}