From 541d375bad320331cfc2b5b51512382da68fffca Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Fri, 15 Nov 2024 16:18:31 +0100 Subject: [PATCH 01/10] sandbox; Only readlink() if the target is a symlink --- mkosi/sandbox.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkosi/sandbox.py b/mkosi/sandbox.py index 0cb86378e..4f2714df0 100755 --- a/mkosi/sandbox.py +++ b/mkosi/sandbox.py @@ -562,7 +562,7 @@ def execute(self, oldroot: str, newroot: str) -> None: try: os.symlink(self.src, dst) except FileExistsError: - if os.readlink(dst) == self.src: + if os.path.islink(dst) and os.readlink(dst) == self.src: return raise From b75120c281785426db2ad0d41b4c6271d43b2422 Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Fri, 15 Nov 2024 16:22:59 +0100 Subject: [PATCH 02/10] sandbox: Make --symlink work on top of existing files and symlinks With the new mount API, we can mount on top of existing symlinks and files, so let's make use of that for --symlink. --- mkosi/resources/man/mkosi-sandbox.1.md | 4 +++- mkosi/sandbox.py | 11 +++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/mkosi/resources/man/mkosi-sandbox.1.md b/mkosi/resources/man/mkosi-sandbox.1.md index 691f170fc..b171e250a 100644 --- a/mkosi/resources/man/mkosi-sandbox.1.md +++ b/mkosi/resources/man/mkosi-sandbox.1.md @@ -54,7 +54,9 @@ host system. : Like `--bind-try`, but does a recursive readonly bind mount. `--symlink SRC DST` -: Creates a symlink at `DST` in the sandbox pointing to `SRC`. +: Creates a symlink at `DST` in the sandbox pointing to `SRC`. If `DST` already + exists and is a file or symlink, a temporary symlink is created and mounted on + top of `DST`. `--write DATA DST` : Writes the string from `DATA` to `DST` in the sandbox. diff --git a/mkosi/sandbox.py b/mkosi/sandbox.py index 4f2714df0..249f3332c 100755 --- a/mkosi/sandbox.py +++ b/mkosi/sandbox.py @@ -560,12 +560,19 @@ def __init__(self, src: str, dst: str) -> None: def execute(self, oldroot: str, newroot: str) -> None: dst = joinpath(newroot, self.dst) try: - os.symlink(self.src, dst) + return os.symlink(self.src, dst) except FileExistsError: if os.path.islink(dst) and os.readlink(dst) == self.src: return - raise + if os.path.isdir(dst): + raise + + # If the target already exists and is not a directory, create the symlink somewhere else and mount + # it over the existing file or symlink. + os.symlink(self.src, "/symlink") + mount_rbind("/symlink", dst) + os.unlink("/symlink") class WriteOperation(FSOperation): From 04cd6d1434aa5dd5dea4ab360ce03dd1958dc403 Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Fri, 15 Nov 2024 16:24:18 +0100 Subject: [PATCH 03/10] Use passwd symlinks instead of bind mounts Bind mounts don't reflect changes to the original files if they're replaced instead of modified. Let's use symlinks instead so that changes to the original files are always reflected. Fixes #3189 --- mkosi/__init__.py | 7 +++---- mkosi/archive.py | 8 ++++---- mkosi/installer/__init__.py | 9 ++++----- mkosi/run.py | 5 ++--- 4 files changed, 13 insertions(+), 16 deletions(-) diff --git a/mkosi/__init__.py b/mkosi/__init__.py index b9f99aafb..9c6d92a41 100644 --- a/mkosi/__init__.py +++ b/mkosi/__init__.py @@ -103,7 +103,7 @@ chroot_cmd, chroot_options, finalize_interpreter, - finalize_passwd_mounts, + finalize_passwd_symlinks, fork_and_wait, run, workdir, @@ -2822,9 +2822,8 @@ def run_tmpfiles(context: Context) -> None: options=[ "--bind", context.root, "/buildroot", # systemd uses acl.h to parse ACLs in tmpfiles snippets which uses the host's - # passwd so we have to mount the image's passwd over it to make ACL parsing - # work. - *finalize_passwd_mounts(context.root), + # passwd so we have to symlink the image's passwd to make ACL parsing work. + *finalize_passwd_symlinks("/buildroot"), # Sometimes directories are configured to be owned by root in tmpfiles snippets # so we want to make sure those chown()'s succeed by making ourselves the root # user so that the root user exists. diff --git a/mkosi/archive.py b/mkosi/archive.py index b735955bf..5b5b6ea91 100644 --- a/mkosi/archive.py +++ b/mkosi/archive.py @@ -6,7 +6,7 @@ from typing import Optional from mkosi.log import log_step -from mkosi.run import SandboxProtocol, finalize_passwd_mounts, nosandbox, run, workdir +from mkosi.run import SandboxProtocol, finalize_passwd_symlinks, nosandbox, run, workdir from mkosi.sandbox import umask from mkosi.types import PathString from mkosi.util import chdir @@ -51,7 +51,7 @@ def make_tar(src: Path, dst: Path, *, sandbox: SandboxProtocol = nosandbox) -> N # Make sure tar uses user/group information from the root directory instead of the host. sandbox=sandbox( binary="tar", - options=["--ro-bind", src, workdir(src), *finalize_passwd_mounts(src)], + options=["--ro-bind", src, workdir(src), *finalize_passwd_symlinks(workdir(src))], ), ) # fmt: skip @@ -98,7 +98,7 @@ def extract_tar( options=[ "--ro-bind", src, workdir(src), "--bind", dst, workdir(dst), - *finalize_passwd_mounts(dst), + *finalize_passwd_symlinks(workdir(dst)), ], ), ) # fmt: skip @@ -136,6 +136,6 @@ def make_cpio( stdout=f, sandbox=sandbox( binary="cpio", - options=["--ro-bind", src, workdir(src), *finalize_passwd_mounts(src)], + options=["--ro-bind", src, workdir(src), *finalize_passwd_symlinks(workdir(src))], ), ) # fmt: skip diff --git a/mkosi/installer/__init__.py b/mkosi/installer/__init__.py index 832f11f22..fc9713420 100644 --- a/mkosi/installer/__init__.py +++ b/mkosi/installer/__init__.py @@ -7,7 +7,7 @@ from mkosi.config import Config, ConfigFeature, OutputFormat from mkosi.context import Context from mkosi.mounts import finalize_crypto_mounts -from mkosi.run import apivfs_options, finalize_interpreter, finalize_passwd_mounts, find_binary +from mkosi.run import apivfs_options, finalize_interpreter, finalize_passwd_symlinks, find_binary from mkosi.tree import rmtree from mkosi.types import PathString from mkosi.util import flatten, startswith @@ -109,10 +109,9 @@ def options(cls, *, root: PathString, apivfs: bool = True) -> list[PathString]: "--suppress-chown", # Make sure /etc/machine-id is not overwritten by any package manager post install scripts. "--ro-bind-try", Path(root) / "etc/machine-id", "/buildroot/etc/machine-id", - # If we're already in the sandbox, we want to pick up use the passwd files from /buildroot since - # the original root won't be available anymore. If we're not in the sandbox yet, we want to pick - # up the passwd files from the original root. - *finalize_passwd_mounts(root), + # Some package managers (e.g. dpkg) read from the host's /etc/passwd instead of the buildroot's + # /etc/passwd so we symlink /etc/passwd from the buildroot to make sure it gets used. + *(finalize_passwd_symlinks("/buildroot") if apivfs else []), ] # fmt: skip @classmethod diff --git a/mkosi/run.py b/mkosi/run.py index 8c507655d..b777fffb7 100644 --- a/mkosi/run.py +++ b/mkosi/run.py @@ -415,15 +415,14 @@ def workdir(path: Path, sandbox: Optional[SandboxProtocol] = None) -> str: return joinpath(subdir, str(path)) -def finalize_passwd_mounts(root: PathString) -> list[PathString]: +def finalize_passwd_symlinks(root: PathString) -> list[PathString]: """ If passwd or a related file exists in the apivfs directory, bind mount it over the host files while we run the command, to make sure that the command we run uses user/group information from the apivfs directory instead of from the host. """ return flatten( - ("--ro-bind-try", Path(root) / "etc" / f, f"/etc/{f}") - for f in ("passwd", "group", "shadow", "gshadow") + ("--symlink", Path(root) / "etc" / f, f"/etc/{f}") for f in ("passwd", "group", "shadow", "gshadow") ) From f447553d170db47d74e3dc02f74bb40c2bf207c3 Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Sat, 16 Nov 2024 15:47:43 +0100 Subject: [PATCH 04/10] Add ToolsTreePackageDirectories= Sometimes, we want to add locally built rpm packages to the default tools tree. For example, systemd-repart depends on mkfs binaries that might not be available on the host system, so the only way to run it is from within the tools tree, which means we need a way to install systemd-repart built from source within the tools tree. --- mkosi/__init__.py | 1 + mkosi/config.py | 10 ++++++++++ mkosi/resources/man/mkosi.1.md | 3 +++ tests/test_json.py | 4 ++++ 4 files changed, 18 insertions(+) diff --git a/mkosi/__init__.py b/mkosi/__init__.py index 9c6d92a41..928a973db 100644 --- a/mkosi/__init__.py +++ b/mkosi/__init__.py @@ -4161,6 +4161,7 @@ def finalize_default_tools(args: Args, config: Config, *, resources: Path) -> Co *(["--package-cache-dir", str(config.package_cache_dir)] if config.package_cache_dir else []), "--incremental", str(config.incremental), *([f"--package={package}" for package in config.tools_tree_packages]), + *([f"--package-directory={directory}" for directory in config.tools_tree_package_directories]), "--output=tools", *(["--source-date-epoch", str(config.source_date_epoch)] if config.source_date_epoch is not None else []), # noqa: E501 *([f"--environment={k}='{v}'" for k, v in config.environment.items()]), diff --git a/mkosi/config.py b/mkosi/config.py index 960705fc2..16d0e7288 100644 --- a/mkosi/config.py +++ b/mkosi/config.py @@ -1818,6 +1818,7 @@ class Config: tools_tree_repositories: list[str] tools_tree_sandbox_trees: list[ConfigTree] tools_tree_packages: list[str] + tools_tree_package_directories: list[Path] tools_tree_certificates: bool incremental: Incremental cacheonly: Cacheonly @@ -3166,6 +3167,14 @@ def parse_ini(path: Path, only_sections: Collection[str] = ()) -> Iterator[tuple parse=config_make_list_parser(delimiter=","), help="Add additional packages to the default tools tree", ), + ConfigSetting( + dest="tools_tree_package_directories", + long="--tools-tree-package-directory", + metavar="PATH", + section="Build", + parse=config_make_list_parser(delimiter=",", parse=make_path_parser()), + help="Specify a directory containing extra tools tree packages", + ), ConfigSetting( dest="tools_tree_certificates", metavar="BOOL", @@ -4813,6 +4822,7 @@ def summary(config: Config) -> str: Tools Tree Repositories: {line_join_list(config.tools_tree_repositories)} Tools Tree Sandbox Trees: {line_join_list(config.tools_tree_sandbox_trees)} Tools Tree Packages: {line_join_list(config.tools_tree_packages)} + Tools Tree Package Directories: {line_join_list(config.tools_tree_package_directories)} Tools Tree Certificates: {yes_no(config.tools_tree_certificates)} Incremental: {config.incremental} diff --git a/mkosi/resources/man/mkosi.1.md b/mkosi/resources/man/mkosi.1.md index 6de5aecc8..70506757d 100644 --- a/mkosi/resources/man/mkosi.1.md +++ b/mkosi/resources/man/mkosi.1.md @@ -1331,6 +1331,9 @@ boolean argument: either `1`, `yes`, or `true` to enable, or `0`, `no`, separated list of package specifications. This option may be used multiple times in which case the specified package lists are combined. +`ToolsTreePackageDirectories=`, `--tools-tree-package-directory=` +: Same as `PackageDirectories=`, but for the default tools tree. + `ToolsTreeCertificates=`, `--tools-tree-certificates=` : Specify whether to use certificates and keys from the tools tree. Enabled by default. If enabled, `/etc/pki`, `/etc/ssl`, diff --git a/tests/test_json.py b/tests/test_json.py index 5a43ed234..4097951d8 100644 --- a/tests/test_json.py +++ b/tests/test_json.py @@ -363,6 +363,9 @@ def test_config() -> None: "ToolsTreeCertificates": true, "ToolsTreeDistribution": "fedora", "ToolsTreeMirror": null, + "ToolsTreePackageDirectories": [ + "/abc" + ], "ToolsTreePackages": [], "ToolsTreeRelease": null, "ToolsTreeRepositories": [ @@ -570,6 +573,7 @@ def test_config() -> None: tools_tree_mirror=None, tools_tree_sandbox_trees=[ConfigTree(Path("/a/b/c"), Path("/"))], tools_tree_packages=[], + tools_tree_package_directories=[Path("/abc")], tools_tree_release=None, tools_tree_repositories=["abc"], unified_kernel_image_format="myuki", From 9d9d98f65840a044324877a04304becf3e276a4f Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Sat, 16 Nov 2024 15:49:03 +0100 Subject: [PATCH 05/10] Don't require sbsigntools for secure boot auto enroll unless required If bootctl 257 or newer is installed, we don't use sbsigntools anymore so don't require it in that case. --- mkosi/__init__.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/mkosi/__init__.py b/mkosi/__init__.py index 928a973db..fec4cf2dc 100644 --- a/mkosi/__init__.py +++ b/mkosi/__init__.py @@ -2631,7 +2631,15 @@ def check_tools(config: Config, verb: Verb) -> None: reason="sign verity roothash signature with OpenSSL engine", ) - if want_efi(config) and config.secure_boot and config.secure_boot_auto_enroll: + if ( + want_efi(config) + and config.secure_boot + and config.secure_boot_auto_enroll + and ( + not config.find_binary("bootctl") + or systemd_tool_version("bootctl", sandbox=config.sandbox) < "257~devel" + ) + ): check_tool(config, "sbsiglist", reason="set up systemd-boot secure boot auto-enrollment") check_tool(config, "sbvarsign", reason="set up systemd-boot secure boot auto-enrollment") From 9a49c9d1e01d82489e1453fc45a9299bc3024fb9 Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Sat, 16 Nov 2024 15:49:45 +0100 Subject: [PATCH 06/10] Fix condition to also check the certificate source type --- mkosi/bootloader.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mkosi/bootloader.py b/mkosi/bootloader.py index a11e3aeb4..76dc6ecc2 100644 --- a/mkosi/bootloader.py +++ b/mkosi/bootloader.py @@ -500,7 +500,11 @@ def run_systemd_sign_tool( sandbox=config.sandbox( binary=cmd[0], options=opt, - devices=devices or key_source.type != KeySourceType.file, + devices=( + devices + or key_source.type != KeySourceType.file + or certificate_source.type != CertificateSourceType.file + ), ), ) From 44ec4299d1562c8c19b39d8fcb7e56b2caa3b1ee Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Sat, 16 Nov 2024 15:50:06 +0100 Subject: [PATCH 07/10] Only allow certificate files when not using systemd-sbsign --- mkosi/bootloader.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mkosi/bootloader.py b/mkosi/bootloader.py index 76dc6ecc2..f244c11db 100644 --- a/mkosi/bootloader.py +++ b/mkosi/bootloader.py @@ -595,6 +595,9 @@ def sign_efi_binary(context: Context, input: Path, output: Path) -> Path: or context.config.secure_boot_sign_tool == SecureBootSignTool.auto and context.config.find_binary("sbsign") is not None ): + if context.config.secure_boot_certificate_source.type != CertificateSourceType.file: + die("Secure boot certificate source must be 'file' when using sbsign as the signing tool") + cmd = [ "sbsign", "--cert", workdir(context.config.secure_boot_certificate), @@ -633,6 +636,9 @@ def sign_efi_binary(context: Context, input: Path, output: Path) -> Path: or context.config.secure_boot_sign_tool == SecureBootSignTool.auto and context.config.find_binary("pesign") is not None ): + if context.config.secure_boot_certificate_source.type != CertificateSourceType.file: + die("Secure boot certificate source must be 'file' when using pesign as the signing tool") + pesign_prepare(context) run( [ From 70a228fcf71fe3d8546506b0fd06f9d872f442d8 Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Sat, 16 Nov 2024 15:50:20 +0100 Subject: [PATCH 08/10] Install pkcs11-provider and opensc in tools tree Required for signing with openssl pkcs11 provider. --- mkosi/resources/man/mkosi.1.md | 2 ++ mkosi/resources/mkosi-tools/mkosi.conf | 1 + mkosi/resources/mkosi-tools/mkosi.conf.d/10-arch.conf | 1 + .../mkosi-tools/mkosi.conf.d/10-debian-kali-ubuntu/mkosi.conf | 1 + mkosi/resources/mkosi-tools/mkosi.conf.d/10-fedora/mkosi.conf | 1 + mkosi/resources/mkosi-tools/mkosi.conf.d/10-opensuse.conf | 1 + 6 files changed, 7 insertions(+) diff --git a/mkosi/resources/man/mkosi.1.md b/mkosi/resources/man/mkosi.1.md index 70506757d..7789cd77d 100644 --- a/mkosi/resources/man/mkosi.1.md +++ b/mkosi/resources/man/mkosi.1.md @@ -1282,8 +1282,10 @@ boolean argument: either `1`, `yes`, or `true` to enable, or `0`, `no`, | `less` | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | | `mtools` | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | | `nano` | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | + | `opensc` | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | | `openssh` | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | | `openssl` | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | + | `pkcs11-provider` | ✓ | | ✓ | ✓ | ✓ | ✓ | ✓ | | `sed` | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | | `pacman` | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | | | `pesign` | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | diff --git a/mkosi/resources/mkosi-tools/mkosi.conf b/mkosi/resources/mkosi-tools/mkosi.conf index 2b4d4be6e..ce34f7195 100644 --- a/mkosi/resources/mkosi-tools/mkosi.conf +++ b/mkosi/resources/mkosi-tools/mkosi.conf @@ -31,6 +31,7 @@ Packages= less mtools nano + opensc openssl sed socat diff --git a/mkosi/resources/mkosi-tools/mkosi.conf.d/10-arch.conf b/mkosi/resources/mkosi-tools/mkosi.conf.d/10-arch.conf index 73c2c7e7c..c31267a34 100644 --- a/mkosi/resources/mkosi-tools/mkosi.conf.d/10-arch.conf +++ b/mkosi/resources/mkosi-tools/mkosi.conf.d/10-arch.conf @@ -21,6 +21,7 @@ Packages= openssh pacman pesign + pkcs11-provider python-cryptography qemu-base reprepro diff --git a/mkosi/resources/mkosi-tools/mkosi.conf.d/10-debian-kali-ubuntu/mkosi.conf b/mkosi/resources/mkosi-tools/mkosi.conf.d/10-debian-kali-ubuntu/mkosi.conf index a02a4e954..ed5763436 100644 --- a/mkosi/resources/mkosi-tools/mkosi.conf.d/10-debian-kali-ubuntu/mkosi.conf +++ b/mkosi/resources/mkosi-tools/mkosi.conf.d/10-debian-kali-ubuntu/mkosi.conf @@ -32,6 +32,7 @@ Packages= ovmf pacman-package-manager pesign + pkcs11-provider policycoreutils python3-cryptography python3-pefile diff --git a/mkosi/resources/mkosi-tools/mkosi.conf.d/10-fedora/mkosi.conf b/mkosi/resources/mkosi-tools/mkosi.conf.d/10-fedora/mkosi.conf index 13137fae9..6f83642dd 100644 --- a/mkosi/resources/mkosi-tools/mkosi.conf.d/10-fedora/mkosi.conf +++ b/mkosi/resources/mkosi-tools/mkosi.conf.d/10-fedora/mkosi.conf @@ -15,6 +15,7 @@ Packages= erofs-utils fish pacman + pkcs11-provider qemu-system-aarch64-core qemu-system-ppc-core qemu-system-s390x-core diff --git a/mkosi/resources/mkosi-tools/mkosi.conf.d/10-opensuse.conf b/mkosi/resources/mkosi-tools/mkosi.conf.d/10-opensuse.conf index 26626680c..7187d0cd4 100644 --- a/mkosi/resources/mkosi-tools/mkosi.conf.d/10-opensuse.conf +++ b/mkosi/resources/mkosi-tools/mkosi.conf.d/10-opensuse.conf @@ -22,6 +22,7 @@ Packages= ovmf patterns-base-minimal_base pesign + pkcs11-provider policycoreutils python3-cryptography python3-pefile From 96f62ea1c94f5642310170885015f8cac9ee2e89 Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Sat, 16 Nov 2024 17:07:18 +0100 Subject: [PATCH 09/10] qemu: Use advanced form of specifying display --- mkosi/qemu.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/mkosi/qemu.py b/mkosi/qemu.py index 41a3fa8a1..058de72d4 100644 --- a/mkosi/qemu.py +++ b/mkosi/qemu.py @@ -1120,8 +1120,13 @@ def run_qemu(args: Args, config: Config) -> None: if config.architecture.is_arm_variant(): cmdline += ["-device", "virtio-gpu-pci"] else: - cmdline += ["-vga", "virtio"] - cmdline += ["-audio", "driver=pipewire,model=virtio"] + cmdline += ["-device", "virtio-vga"] + + cmdline += [ + "-nodefaults", + "-display", "sdl,gl=on", + "-audio", "driver=pipewire,model=virtio", + ] # fmt: skip else: # -nodefaults removes the default CDROM device which avoids an error message during boot # -serial mon:stdio adds back the serial device removed by -nodefaults. From aa56f90687a652d42085b1953c0e0bd9a55ba232 Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Sat, 16 Nov 2024 17:07:40 +0100 Subject: [PATCH 10/10] Add packages required for --qemu-gui to Arch tools tree --- mkosi/resources/mkosi-tools/mkosi.conf.d/10-arch.conf | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/mkosi/resources/mkosi-tools/mkosi.conf.d/10-arch.conf b/mkosi/resources/mkosi-tools/mkosi.conf.d/10-arch.conf index c31267a34..e51d4d74c 100644 --- a/mkosi/resources/mkosi-tools/mkosi.conf.d/10-arch.conf +++ b/mkosi/resources/mkosi-tools/mkosi.conf.d/10-arch.conf @@ -21,9 +21,18 @@ Packages= openssh pacman pesign + pipewire + pipewire-audio pkcs11-provider python-cryptography + qemu-audio-pipewire qemu-base + qemu-hw-display-virtio-gpu + qemu-hw-display-virtio-gpu-gl + qemu-hw-display-virtio-vga + qemu-hw-display-virtio-vga-gl + qemu-ui-opengl + qemu-ui-sdl reprepro sbsigntools shadow