Merge pull request #3200 from DaanDeMeyer/passwd-symlink

Use passwd symlinks instead of bind mounts

mkosi/__init__.py

@@ -103,7 +103,7 @@
     chroot_cmd,
     chroot_options,
     finalize_interpreter,
-    finalize_passwd_mounts,
+    finalize_passwd_symlinks,
     fork_and_wait,
     run,
     workdir,
@@ -2631,7 +2631,15 @@ def check_tools(config: Config, verb: Verb) -> None:
                 reason="sign verity roothash signature with OpenSSL engine",
             )
 
-        if want_efi(config) and config.secure_boot and config.secure_boot_auto_enroll:
+        if (
+            want_efi(config)
+            and config.secure_boot
+            and config.secure_boot_auto_enroll
+            and (
+                not config.find_binary("bootctl")
+                or systemd_tool_version("bootctl", sandbox=config.sandbox) < "257~devel"
+            )
+        ):
             check_tool(config, "sbsiglist", reason="set up systemd-boot secure boot auto-enrollment")
             check_tool(config, "sbvarsign", reason="set up systemd-boot secure boot auto-enrollment")
 
@@ -2822,9 +2830,8 @@ def run_tmpfiles(context: Context) -> None:
                 options=[
                     "--bind", context.root, "/buildroot",
                     # systemd uses acl.h to parse ACLs in tmpfiles snippets which uses the host's
-                    # passwd so we have to mount the image's passwd over it to make ACL parsing
-                    # work.
-                    *finalize_passwd_mounts(context.root),
+                    # passwd so we have to symlink the image's passwd to make ACL parsing work.
+                    *finalize_passwd_symlinks("/buildroot"),
                     # Sometimes directories are configured to be owned by root in tmpfiles snippets
                     # so we want to make sure those chown()'s succeed by making ourselves the root
                     # user so that the root user exists.
@@ -4162,6 +4169,7 @@ def finalize_default_tools(args: Args, config: Config, *, resources: Path) -> Co
         *(["--package-cache-dir", str(config.package_cache_dir)] if config.package_cache_dir else []),
         "--incremental", str(config.incremental),
         *([f"--package={package}" for package in config.tools_tree_packages]),
+        *([f"--package-directory={directory}" for directory in config.tools_tree_package_directories]),
         "--output=tools",
         *(["--source-date-epoch", str(config.source_date_epoch)] if config.source_date_epoch is not None else []),  # noqa: E501
         *([f"--environment={k}='{v}'" for k, v in config.environment.items()]),

mkosi/archive.py

@@ -6,7 +6,7 @@
 from typing import Optional
 
 from mkosi.log import log_step
-from mkosi.run import SandboxProtocol, finalize_passwd_mounts, nosandbox, run, workdir
+from mkosi.run import SandboxProtocol, finalize_passwd_symlinks, nosandbox, run, workdir
 from mkosi.sandbox import umask
 from mkosi.types import PathString
 from mkosi.util import chdir
@@ -51,7 +51,7 @@ def make_tar(src: Path, dst: Path, *, sandbox: SandboxProtocol = nosandbox) -> N
             # Make sure tar uses user/group information from the root directory instead of the host.
             sandbox=sandbox(
                 binary="tar",
-                options=["--ro-bind", src, workdir(src), *finalize_passwd_mounts(src)],
+                options=["--ro-bind", src, workdir(src), *finalize_passwd_symlinks(workdir(src))],
             ),
         )  # fmt: skip
 
@@ -98,7 +98,7 @@ def extract_tar(
             options=[
                 "--ro-bind", src, workdir(src),
                 "--bind", dst, workdir(dst),
-                *finalize_passwd_mounts(dst),
+                *finalize_passwd_symlinks(workdir(dst)),
             ],
         ),
     )  # fmt: skip
@@ -136,6 +136,6 @@ def make_cpio(
             stdout=f,
             sandbox=sandbox(
                 binary="cpio",
-                options=["--ro-bind", src, workdir(src), *finalize_passwd_mounts(src)],
+                options=["--ro-bind", src, workdir(src), *finalize_passwd_symlinks(workdir(src))],
             ),
         )  # fmt: skip

mkosi/bootloader.py

@@ -500,7 +500,11 @@ def run_systemd_sign_tool(
         sandbox=config.sandbox(
             binary=cmd[0],
             options=opt,
-            devices=devices or key_source.type != KeySourceType.file,
+            devices=(
+                devices
+                or key_source.type != KeySourceType.file
+                or certificate_source.type != CertificateSourceType.file
+            ),
         ),
     )
 
@@ -591,6 +595,9 @@ def sign_efi_binary(context: Context, input: Path, output: Path) -> Path:
         or context.config.secure_boot_sign_tool == SecureBootSignTool.auto
         and context.config.find_binary("sbsign") is not None
     ):
+        if context.config.secure_boot_certificate_source.type != CertificateSourceType.file:
+            die("Secure boot certificate source must be 'file' when using sbsign as the signing tool")
+
         cmd = [
             "sbsign",
             "--cert", workdir(context.config.secure_boot_certificate),
@@ -629,6 +636,9 @@ def sign_efi_binary(context: Context, input: Path, output: Path) -> Path:
         or context.config.secure_boot_sign_tool == SecureBootSignTool.auto
         and context.config.find_binary("pesign") is not None
     ):
+        if context.config.secure_boot_certificate_source.type != CertificateSourceType.file:
+            die("Secure boot certificate source must be 'file' when using pesign as the signing tool")
+
         pesign_prepare(context)
         run(
             [

mkosi/config.py

@@ -1818,6 +1818,7 @@ class Config:
     tools_tree_repositories: list[str]
     tools_tree_sandbox_trees: list[ConfigTree]
     tools_tree_packages: list[str]
+    tools_tree_package_directories: list[Path]
     tools_tree_certificates: bool
     incremental: Incremental
     cacheonly: Cacheonly
@@ -3166,6 +3167,14 @@ def parse_ini(path: Path, only_sections: Collection[str] = ()) -> Iterator[tuple
         parse=config_make_list_parser(delimiter=","),
         help="Add additional packages to the default tools tree",
     ),
+    ConfigSetting(
+        dest="tools_tree_package_directories",
+        long="--tools-tree-package-directory",
+        metavar="PATH",
+        section="Build",
+        parse=config_make_list_parser(delimiter=",", parse=make_path_parser()),
+        help="Specify a directory containing extra tools tree packages",
+    ),
     ConfigSetting(
         dest="tools_tree_certificates",
         metavar="BOOL",
@@ -4813,6 +4822,7 @@ def summary(config: Config) -> str:
             Tools Tree Repositories: {line_join_list(config.tools_tree_repositories)}
            Tools Tree Sandbox Trees: {line_join_list(config.tools_tree_sandbox_trees)}
                 Tools Tree Packages: {line_join_list(config.tools_tree_packages)}
+     Tools Tree Package Directories: {line_join_list(config.tools_tree_package_directories)}
             Tools Tree Certificates: {yes_no(config.tools_tree_certificates)}
 
                         Incremental: {config.incremental}

mkosi/installer/__init__.py

@@ -7,7 +7,7 @@
 from mkosi.config import Config, ConfigFeature, OutputFormat
 from mkosi.context import Context
 from mkosi.mounts import finalize_crypto_mounts
-from mkosi.run import apivfs_options, finalize_interpreter, finalize_passwd_mounts, find_binary
+from mkosi.run import apivfs_options, finalize_interpreter, finalize_passwd_symlinks, find_binary
 from mkosi.tree import rmtree
 from mkosi.types import PathString
 from mkosi.util import flatten, startswith
@@ -109,10 +109,9 @@ def options(cls, *, root: PathString, apivfs: bool = True) -> list[PathString]:
             "--suppress-chown",
             # Make sure /etc/machine-id is not overwritten by any package manager post install scripts.
             "--ro-bind-try", Path(root) / "etc/machine-id", "/buildroot/etc/machine-id",
-            # If we're already in the sandbox, we want to pick up use the passwd files from /buildroot since
-            # the original root won't be available anymore. If we're not in the sandbox yet, we want to pick
-            # up the passwd files from the original root.
-            *finalize_passwd_mounts(root),
+            # Some package managers (e.g. dpkg) read from the host's /etc/passwd instead of the buildroot's
+            # /etc/passwd so we symlink /etc/passwd from the buildroot to make sure it gets used.
+            *(finalize_passwd_symlinks("/buildroot") if apivfs else []),
         ]  # fmt: skip
 
     @classmethod

mkosi/qemu.py

@@ -1120,8 +1120,13 @@ def run_qemu(args: Args, config: Config) -> None:
         if config.architecture.is_arm_variant():
             cmdline += ["-device", "virtio-gpu-pci"]
         else:
-            cmdline += ["-vga", "virtio"]
-        cmdline += ["-audio", "driver=pipewire,model=virtio"]
+            cmdline += ["-device", "virtio-vga"]
+
+        cmdline += [
+            "-nodefaults",
+            "-display", "sdl,gl=on",
+            "-audio", "driver=pipewire,model=virtio",
+        ]  # fmt: skip
     else:
         # -nodefaults removes the default CDROM device which avoids an error message during boot
         # -serial mon:stdio adds back the serial device removed by -nodefaults.

mkosi/resources/man/mkosi-sandbox.1.md

@@ -54,7 +54,9 @@ host system.
 :   Like `--bind-try`, but does a recursive readonly bind mount.
 
 `--symlink SRC DST`
-:   Creates a symlink at `DST` in the sandbox pointing to `SRC`.
+:   Creates a symlink at `DST` in the sandbox pointing to `SRC`. If `DST` already
+    exists and is a file or symlink, a temporary symlink is created and mounted on
+    top of `DST`.
 
 `--write DATA DST`
 :   Writes the string from `DATA` to `DST` in the sandbox.

mkosi/resources/man/mkosi.1.md

@@ -1282,8 +1282,10 @@ boolean argument: either `1`, `yes`, or `true` to enable, or `0`, `no`,
     | `less`                  | ✓      | ✓      | ✓      | ✓    | ✓      | ✓    | ✓        |
     | `mtools`                | ✓      | ✓      | ✓      | ✓    | ✓      | ✓    | ✓        |
     | `nano`                  | ✓      | ✓      | ✓      | ✓    | ✓      | ✓    | ✓        |
+    | `opensc`                | ✓      | ✓      | ✓      | ✓    | ✓      | ✓    | ✓        |
     | `openssh`               | ✓      | ✓      | ✓      | ✓    | ✓      | ✓    | ✓        |
     | `openssl`               | ✓      | ✓      | ✓      | ✓    | ✓      | ✓    | ✓        |
+    | `pkcs11-provider`       | ✓      |        | ✓      | ✓    | ✓      | ✓    | ✓        |
     | `sed`                   | ✓      | ✓      | ✓      | ✓    | ✓      | ✓    | ✓        |
     | `pacman`                | ✓      | ✓      | ✓      | ✓    | ✓      | ✓    |          |
     | `pesign`                | ✓      | ✓      | ✓      | ✓    | ✓      | ✓    | ✓        |
@@ -1331,6 +1333,9 @@ boolean argument: either `1`, `yes`, or `true` to enable, or `0`, `no`,
     separated list of package specifications. This option may be used
     multiple times in which case the specified package lists are combined.
 
+`ToolsTreePackageDirectories=`, `--tools-tree-package-directory=`
+:   Same as `PackageDirectories=`, but for the default tools tree.
+
 `ToolsTreeCertificates=`, `--tools-tree-certificates=`
 :   Specify whether to use certificates and keys from the tools tree.
     Enabled by default. If enabled, `/etc/pki`, `/etc/ssl`,

mkosi/resources/mkosi-tools/mkosi.conf

@@ -31,6 +31,7 @@ Packages=
         less
         mtools
         nano
+        opensc
         openssl
         sed
         socat

mkosi/resources/mkosi-tools/mkosi.conf.d/10-arch.conf

@@ -21,8 +21,18 @@ Packages=
         openssh
         pacman
         pesign
+        pipewire
+        pipewire-audio
+        pkcs11-provider
         python-cryptography
+        qemu-audio-pipewire
         qemu-base
+        qemu-hw-display-virtio-gpu
+        qemu-hw-display-virtio-gpu-gl
+        qemu-hw-display-virtio-vga
+        qemu-hw-display-virtio-vga-gl
+        qemu-ui-opengl
+        qemu-ui-sdl
         reprepro
         sbsigntools
         shadow

mkosi/resources/mkosi-tools/mkosi.conf.d/10-debian-kali-ubuntu/mkosi.conf

@@ -32,6 +32,7 @@ Packages=
         ovmf
         pacman-package-manager
         pesign
+        pkcs11-provider
         policycoreutils
         python3-cryptography
         python3-pefile

mkosi/resources/mkosi-tools/mkosi.conf.d/10-fedora/mkosi.conf

@@ -15,6 +15,7 @@ Packages=
         erofs-utils
         fish
         pacman
+        pkcs11-provider
         qemu-system-aarch64-core
         qemu-system-ppc-core
         qemu-system-s390x-core

mkosi/resources/mkosi-tools/mkosi.conf.d/10-opensuse.conf

@@ -22,6 +22,7 @@ Packages=
         ovmf
         patterns-base-minimal_base
         pesign
+        pkcs11-provider
         policycoreutils
         python3-cryptography
         python3-pefile

mkosi/run.py

@@ -415,15 +415,14 @@ def workdir(path: Path, sandbox: Optional[SandboxProtocol] = None) -> str:
     return joinpath(subdir, str(path))
 
 
-def finalize_passwd_mounts(root: PathString) -> list[PathString]:
+def finalize_passwd_symlinks(root: PathString) -> list[PathString]:
     """
     If passwd or a related file exists in the apivfs directory, bind mount it over the host files while we
     run the command, to make sure that the command we run uses user/group information from the apivfs
     directory instead of from the host.
     """
     return flatten(
-        ("--ro-bind-try", Path(root) / "etc" / f, f"/etc/{f}")
-        for f in ("passwd", "group", "shadow", "gshadow")
+        ("--symlink", Path(root) / "etc" / f, f"/etc/{f}") for f in ("passwd", "group", "shadow", "gshadow")
     )
 
 

mkosi/sandbox.py

@@ -560,12 +560,19 @@ def __init__(self, src: str, dst: str) -> None:
     def execute(self, oldroot: str, newroot: str) -> None:
         dst = joinpath(newroot, self.dst)
         try:
-            os.symlink(self.src, dst)
+            return os.symlink(self.src, dst)
         except FileExistsError:
-            if os.readlink(dst) == self.src:
+            if os.path.islink(dst) and os.readlink(dst) == self.src:
                 return
 
-            raise
+            if os.path.isdir(dst):
+                raise
+
+        # If the target already exists and is not a directory, create the symlink somewhere else and mount
+        # it over the existing file or symlink.
+        os.symlink(self.src, "/symlink")
+        mount_rbind("/symlink", dst)
+        os.unlink("/symlink")
 
 
 class WriteOperation(FSOperation):

tests/test_json.py

@@ -363,6 +363,9 @@ def test_config() -> None:
             "ToolsTreeCertificates": true,
             "ToolsTreeDistribution": "fedora",
             "ToolsTreeMirror": null,
+            "ToolsTreePackageDirectories": [
+                "/abc"
+            ],
             "ToolsTreePackages": [],
             "ToolsTreeRelease": null,
             "ToolsTreeRepositories": [
@@ -570,6 +573,7 @@ def test_config() -> None:
         tools_tree_mirror=None,
         tools_tree_sandbox_trees=[ConfigTree(Path("/a/b/c"), Path("/"))],
         tools_tree_packages=[],
+        tools_tree_package_directories=[Path("/abc")],
         tools_tree_release=None,
         tools_tree_repositories=["abc"],
         unified_kernel_image_format="myuki",