Implement build overlay mounting with mkosi-sandbox

mkosi/__init__.py

@@ -298,7 +298,7 @@ def install_build_packages(context: Context) -> None:
 
     with (
         complete_step(f"Installing build packages for {context.config.distribution.pretty_name()}"),
-        mount_build_overlay(context),
+        setup_build_overlay(context),
     ):
         context.config.distribution.install_packages(context, context.config.build_packages)
 
@@ -474,24 +474,40 @@ def configure_autologin(context: Context) -> None:
 
 
 @contextlib.contextmanager
-def mount_build_overlay(context: Context, volatile: bool = False) -> Iterator[Path]:
+def setup_build_overlay(context: Context, volatile: bool = False) -> Iterator[None]:
     d = context.workspace / "build-overlay"
     if not d.is_symlink():
         with umask(~0o755):
             d.mkdir(exist_ok=True)
 
-    with contextlib.ExitStack() as stack:
-        lower = [context.root]
+    # We don't support multiple levels of root overlay.
+    assert not context.lowerdirs
+    assert not context.upperdir
+    assert not context.workdir
 
+    with contextlib.ExitStack() as stack:
         if volatile:
-            lower += [d]
-            upper = None
+            context.lowerdirs = [d]
+            context.upperdir = Path(
+                stack.enter_context(tempfile.TemporaryDirectory(prefix="volatile-overlay"))
+            )
+            os.chmod(context.upperdir, d.stat().st_mode)
         else:
-            upper = d
+            context.upperdir = d
 
-        stack.enter_context(mount_overlay(lower, context.root, upperdir=upper))
+        context.workdir = stack.enter_context(
+            tempfile.TemporaryDirectory(
+                dir=Path(context.upperdir).parent,
+                prefix=f"{Path(context.upperdir).name}-workdir",
+            )
+        )
 
-        yield context.root
+        try:
+            yield
+        finally:
+            context.lowerdirs = []
+            context.upperdir = None
+            context.workdir = None
 
 
 @contextlib.contextmanager
@@ -700,7 +716,7 @@ def script_maybe_chroot_sandbox(
                 network=network,
                 options=[
                     *options,
-                    "--bind", context.root, "/buildroot",
+                    *context.rootoptions(),
                     *context.config.distribution.package_manager(context.config).mounts(context),
                 ],
                 scripts=hd,
@@ -711,7 +727,7 @@ def script_maybe_chroot_sandbox(
                 options += ["--suppress-chown"]
 
             with chroot_cmd(
-                root=context.root,
+                root=context.rootoptions,
                 network=network,
                 options=options,
             ) as sandbox:
@@ -751,7 +767,7 @@ def run_prepare_scripts(context: Context, build: bool) -> None:
     env |= context.config.finalize_environment()
 
     with (
-        mount_build_overlay(context) if build else contextlib.nullcontext(),
+        setup_build_overlay(context) if build else contextlib.nullcontext(),
         finalize_source_mounts(
             context.config,
             ephemeral=bool(context.config.build_sources_ephemeral),
@@ -827,7 +843,7 @@ def run_build_scripts(context: Context) -> None:
     env |= context.config.finalize_environment()
 
     with (
-        mount_build_overlay(context, volatile=True),
+        setup_build_overlay(context, volatile=True),
         finalize_source_mounts(context.config, ephemeral=context.config.build_sources_ephemeral) as sources,
         finalize_config_json(context.config) as json,
     ):
@@ -1887,7 +1903,7 @@ def find_entry_token(context: Context) -> str:
     output = json.loads(
         run(
             ["kernel-install", "--root=/buildroot", "--json=pretty", "inspect"],
-            sandbox=context.sandbox(options=["--ro-bind", context.root, "/buildroot"]),
+            sandbox=context.sandbox(options=context.rootoptions(readonly=True)),
             stdout=subprocess.PIPE,
             env={"BOOT_ROOT": "/boot"},
         ).stdout
@@ -2972,7 +2988,7 @@ def run_depmod(context: Context, *, cache: bool = False) -> None:
                 continue
 
         with complete_step(f"Running depmod for {kver}"):
-            run(["depmod", "--all", kver], sandbox=chroot_cmd(root=context.root))
+            run(["depmod", "--all", kver], sandbox=chroot_cmd(root=context.rootoptions))
 
 
 def run_sysusers(context: Context) -> None:
@@ -2986,7 +3002,7 @@ def run_sysusers(context: Context) -> None:
     with complete_step("Generating system users"):
         run(
             ["systemd-sysusers", "--root=/buildroot"],
-            sandbox=context.sandbox(options=["--bind", context.root, "/buildroot"]),
+            sandbox=context.sandbox(options=context.rootoptions()),
         )
 
 
@@ -3018,7 +3034,7 @@ def run_tmpfiles(context: Context) -> None:
             success_exit_status=(0, 65, 73),
             sandbox=context.sandbox(
                 options=[
-                    "--bind", context.root, "/buildroot",
+                    *context.rootoptions(),
                     # systemd uses acl.h to parse ACLs in tmpfiles snippets which uses the host's
                     # passwd so we have to symlink the image's passwd to make ACL parsing work.
                     *finalize_passwd_symlinks("/buildroot"),
@@ -3042,11 +3058,11 @@ def run_preset(context: Context) -> None:
     with complete_step("Applying presets…"):
         run(
             ["systemctl", "--root=/buildroot", "preset-all"],
-            sandbox=context.sandbox(options=["--bind", context.root, "/buildroot"]),
+            sandbox=context.sandbox(options=context.rootoptions()),
         )
         run(
             ["systemctl", "--root=/buildroot", "--global", "preset-all"],
-            sandbox=context.sandbox(options=["--bind", context.root, "/buildroot"]),
+            sandbox=context.sandbox(options=context.rootoptions()),
         )
 
 
@@ -3061,7 +3077,7 @@ def run_hwdb(context: Context) -> None:
     with complete_step("Generating hardware database"):
         run(
             ["systemd-hwdb", "--root=/buildroot", "--usr", "--strict", "update"],
-            sandbox=context.sandbox(options=["--bind", context.root, "/buildroot"]),
+            sandbox=context.sandbox(options=context.rootoptions()),
         )
 
     # Remove any existing hwdb in /etc in favor of the one we just put in /usr.
@@ -3114,7 +3130,7 @@ def run_firstboot(context: Context) -> None:
     with complete_step("Applying first boot settings"):
         run(
             ["systemd-firstboot", "--root=/buildroot", "--force", *options],
-            sandbox=context.sandbox(options=["--bind", context.root, "/buildroot"]),
+            sandbox=context.sandbox(options=context.rootoptions()),
         )
 
         # Initrds generally don't ship with only /usr so there's not much point in putting the
@@ -3139,7 +3155,7 @@ def run_selinux_relabel(context: Context) -> None:
     with complete_step(f"Relabeling files using {policy} policy"):
         run(
             [setfiles, "-mFr", "/buildroot", "-T0", "-c", binpolicy, fc, "/buildroot"],
-            sandbox=context.sandbox(options=["--bind", context.root, "/buildroot"]),
+            sandbox=context.sandbox(options=context.rootoptions()),
             check=context.config.selinux_relabel == ConfigFeature.enabled,
         )
 
@@ -3290,7 +3306,6 @@ def make_image(
     split: bool = False,
     tabs: bool = False,
     verity: Verity = Verity.disabled,
-    root: Optional[Path] = None,
     definitions: Sequence[Path] = [],
     options: Sequence[PathString] = (),
 ) -> list[Partition]:
@@ -3301,6 +3316,7 @@ def make_image(
         "--dry-run=no",
         "--json=pretty",
         "--no-pager",
+        "--root=/buildroot",
         f"--offline={yes_no(context.config.repart_offline)}",
         "--seed", str(context.config.seed),
         workdir(context.staging / context.config.output_with_format),
@@ -3311,11 +3327,9 @@ def make_image(
         # that go into the disk image are owned by root.
         "--become-root",
         "--bind", context.staging, workdir(context.staging),
+        *context.rootoptions(),
     ]  # fmt: skip
 
-    if root:
-        cmdline += ["--root=/buildroot"]
-        opts += ["--bind", root, "/buildroot"]
     if not context.config.architecture.is_native():
         cmdline += ["--architecture", str(context.config.architecture)]
     if not (context.staging / context.config.output_with_format).exists():
@@ -3469,7 +3483,6 @@ def make_disk(
         split=split,
         tabs=tabs,
         verity=context.config.verity,
-        root=context.root,
         definitions=definitions,
     )
 
@@ -3631,7 +3644,6 @@ def make_esp(
     return make_image(
         context,
         msg="Generating ESP image",
-        root=context.root,
         definitions=[definitions],
     )
 
@@ -3665,7 +3677,7 @@ def make_extension_or_portable_image(context: Context, output: Path) -> None:
         # that go into the disk image are owned by root.
         "--become-root",
         "--bind", output.parent, workdir(output.parent),
-        "--ro-bind", context.root, "/buildroot",
+        *context.rootoptions(readonly=True),
         "--ro-bind", r, workdir(r),
     ]  # fmt: skip
 

mkosi/bootloader.py

@@ -673,7 +673,6 @@ def install_systemd_boot(context: Context) -> None:
         "--all-architectures",
         "--no-variables",
     ]
-    options: list[PathString] = ["--bind", context.root, "/buildroot"]
 
     bootctlver = systemd_tool_version("bootctl", sandbox=context.sandbox)
 
@@ -686,7 +685,7 @@ def install_systemd_boot(context: Context) -> None:
         run_systemd_sign_tool(
             context.config,
             cmdline=cmd,
-            options=options,
+            options=context.rootoptions(),
             certificate=context.config.secure_boot_certificate if want_bootctl_auto_enroll else None,
             certificate_source=context.config.secure_boot_certificate_source,
             key=context.config.secure_boot_key if want_bootctl_auto_enroll else None,
@@ -756,7 +755,7 @@ def install_systemd_boot(context: Context) -> None:
                         "--cert", workdir(context.config.secure_boot_certificate),
                         "--output", workdir(keys / f"{db}.auth"),
                     ]  # fmt: skip
-                    options = [
+                    options: list[PathString] = [
                         "--ro-bind",
                         context.config.secure_boot_certificate,
                         workdir(context.config.secure_boot_certificate),

mkosi/context.py

@@ -1,12 +1,13 @@
 # SPDX-License-Identifier: LGPL-2.1-or-later
 
+import os
 from collections.abc import Sequence
 from contextlib import AbstractContextManager
 from pathlib import Path
 from typing import Optional
 
 from mkosi.config import Args, Config
-from mkosi.util import PathString
+from mkosi.util import PathString, flatten
 
 
 class Context:
@@ -30,6 +31,9 @@ def __init__(
         self.keyring_dir = keyring_dir
         self.metadata_dir = metadata_dir
         self.package_dir = package_dir or (self.workspace / "packages")
+        self.lowerdirs: list[PathString] = []
+        self.upperdir: Optional[PathString] = None
+        self.workdir: Optional[PathString] = None
 
         self.package_dir.mkdir(exist_ok=True)
         self.staging.mkdir()
@@ -42,6 +46,22 @@ def __init__(
     def root(self) -> Path:
         return self.workspace / "root"
 
+    def rootoptions(self, dst: PathString = "/buildroot", *, readonly: bool = False) -> list[str]:
+        if self.lowerdirs or self.upperdir:
+            return [
+                "--overlay-lowerdir", os.fspath(self.root),
+                *flatten(["--overlay-lowerdir", os.fspath(lowerdir)] for lowerdir in self.lowerdirs),
+                *(
+                    ["--overlay-lowerdir" if readonly else "--overlay-upperdir", os.fspath(self.upperdir)]
+                    if self.upperdir
+                    else []
+                ),
+                *(["--overlay-workdir", os.fspath(self.workdir)] if self.workdir and not readonly else []),
+                "--overlay", os.fspath(dst),
+            ]  # fmt: skip
+        else:
+            return ["--ro-bind" if readonly else "--bind", os.fspath(self.root), os.fspath(dst)]
+
     @property
     def staging(self) -> Path:
         return self.workspace / "staging"

mkosi/distributions/debian.py

@@ -297,7 +297,7 @@ def fixup_os_release(context: Context) -> None:
                     f"/{candidate}.dpkg",
                     f"/{candidate}",
                 ],
-                sandbox=context.sandbox(options=["--bind", context.root, "/buildroot"]),
+                sandbox=context.sandbox(options=context.rootoptions()),
             )
 
         newosrelease.rename(osrelease)

mkosi/distributions/opensuse.py

@@ -116,7 +116,7 @@ def repositories(cls, context: Context) -> Iterable[RpmRepository]:
                     ],
                     sandbox=context.sandbox(
                         options=[
-                            "--bind", context.root, "/buildroot",
+                            *context.rootoptions(),
                             *finalize_certificate_mounts(context.config),
                         ],
                     ),

mkosi/installer/__init__.py

@@ -138,7 +138,7 @@ def sandbox(
         return context.sandbox(
             network=True,
             options=[
-                "--bind", context.root, "/buildroot",
+                *context.rootoptions(),
                 *cls.mounts(context),
                 *cls.options(root=context.root, apivfs=apivfs),
                 *options,

mkosi/kmod.py

@@ -101,9 +101,9 @@ def modinfo(context: Context, kver: str, modules: Iterable[str]) -> str:
 
     if context.config.output_format.is_extension_image() and not context.config.overlay:
         cmdline += ["--basedir", "/buildroot"]
-        sandbox = context.sandbox(options=["--ro-bind", context.root, "/buildroot"])
+        sandbox = context.sandbox(options=context.rootoptions(readonly=True))
     else:
-        sandbox = chroot_cmd(root=context.root)
+        sandbox = chroot_cmd(root=context.rootoptions)
 
     cmdline += [*modules]
 

mkosi/run.py

@@ -637,14 +637,14 @@ def chroot_options() -> list[PathString]:
 @contextlib.contextmanager
 def chroot_cmd(
     *,
-    root: Path,
+    root: Callable[[PathString], list[str]],
     network: bool = False,
     options: Sequence[PathString] = (),
 ) -> Iterator[list[PathString]]:
     with vartmpdir() as dir, resource_path(sys.modules[__package__ or __name__]) as module:
         cmdline: list[PathString] = [
             sys.executable, "-SI", module / "sandbox.py",
-            "--bind", root, "/",
+            *root("/"),
             # We mounted a subdirectory of TMPDIR to /var/tmp so we unset TMPDIR so that /tmp or /var/tmp are
             # used instead.
             "--unsetenv", "TMPDIR",