mkosi: 24.3-unstable-2024-08-28 -> 25.3 (#376421)

From eb36791f873dd645b1cbfa693b9c246943647190 Mon Sep 17 00:00:00 2001

From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001

From: Moritz Sanft <58110325+msanft@users.noreply.github.com>

Date: Tue, 3 Sep 2024 08:57:26 +0200

Subject: [PATCH 1/3] Use wrapped binaries instead of Python interpreter

Subject: [PATCH] Use wrapped binaries instead of Python interpreter

Rather than calling ukify and mkosi with sys.executable, which doesn't use the Python wrappers for PATH and PYTHONPATH, we call the wrapped binaries directly.

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

---

 mkosi/__init__.py | 19 ++++---------------

 mkosi/__init__.py   | 10 +++++-----

 mkosi/bootloader.py |  5 +----

 mkosi/run.py        |  8 ++++----

 2 files changed, 8 insertions(+), 19 deletions(-)

 3 files changed, 10 insertions(+), 13 deletions(-)

diff --git a/mkosi/__init__.py b/mkosi/__init__.py

index cc8482c4..ba44ad31 100644

index c5f785d4f18564378021352fde76be62078324ed..580a5c6464397be5a1977b794d7c044724d43490 100644

--- a/mkosi/__init__.py

+++ b/mkosi/__init__.py

@@ -2059,16 +2059,7 @@ def join_initrds(initrds: Sequence[Path], output: Path) -> Path:

@@ -658,7 +658,7 @@ def script_maybe_chroot_sandbox(

     helpers = {

         "mkosi-chroot": [

-            finalize_interpreter(bool(context.config.tools_tree)), "-SI", "/sandbox.py",

+            @MKOSI_SANDBOX@,

             "--bind", "/buildroot", "/",

             "--bind", "/var/tmp", "/var/tmp",

             *apivfs_options(root=Path("/")),

@@ -1534,7 +1534,7 @@ def run_ukify(

     options: Sequence[PathString] = (),

     sign: bool = True,

 ) -> None:

-    ukify = context.config.find_binary("ukify", "/usr/lib/systemd/ukify")

+    ukify = context.config.find_binary("ukify", "@UKIFY@")

     if not ukify:

         die("Could not find ukify")

 def python_binary(config: Config, *, binary: Optional[PathString]) -> PathString:

-    tools = (

-        not binary or

-        not (path := config.find_binary(binary)) or

-        not any(path.is_relative_to(d) for d in config.extra_search_paths)

-    )

-

-    # If there's no tools tree, prefer the interpreter from MKOSI_INTERPRETER. If there is a tools

-    # tree, just use the default python3 interpreter.

-    exe = Path(sys.executable)

-    return "python3" if (tools and config.tools_tree) or not exe.is_relative_to("/usr") else exe

+    return "@PYTHON_PEFILE@"

 def extract_pe_section(context: Context, binary: Path, section: str, output: Path) -> Path:

@@ -2135,11 +2126,10 @@ def build_uki(

     if not (arch := context.config.architecture.to_efi()):

         die(f"Architecture {context.config.architecture} does not support UEFI")

@@ -1629,7 +1629,7 @@ def build_uki(

     profiles: Sequence[Path],

     output: Path,

 ) -> None:

-    if not (ukify := context.config.find_binary("ukify", "/usr/lib/systemd/ukify")):

+    if not (ukify := context.config.find_binary("ukify", "@UKIFY@")):

         die("Could not find ukify")

     cmd: list[PathString] = [

-        python_binary(context.config, binary=ukify),

         ukify,

         *(["--cmdline", f"@{context.workspace / 'cmdline'}"] if cmdline else []),

         "--os-release", f"@{context.root / 'usr/lib/os-release'}",

@@ -2213,7 +2203,6 @@ def build_uki(

         # new .ucode section support?

         if (

             systemd_tool_version(

-                python_binary(context.config, binary=ukify),

                 ukify,

                 sandbox=context.sandbox,

             ) >= "256" and

@@ -2303,7 +2292,7 @@ def want_uki(context: Context) -> bool:

             context.config.unified_kernel_images == ConfigFeature.enabled or (

                 context.config.unified_kernel_images == ConfigFeature.auto and

                 systemd_stub_binary(context).exists() and

-                context.config.find_binary("ukify", "/usr/lib/systemd/ukify") is not None

+                context.config.find_binary("ukify", "@UKIFY@") is not None

     arguments: list[PathString] = [

@@ -1764,7 +1764,7 @@ def want_uki(context: Context) -> bool:

         or (

             context.config.unified_kernel_images == ConfigFeature.auto

             and systemd_stub_binary(context).exists()

-            and context.config.find_binary("ukify", "/usr/lib/systemd/ukify") is not None

+            and context.config.find_binary("ukify", "@UKIFY@") is not None

         )

     )

@@ -2914,7 +2903,7 @@ def check_ukify(

@@ -2641,7 +2641,7 @@ def check_ukify(

     reason: str,

     hint: Optional[str] = None,

 ) -> None:

-    ukify = check_tool(config, "ukify", "/usr/lib/systemd/ukify", reason=reason, hint=hint)

+    ukify = check_tool(config, "ukify", "@UKIFY@", reason=reason, hint=hint)

     v = systemd_tool_version(python_binary(config, binary=ukify), ukify, sandbox=config.sandbox)

     v = systemd_tool_version(python_binary(config), ukify, sandbox=config.sandbox)

     if v < version:

diff --git a/mkosi/bootloader.py b/mkosi/bootloader.py

index 87a56cc03cc42df41babfe15195e84ab956a3a21..060d3816a5cc287ea61df55fd0696e49b621908f 100644

--- a/mkosi/bootloader.py

+++ b/mkosi/bootloader.py

@@ -264,10 +264,7 @@ def find_signed_grub_image(context: Context) -> Optional[Path]:

 def python_binary(config: Config) -> PathString:

-    # If there's no tools tree, prefer the interpreter from MKOSI_INTERPRETER. If there is a tools

-    # tree, just use the default python3 interpreter.

-    exe = Path(sys.executable)

-    return "python3" if config.tools_tree or not exe.is_relative_to("/usr") else exe

+    return "@PYTHON_PEFILE@"

 def extract_pe_section(context: Context, binary: Path, section: str, output: Path) -> Path:

diff --git a/mkosi/run.py b/mkosi/run.py

index fd3bc98e..de47349a 100644

index a0e2b9cf00b5e9e9d14b570b052f1b2807b46407..fe1a1cdee6f9a586a85a6894d7a97fa5688b29f7 100644

--- a/mkosi/run.py

+++ b/mkosi/run.py

@@ -450,7 +450,7 @@ def sandbox_cmd(

 ) -> Iterator[list[PathString]]:

@@ -378,7 +378,7 @@ def finalize_path(

         # Make sure that /usr/bin and /usr/sbin are always in $PATH.

         path += [s for s in ("/usr/bin", "/usr/sbin") if s not in path]

     else:

-        path += ["/usr/bin", "/usr/sbin"]

+        path += ["/usr/bin", "/usr/sbin", "@NIX_PATH@"]

     if prefix_usr:

         path = [os.fspath(root / s.lstrip("/")) if s in ("/usr/bin", "/usr/sbin") else s for s in path]

@@ -564,7 +564,7 @@ def sandbox_cmd(

         cmdline: list[PathString] = [

             *setup,

-        sys.executable, "-SI", mkosi.sandbox.__file__,

             *(["strace", "--detach-on=execve"] if ARG_DEBUG_SANDBOX.get() else []),

-            sys.executable, "-SI", module / "sandbox.py",

+            @MKOSI_SANDBOX@,

             "--proc", "/proc",

         # We mounted a subdirectory of TMPDIR to /var/tmp so we unset TMPDIR so that /tmp or /var/tmp are used instead.

         "--unsetenv", "TMPDIR",

@@ -563,7 +563,7 @@ def apivfs_options(*, root: Path = Path("/buildroot")) -> list[PathString]:

 def apivfs_script_cmd(*, tools: bool, options: Sequence[PathString] = ()) -> list[PathString]:

     exe = Path(sys.executable)

     return [

-        "python3" if tools or not exe.is_relative_to("/usr") else exe, "-SI", "/sandbox.py",

+        @MKOSI_SANDBOX@,

         "--bind", "/", "/",

         "--same-dir",

         "--bind", "/var/tmp", "/buildroot/var/tmp",

@@ -597,7 +597,7 @@ def chroot_cmd(

     options: Sequence[PathString] = (),

             # We mounted a subdirectory of TMPDIR to /var/tmp so we unset TMPDIR so that /tmp or /var/tmp are

             # used instead.

@@ -730,7 +730,7 @@ def chroot_options() -> list[PathString]:

         "--unshare-ipc",

         "--setenv", "container", "mkosi",

         "--setenv", "HOME", "/",

-        "--setenv", "PATH", "/usr/bin:/usr/sbin",

+        "--setenv", "PATH", "/usr/bin:/usr/sbin:@NIX_PATH@",

         "--setenv", "BUILDROOT", "/",

     ]  # fmt: skip

@@ -744,7 +744,7 @@ def chroot_cmd(

 ) -> Iterator[list[PathString]]:

     with vartmpdir() as dir, resource_path(sys.modules[__package__ or __name__]) as module:

         cmdline: list[PathString] = [

-        sys.executable, "-SI", mkosi.sandbox.__file__,

-            sys.executable, "-SI", module / "sandbox.py",

+            @MKOSI_SANDBOX@,

             "--bind", root, "/",

         # We mounted a subdirectory of TMPDIR to /var/tmp so we unset TMPDIR so that /tmp or /var/tmp are used instead.

         "--unsetenv", "TMPDIR",

@@ -619,7 +619,7 @@ def chroot_cmd(

 def chroot_script_cmd(*, tools: bool, network: bool = False, work: bool = False) -> list[PathString]:

     exe = Path(sys.executable)

     return [

-        "python3" if tools or not exe.is_relative_to("/usr") else exe, "-SI", "/sandbox.py",

+        @MKOSI_SANDBOX@,

         "--bind", "/buildroot", "/",

         "--bind", "/var/tmp", "/var/tmp",

         *apivfs_options(root=Path("/")),

--

2.45.2

             # We mounted a subdirectory of TMPDIR to /var/tmp so we unset TMPDIR so that /tmp or /var/tmp are

             # used instead.

From a1e6ccfeaf8ef10361280b9ecad958e9d556005b Mon Sep 17 00:00:00 2001

From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001

From: Moritz Sanft <58110325+msanft@users.noreply.github.com>

Date: Tue, 3 Sep 2024 09:00:34 +0200

Subject: [PATCH 2/3] Fix library resolving

Subject: [PATCH] Fix library resolving

As ctypes doesn't do lookups in the Nix store for libraries, we supply the exact paths.

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

---

 mkosi/sandbox/__init__.py | 4 ++--

 mkosi/sandbox.py | 4 ++--

 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/mkosi/sandbox/__init__.py b/mkosi/sandbox/__init__.py

index 7db340c5..3d0a0e56 100644

--- a/mkosi/sandbox/__init__.py

+++ b/mkosi/sandbox/__init__.py

@@ -78,7 +78,7 @@ class cap_user_data_t(ctypes.Structure):

diff --git a/mkosi/sandbox.py b/mkosi/sandbox.py

index bfcdb8bc62090235ebbe797982493194b575a5c0..4efee355a5f1ae036ccc5ba39595129a32705ee7 100755

--- a/mkosi/sandbox.py

+++ b/mkosi/sandbox.py

@@ -86,7 +86,7 @@ class cap_user_data_t(ctypes.Structure):

     ]

 libc.syscall.restype = ctypes.c_long

 libc.unshare.argtypes = (ctypes.c_int,)

@@ -175,7 +175,7 @@ def seccomp_suppress_chown() -> None:

     Unfortunately, non-root users can only create files owned by their own uid. To still allow non-root users to build

     images, if requested we install a seccomp filter that makes calls to chown() and friends a noop.

@@ -198,7 +198,7 @@ def seccomp_suppress_chown() -> None:

     owned by their own uid. To still allow non-root users to build images, if requested we install

     a seccomp filter that makes calls to chown() and friends a noop.

     """

-    libseccomp = ctypes.CDLL("libseccomp.so.2")

+    libseccomp = ctypes.CDLL("@LIBSECCOMP@")

     if libseccomp is None:

         raise FileNotFoundError("libseccomp.so.2")

--

2.45.2

From e834d51aa2542b141ceafdd42285ded6a9997c90 Mon Sep 17 00:00:00 2001

From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001

From: Moritz Sanft <58110325+msanft@users.noreply.github.com>

Date: Tue, 3 Sep 2024 09:09:19 +0200

Subject: [PATCH 3/3] Fix QEMU firmware path

Subject: [PATCH] Fix QEMU firmware path

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

---

 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mkosi/qemu.py b/mkosi/qemu.py

index b98bec65..886598aa 100644

index 7be900f4e99d0db94fec6dfb2534ed23a6edcf80..e2c741d720bca5bf878ca229119471cacb72d3e1 100644

--- a/mkosi/qemu.py

+++ b/mkosi/qemu.py

@@ -182,7 +182,7 @@ def find_ovmf_firmware(config: Config, qemu: Path, firmware: QemuFirmware) -> Op

@@ -199,7 +199,7 @@ def find_ovmf_firmware(config: Config, firmware: Firmware) -> Optional[OvmfConfi

     if not firmware.is_uefi():

         return None

     tools = Path("/") if any(qemu.is_relative_to(d) for d in config.extra_search_paths) else config.tools()

-    desc = list((config.tools() / "usr/share/qemu/firmware").glob("*"))

+    desc = list((config.tools() / "@QEMU_FIRMWARE@").glob("*"))

     if config.tools() == Path("/"):

         desc += list((config.tools() / "etc/qemu/firmware").glob("*"))

-    desc = list((tools / "usr/share/qemu/firmware").glob("*"))

+    desc = list((tools / "@QEMU_FIRMWARE@").glob("*"))

     if tools == Path("/"):

         desc += list((tools / "etc/qemu/firmware").glob("*"))

--

2.45.2

  # Optional dependencies

  withQemu ? false,

  qemu,

  # Workaround for supporting providing additional package manager

  # dependencies in the recursive use in the binary path.

  # This can / should be removed once the `finalAttrs` pattern is

  # available for Python packages.

  extraDeps ? [ ],

}:

let

  # For systemd features used by mkosi, see

    withKernelInstall = true;

  };

  python3pefile = python3.withPackages (

    ps: with ps; [

      pefile

  python3pefile = python3.withPackages (_: [ pefile ]);

  deps =

    [

      bash

      btrfs-progs

      coreutils

      cpio

      gnutar

      kmod

      systemdForMkosi

      util-linux

    ]

  );

    ++ extraDeps

    ++ lib.optionals withQemu [

      qemu

    ];

in

buildPythonApplication rec {

  pname = "mkosi";

  version = "24.3-unstable-2024-08-28";

  version = "25.3";

  format = "pyproject";

  outputs = [

  src = fetchFromGitHub {

    owner = "systemd";

    repo = "mkosi";

    rev = "8c2f828701a1bdb3dc9b80d6f2ab979f0430a6b8";

    hash = "sha256-rO/4ki2nAJQN2slmYuHKESGBBDMXC/ikGf6dMDcKFr4=";

    tag = "v${version}";

    hash = "sha256-CTOVFZORLrVqehhPCgOoEaU3fhwu8fO8jGDNLxoELgE=";

  };

  patches =

      (replaceVars ./0001-Use-wrapped-binaries-instead-of-Python-interpreter.patch {

        UKIFY = "${systemdForMkosi}/lib/systemd/ukify";

        PYTHON_PEFILE = "${python3pefile}/bin/python3.12";

        NIX_PATH = toString (lib.makeBinPath deps);

        MKOSI_SANDBOX = null; # will be replaced in postPatch

      })

      (replaceVars ./0002-Fix-library-resolving.patch {

  postPatch = ''

    # As we need the $out reference, we can't use `replaceVars` here.

    substituteInPlace mkosi/run.py \

    substituteInPlace mkosi/{run,__init__}.py \

      --replace-fail '@MKOSI_SANDBOX@' "\"$out/bin/mkosi-sandbox\""

  '';

    wheel

  ];

  propagatedBuildInputs =

    [

      bash

      btrfs-progs

      coreutils

      cpio

      gnutar

      kmod

      systemdForMkosi

      util-linux

    ]

    ++ lib.optional withQemu [

      qemu

    ];

  dependencies = deps;

  postBuild = ''

    ./tools/make-man-page.sh

  postInstall = ''

    mkdir -p $out/share/man/man1

    mv mkosi/resources/mkosi.1 $out/share/man/man1/

    mv mkosi/resources/man/mkosi.1 $out/share/man/man1/

  '';

  meta = with lib; {

      msanft

    ];

    platforms = platforms.linux;

    # `mkosi qemu` boot fails in the uefi shell, image isn't found.

    broken = withQemu;

  };

}