Unverified Commit 7d053c81 authored by Ryan Lahfa's avatar Ryan Lahfa Committed by GitHub
Browse files

Merge pull request #245250 from nikstur/images

parents b5f48a3b cb2d047c
Loading
Loading
Loading
Loading
+17 −0
Original line number Diff line number Diff line
@@ -443,4 +443,21 @@ in rec {
          ${attrsToSection def.sliceConfig}
        '';
    };

  # Create a directory that contains systemd definition files from an attrset
  # that contains the file names as keys and the content as values. The values
  # in that attrset are determined by the supplied format.
  definitions = directoryName: format: definitionAttrs:
    let
      listOfDefinitions = lib.mapAttrsToList
        (name: format.generate "${name}.conf")
        definitionAttrs;
    in
    pkgs.runCommand directoryName { } ''
      mkdir -p $out
      ${(lib.concatStringsSep "\n"
        (map (pkg: "cp ${pkg} $out/${pkg.name}") listOfDefinitions)
      )}
    '';

}
+113 −0
Original line number Diff line number Diff line
#!/usr/bin/env python

"""Amend systemd-repart definiton files.

In order to avoid Import-From-Derivation (IFD) when building images with
systemd-repart, the definition files created by Nix need to be amended with the
store paths from the closure.

This is achieved by adding CopyFiles= instructions to the definition files.

The arbitrary files configured via `contents` are also added to the definition
files using the same mechanism.
"""

import json
import sys
import shutil
import os
import tempfile
from pathlib import Path


def add_contents_to_definition(
    definition: Path, contents: dict[str, dict[str, str]] | None
) -> None:
    """Add CopyFiles= instructions to a definition for all files in contents."""
    if not contents:
        return

    copy_files_lines: list[str] = []
    for target, options in contents.items():
        source = options["source"]

        copy_files_lines.append(f"CopyFiles={source}:{target}\n")

    with open(definition, "a") as f:
        f.writelines(copy_files_lines)


def add_closure_to_definition(
    definition: Path, closure: Path | None, strip_nix_store_prefix: bool | None
) -> None:
    """Add CopyFiles= instructions to a definition for all paths in the closure.

    If strip_nix_store_prefix is True, `/nix/store` is stripped from the target path.
    """
    if not closure:
        return

    copy_files_lines: list[str] = []
    with open(closure, "r") as f:
        for line in f:
            if not isinstance(line, str):
                continue

            source = Path(line.strip())
            target = str(source.relative_to("/nix/store/"))
            target = f":{target}" if strip_nix_store_prefix else ""

            copy_files_lines.append(f"CopyFiles={source}{target}\n")

    with open(definition, "a") as f:
        f.writelines(copy_files_lines)


def main() -> None:
    """Amend the provided repart definitions by adding CopyFiles= instructions.

    For each file specified in the `contents` field of a partition in the
    partiton config file, a `CopyFiles=` instruction is added to the
    corresponding definition file.

    The same is done for every store path of the `closure` field.

    Print the path to a directory that contains the amended repart
    definitions to stdout.
    """
    partition_config_file = sys.argv[1]
    if not partition_config_file:
        print("No partition config file was supplied.")
        sys.exit(1)

    repart_definitions = sys.argv[2]
    if not repart_definitions:
        print("No repart definitions were supplied.")
        sys.exit(1)

    with open(partition_config_file, "rb") as f:
        partition_config = json.load(f)

    if not partition_config:
        print("Partition config is empty.")
        sys.exit(1)

    temp = tempfile.mkdtemp()
    shutil.copytree(repart_definitions, temp, dirs_exist_ok=True)

    for name, config in partition_config.items():
        definition = Path(f"{temp}/{name}.conf")
        os.chmod(definition, 0o644)

        contents = config.get("contents")
        add_contents_to_definition(definition, contents)

        closure = config.get("closure")
        strip_nix_store_prefix = config.get("stripStorePaths")
        add_closure_to_definition(definition, closure, strip_nix_store_prefix)

    print(temp)


if __name__ == "__main__":
    main()
+137 −0
Original line number Diff line number Diff line
# Building Images via `systemd-repart` {#sec-image-repart}

You can build disk images in NixOS with the `image.repart` option provided by
the module [image/repart.nix][]. This module uses `systemd-repart` to build the
images and exposes it's entire interface via the `repartConfig` option.

[image/repart.nix]: https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/image/repart.nix

An example of how to build an image:

```nix
{ config, modulesPath, ... }: {

  imports = [ "${modulesPath}/image/repart.nix" ];

  image.repart = {
    name = "image";
    partitions = {
      "esp" = {
        contents = {
          ...
        };
        repartConfig = {
          Type = "esp";
          ...
        };
      };
      "root" = {
        storePaths = [ config.system.build.toplevel ];
        repartConfig = {
          Type = "root";
          Label = "nixos";
          ...
        };
      };
    };
  };

}
```

## Nix Store Partition {#sec-image-repart-store-partition}

You can define a partition that only contains the Nix store and then mount it
under `/nix/store`. Because the `/nix/store` part of the paths is already
determined by the mount point, you have to set `stripNixStorePrefix = true;` so
that the prefix is stripped from the paths before copying them into the image.

```nix
fileSystems."/nix/store".device = "/dev/disk/by-partlabel/nix-store"

image.repart.partitions = {
  "store" = {
    storePaths = [ config.system.build.toplevel ];
    stripNixStorePrefix = true;
    repartConfig = {
      Type = "linux-generic";
      Label = "nix-store";
      ...
    };
  };
};
```

## Appliance Image {#sec-image-repart-appliance}

The `image/repart.nix` module can also be used to build self-contained [software
appliances][].

[software appliances]: https://en.wikipedia.org/wiki/Software_appliance

The generation based update mechanism of NixOS is not suited for appliances.
Updates of appliances are usually either performed by replacing the entire
image with a new one or by updating partitions via an A/B scheme. See the
[Chrome OS update process][chrome-os-update] for an example of how to achieve
this. The appliance image built in the following example does not contain a
`configuration.nix` and thus you will not be able to call `nixos-rebuild` from
this system.

[chrome-os-update]: https://chromium.googlesource.com/aosp/platform/system/update_engine/+/HEAD/README.md

```nix
let
  pkgs = import <nixpkgs> { };
  efiArch = pkgs.stdenv.hostPlatform.efiArch;
in
(pkgs.nixos [
  ({ config, lib, pkgs, modulesPath, ... }: {

    imports = [ "${modulesPath}/image/repart.nix" ];

    boot.loader.grub.enable = false;

    fileSystems."/".device = "/dev/disk/by-label/nixos";

    image.repart = {
      name = "image";
      partitions = {
        "esp" = {
          contents = {
            "/EFI/BOOT/BOOT${lib.toUpper efiArch}.EFI".source =
              "${pkgs.systemd}/lib/systemd/boot/efi/systemd-boot${efiArch}.efi";

            "/loader/entries/nixos.conf".source = pkgs.writeText "nixos.conf" ''
              title NixOS
              linux /EFI/nixos/kernel.efi
              initrd /EFI/nixos/initrd.efi
              options init=${config.system.build.toplevel}/init ${toString config.boot.kernelParams}
            '';

            "/EFI/nixos/kernel.efi".source =
              "${config.boot.kernelPackages.kernel}/${config.system.boot.loader.kernelFile}";

            "/EFI/nixos/initrd.efi".source =
              "${config.system.build.initialRamdisk}/${config.system.boot.loader.initrdFile}";
          };
          repartConfig = {
            Type = "esp";
            Format = "vfat";
            SizeMinBytes = "96M";
          };
        };
        "root" = {
          storePaths = [ config.system.build.toplevel ];
          repartConfig = {
            Type = "root";
            Format = "ext4";
            Label = "nixos";
            Minimize = "guess";
          };
        };
      };
    };

  })
]).image
```
+207 −0
Original line number Diff line number Diff line
# This module exposes options to build a disk image with a GUID Partition Table
# (GPT). It uses systemd-repart to build the image.

{ config, pkgs, lib, utils, ... }:

let
  cfg = config.image.repart;

  partitionOptions = {
    options = {
      storePaths = lib.mkOption {
        type = with lib.types; listOf path;
        default = [ ];
        description = lib.mdDoc "The store paths to include in the partition.";
      };

      stripNixStorePrefix = lib.mkOption {
        type = lib.types.bool;
        default = false;
        description = lib.mdDoc ''
          Whether to strip `/nix/store/` from the store paths. This is useful
          when you want to build a partition that only contains store paths and
          is mounted under `/nix/store`.
        '';
      };

      contents = lib.mkOption {
        type = with lib.types; attrsOf (submodule {
          options = {
            source = lib.mkOption {
              type = types.path;
              description = lib.mdDoc "Path of the source file.";
            };
          };
        });
        default = { };
        example = lib.literalExpression '' {
          "/EFI/BOOT/BOOTX64.EFI".source =
            "''${pkgs.systemd}/lib/systemd/boot/efi/systemd-bootx64.efi";

          "/loader/entries/nixos.conf".source = systemdBootEntry;
        }
        '';
        description = lib.mdDoc "The contents to end up in the filesystem image.";
      };

      repartConfig = lib.mkOption {
        type = with lib.types; attrsOf (oneOf [ str int bool ]);
        example = {
          Type = "home";
          SizeMinBytes = "512M";
          SizeMaxBytes = "2G";
        };
        description = lib.mdDoc ''
          Specify the repart options for a partiton as a structural setting.
          See <https://www.freedesktop.org/software/systemd/man/repart.d.html>
          for all available options.
        '';
      };
    };
  };
in
{
  options.image.repart = {

    name = lib.mkOption {
      type = lib.types.str;
      description = lib.mdDoc "The name of the image.";
    };

    seed = lib.mkOption {
      type = with lib.types; nullOr str;
      # Generated with `uuidgen`. Random but fixed to improve reproducibility.
      default = "0867da16-f251-457d-a9e8-c31f9a3c220b";
      description = lib.mdDoc ''
        A UUID to use as a seed. You can set this to `null` to explicitly
        randomize the partition UUIDs.
      '';
    };

    split = lib.mkOption {
      type = lib.types.bool;
      default = false;
      description = lib.mdDoc ''
        Enables generation of split artifacts from partitions. If enabled, for
        each partition with SplitName= set, a separate output file containing
        just the contents of that partition is generated.
      '';
    };

    partitions = lib.mkOption {
      type = with lib.types; attrsOf (submodule partitionOptions);
      default = { };
      example = lib.literalExpression '' {
        "10-esp" = {
          contents = {
            "/EFI/BOOT/BOOTX64.EFI".source =
              "''${pkgs.systemd}/lib/systemd/boot/efi/systemd-bootx64.efi";
          }
          repartConfig = {
            Type = "esp";
            Format = "fat";
          };
        };
        "20-root" = {
          storePaths = [ config.system.build.toplevel ];
          repartConfig = {
            Type = "root";
            Format = "ext4";
            Minimize = "guess";
          };
        };
      };
      '';
      description = lib.mdDoc ''
        Specify partitions as a set of the names of the partitions with their
        configuration as the key.
      '';
    };

  };

  config = {

    system.build.image =
      let
        fileSystemToolMapping = with pkgs; {
          "vfat" = [ dosfstools mtools ];
          "ext4" = [ e2fsprogs.bin ];
          "squashfs" = [ squashfsTools ];
          "erofs" = [ erofs-utils ];
          "btrfs" = [ btrfs-progs ];
          "xfs" = [ xfsprogs ];
        };

        fileSystems = lib.filter
          (f: f != null)
          (lib.mapAttrsToList (_n: v: v.repartConfig.Format or null) cfg.partitions);

        fileSystemTools = builtins.concatMap (f: fileSystemToolMapping."${f}") fileSystems;


        makeClosure = paths: pkgs.closureInfo { rootPaths = paths; };

        # Add the closure of the provided Nix store paths to cfg.partitions so
        # that amend-repart-definitions.py can read it.
        addClosure = _name: partitionConfig: partitionConfig // (
          lib.optionalAttrs
            (partitionConfig.storePaths or [ ] != [ ])
            { closure = "${makeClosure partitionConfig.storePaths}/store-paths"; }
        );


        finalPartitions = lib.mapAttrs addClosure cfg.partitions;


        amendRepartDefinitions = pkgs.runCommand "amend-repart-definitions.py"
          {
            nativeBuildInputs = with pkgs; [ black ruff mypy ];
            buildInputs = [ pkgs.python3 ];
          } ''
          install ${./amend-repart-definitions.py} $out
          patchShebangs --host $out

          black --check --diff $out
          ruff --line-length 88 $out
          mypy --strict $out
        '';

        format = pkgs.formats.ini { };

        definitionsDirectory = utils.systemdUtils.lib.definitions
          "repart.d"
          format
          (lib.mapAttrs (_n: v: { Partition = v.repartConfig; }) finalPartitions);

        partitions = pkgs.writeText "partitions.json" (builtins.toJSON finalPartitions);
      in
      pkgs.runCommand cfg.name
        {
          nativeBuildInputs = with pkgs; [
            fakeroot
            systemd
          ] ++ fileSystemTools;
        } ''
        amendedRepartDefinitions=$(${amendRepartDefinitions} ${partitions} ${definitionsDirectory})

        mkdir -p $out
        cd $out

        fakeroot systemd-repart \
          --dry-run=no \
          --empty=create \
          --size=auto \
          --seed="${cfg.seed}" \
          --definitions="$amendedRepartDefinitions" \
          --split="${lib.boolToString cfg.split}" \
          image.raw
      '';

    meta = {
      maintainers = with lib.maintainers; [ nikstur ];
      doc = ./repart.md;
    };

  };
}
+7 −20
Original line number Diff line number Diff line
{ config, pkgs, lib, utils, ... }:
{ config, lib, pkgs, utils, ... }:

let
  cfg = config.systemd.repart;
  initrdCfg = config.boot.initrd.systemd.repart;

  writeDefinition = name: partitionConfig: pkgs.writeText
    "${name}.conf"
    (lib.generators.toINI { } { Partition = partitionConfig; });

  listOfDefinitions = lib.mapAttrsToList
    writeDefinition
    (lib.filterAttrs (k: _: !(lib.hasPrefix "_" k)) cfg.partitions);

  # Create a directory in the store that contains a copy of all definition
  # files. This is then passed to systemd-repart in the initrd so it can access
  # the definition files after the sysroot has been mounted but before
  # activation. This needs a hard copy of the files and not just symlinks
  # because otherwise the files do not show up in the sysroot.
  definitionsDirectory = pkgs.runCommand "systemd-repart-definitions" { } ''
    mkdir -p $out
    ${(lib.concatStringsSep "\n"
      (map (pkg: "cp ${pkg} $out/${pkg.name}") listOfDefinitions)
    )}
  '';
  format = pkgs.formats.ini { };

  definitionsDirectory = utils.systemdUtils.lib.definitions
    "repart.d"
    format
    (lib.mapAttrs (_n: v: { Partition = v; }) cfg.partitions);
in
{
  options = {
Loading