Commit 471ff2c3 authored by Rafael Fernández López's avatar Rafael Fernández López
Browse files

nixos/nvidia-container-toolkit: rename from `virtualisation.containers.cdi.dynamic.nvidia.enable` 

Add the NixOS option `hardware.nvidia-container-toolkit-cdi-generator.enable`.

This enables the ability to expose GPU's in containers for container
runtimes that support the Container Device Interface (CDI)

Remove `cdi.static` and `cdi.dynamic.nvidia.enable` attributes.
parent 60807215

nixos/doc/manual/release-notes/rl-2405.section.md

+1 −3
Original line number Diff line number Diff line
@@ -42,9 +42,7 @@ Use `services.pipewire.extraConfig` or `services.pipewire.configPackages` for Pi 


- The default dbus implementation has transitioned to dbus-broker from the classic dbus daemon for better performance and reliability. Users can revert to the classic dbus daemon by setting `services.dbus.implementation = "dbus";`. For detailed deviations, refer to [dbus-broker's deviations page](https://github.com/bus1/dbus-broker/wiki/Deviations).



- A new option `virtualisation.containers.cdi` was added. It contains `static` and `dynamic` attributes (corresponding to `/etc/cdi` and `/run/cdi` respectively) to configure the Container Device Interface (CDI).



- `virtualisation.docker.enableNvidia` and `virtualisation.podman.enableNvidia` options are deprecated. `virtualisation.containers.cdi.dynamic.nvidia.enable` should be used instead. This option will expose GPUs on containers with the `--device` CLI option. This is supported by Docker 25, Podman 3.2.0 and Singularity 4. Any container runtime that supports the CDI specification will take advantage of this feature.

- `virtualisation.docker.enableNvidia` and `virtualisation.podman.enableNvidia` options are deprecated. `hardware.nvidia-container-toolkit.enable` should be used instead. This option will expose GPUs on containers with the `--device` CLI option. This is supported by Docker 25, Podman 3.2.0 and Singularity 4. Any container runtime that supports the CDI specification will take advantage of this feature.



- A new option `system.etc.overlay.enable` was added. If enabled, `/etc` is

  mounted via an overlayfs instead of being created by a custom perl script.

nixos/modules/module-list.nix

+1 −1
Original line number Diff line number Diff line
@@ -557,7 +557,7 @@ 
  ./services/hardware/kanata.nix

  ./services/hardware/lcd.nix

  ./services/hardware/lirc.nix

  ./services/hardware/nvidia-container-toolkit-cdi-generator

  ./services/hardware/nvidia-container-toolkit

  ./services/hardware/monado.nix

  ./services/hardware/nvidia-optimus.nix

  ./services/hardware/openrgb.nix

nixos/modules/services/hardware/nvidia-container-toolkit-cdi-generator/default.nix

deleted100644 → 0
+0 −40
Original line number Diff line number Diff line 
{ config, lib, pkgs, ... }:



{



  options = {



    hardware.nvidia-container-toolkit-cdi-generator.enable = lib.mkOption {

      default = false;

      internal = true;

      visible = false;

      type = lib.types.bool;

      description = ''

        Enable dynamic CDI configuration for NVidia devices by running

        nvidia-container-toolkit on boot.

      '';

    };



  };



  config = {



    systemd.services.nvidia-container-toolkit-cdi-generator = lib.mkIf config.hardware.nvidia-container-toolkit-cdi-generator.enable {

      description = "Container Device Interface (CDI) for Nvidia generator";

      wantedBy = [ "multi-user.target" ];

      after = [ "systemd-udev-settle.service" ];

      serviceConfig = {

        RuntimeDirectory = "cdi";

        RemainAfterExit = true;

        ExecStart =

          let

            script = pkgs.callPackage ./cdi-generate.nix { nvidia-driver = config.hardware.nvidia.package; };

          in

          lib.getExe script;

        Type = "oneshot";

      };

    };



  };



}

nixos/modules/services/hardware/nvidia-container-toolkit-cdi-generator/cdi-generate.nix→nixos/modules/services/hardware/nvidia-container-toolkit/cdi-generate.nix

+35 −0
Original line number Diff line number Diff line 
{

  addDriverRunpath,

  glibc,

  jq,

  lib,

  mounts,

  nvidia-container-toolkit,

  nvidia-driver,

  runtimeShell,

  writeScriptBin,

}:

let

  mountOptions = { options = ["ro" "nosuid" "nodev" "bind"]; };

  mounts = [

    # FIXME: Making /usr mounts optional

    { hostPath = lib.getExe' nvidia-driver "nvidia-cuda-mps-control";

      containerPath = "/usr/bin/nvidia-cuda-mps-control"; }

    { hostPath = lib.getExe' nvidia-driver "nvidia-cuda-mps-server";

      containerPath = "/usr/bin/nvidia-cuda-mps-server"; }

    { hostPath = lib.getExe' nvidia-driver "nvidia-debugdump";

      containerPath = "/usr/bin/nvidia-debugdump"; }

    { hostPath = lib.getExe' nvidia-driver "nvidia-powerd";

      containerPath = "/usr/bin/nvidia-powerd"; }

    { hostPath = lib.getExe' nvidia-driver "nvidia-smi";

      containerPath = "/usr/bin/nvidia-smi"; }

    { hostPath = lib.getExe' nvidia-container-toolkit "nvidia-ctk";

      containerPath = "/usr/bin/nvidia-ctk"; }

    { hostPath = "${lib.getLib glibc}/lib";

      containerPath = "${lib.getLib glibc}/lib"; }



    # FIXME: use closureinfo

    {

      hostPath = addDriverRunpath.driverLink;

      containerPath = addDriverRunpath.driverLink;

    }

    { hostPath = "${lib.getLib glibc}/lib";

      containerPath = "${lib.getLib glibc}/lib"; }

    { hostPath = "${lib.getLib glibc}/lib64";

      containerPath = "${lib.getLib glibc}/lib64"; }

  ];

}: let

  mkMount = {hostPath, containerPath, mountOptions}: {

    inherit hostPath containerPath;

    options = mountOptions;

  };

  jqAddMountExpression = ".containerEdits.mounts[.containerEdits.mounts | length] |= . +";

  mountsToJq = lib.concatMap

  allJqMounts = lib.concatMap

    (mount:

      ["${lib.getExe jq} '${jqAddMountExpression} ${builtins.toJSON (mount // mountOptions)}'"])

      ["${lib.getExe jq} '${jqAddMountExpression} ${builtins.toJSON (mkMount mount)}'"])

    mounts;

in

writeScriptBin "nvidia-cdi-generator"
@@ -56,5 +31,5 @@ function cdiGenerate { 
}



cdiGenerate | \

  ${lib.concatStringsSep " | " mountsToJq} > $RUNTIME_DIRECTORY/nvidia-container-toolkit.json

  ${lib.concatStringsSep " | " allJqMounts} > $RUNTIME_DIRECTORY/nvidia-container-toolkit.json

''

nixos/modules/services/hardware/nvidia-container-toolkit/default.nix

0 → 100644
+83 −0
Original line number Diff line number Diff line 
{ config, lib, pkgs, ... }:



{

  imports = [

    (lib.mkRenamedOptionModule

      [ "virtualisation" "containers" "cdi" "dynamic" "nvidia" "enable" ]

      [ "hardware" "nvidia-container-toolkit" "enable" ])

  ];



  options = let

    mountType = {

      options = {

        hostPath = lib.mkOption {

          type = lib.types.str;

          description = "Host path.";

        };

        containerPath = lib.mkOption {

          type = lib.types.str;

          description = "Container path.";

        };

        mountOptions = lib.mkOption {

          default = [ "ro" "nosuid" "nodev" "bind" ];

          type = lib.types.listOf lib.types.str;

          description = "Mount options.";

        };

      };

    };

  in {



    hardware.nvidia-container-toolkit = {

      enable = lib.mkOption {

        default = false;

        type = lib.types.bool;

        description = ''

          Enable dynamic CDI configuration for NVidia devices by running

          nvidia-container-toolkit on boot.

        '';

      };



      mounts = lib.mkOption {

        type = lib.types.listOf (lib.types.submodule mountType);

        default = [];

        description = "Mounts to be added to every container under the Nvidia CDI profile.";

      };

    };



  };



  config = {



    hardware.nvidia-container-toolkit.mounts = let

      nvidia-driver = config.hardware.nvidia.package;

    in (lib.mkMerge [

      [{ hostPath = pkgs.addDriverRunpath.driverLink;

         containerPath = pkgs.addDriverRunpath.driverLink; }

       { hostPath = "${lib.getLib pkgs.glibc}/lib";

         containerPath = "${lib.getLib pkgs.glibc}/lib"; }

       { hostPath = "${lib.getLib pkgs.glibc}/lib64";

         containerPath = "${lib.getLib pkgs.glibc}/lib64"; }]

    ]);



    systemd.services.nvidia-container-toolkit-cdi-generator = lib.mkIf config.hardware.nvidia-container-toolkit.enable {

      description = "Container Device Interface (CDI) for Nvidia generator";

      wantedBy = [ "multi-user.target" ];

      after = [ "systemd-udev-settle.service" ];

      serviceConfig = {

        RuntimeDirectory = "cdi";

        RemainAfterExit = true;

        ExecStart =

          let

            script = pkgs.callPackage ./cdi-generate.nix {

              inherit (config.hardware.nvidia-container-toolkit) mounts;

              nvidia-driver = config.hardware.nvidia.package;

            };

          in

          lib.getExe script;

        Type = "oneshot";

      };

    };



  };



}