Unverified Commit 70359688 authored by Someone's avatar Someone Committed by GitHub
Browse files

Merge pull request #290979 from ereslibre/cdi-add-nvidia-docker-1-directories 

CDI: Add `mount-nvidia-binaries` and `mount-nvidia-docker-1-directories` options
parents 83cbc65d de3ce5ff

nixos/doc/manual/release-notes/rl-2405.section.md

+1 −3
Original line number Diff line number Diff line
@@ -46,9 +46,7 @@ Use `services.pipewire.extraConfig` or `services.pipewire.configPackages` for Pi 


- The default dbus implementation has transitioned to dbus-broker from the classic dbus daemon for better performance and reliability. Users can revert to the classic dbus daemon by setting `services.dbus.implementation = "dbus";`. For detailed deviations, refer to [dbus-broker's deviations page](https://github.com/bus1/dbus-broker/wiki/Deviations).



- A new option `virtualisation.containers.cdi` was added. It contains `static` and `dynamic` attributes (corresponding to `/etc/cdi` and `/run/cdi` respectively) to configure the Container Device Interface (CDI).



- `virtualisation.docker.enableNvidia` and `virtualisation.podman.enableNvidia` options are deprecated. `virtualisation.containers.cdi.dynamic.nvidia.enable` should be used instead. This option will expose GPUs on containers with the `--device` CLI option. This is supported by Docker 25, Podman 3.2.0 and Singularity 4. Any container runtime that supports the CDI specification will take advantage of this feature.

- `virtualisation.docker.enableNvidia` and `virtualisation.podman.enableNvidia` options are deprecated. `hardware.nvidia-container-toolkit.enable` should be used instead. This option will expose GPUs on containers with the `--device` CLI option. This is supported by Docker 25, Podman 3.2.0 and Singularity 4. Any container runtime that supports the CDI specification will take advantage of this feature.



- `system.etc.overlay.enable` option was added. If enabled, `/etc` is

  mounted via an overlayfs instead of being created by a custom perl script.

nixos/modules/module-list.nix

+1 −1
Original line number Diff line number Diff line
@@ -559,7 +559,7 @@ 
  ./services/hardware/kanata.nix

  ./services/hardware/lcd.nix

  ./services/hardware/lirc.nix

  ./services/hardware/nvidia-container-toolkit-cdi-generator

  ./services/hardware/nvidia-container-toolkit

  ./services/hardware/monado.nix

  ./services/hardware/nvidia-optimus.nix

  ./services/hardware/openrgb.nix

nixos/modules/services/hardware/nvidia-container-toolkit-cdi-generator/default.nix

deleted100644 → 0
+0 −40
Original line number Diff line number Diff line 
{ config, lib, pkgs, ... }:



{



  options = {



    hardware.nvidia-container-toolkit-cdi-generator.enable = lib.mkOption {

      default = false;

      internal = true;

      visible = false;

      type = lib.types.bool;

      description = ''

        Enable dynamic CDI configuration for NVidia devices by running

        nvidia-container-toolkit on boot.

      '';

    };



  };



  config = {



    systemd.services.nvidia-container-toolkit-cdi-generator = lib.mkIf config.hardware.nvidia-container-toolkit-cdi-generator.enable {

      description = "Container Device Interface (CDI) for Nvidia generator";

      wantedBy = [ "multi-user.target" ];

      after = [ "systemd-udev-settle.service" ];

      serviceConfig = {

        RuntimeDirectory = "cdi";

        RemainAfterExit = true;

        ExecStart =

          let

            script = pkgs.callPackage ./cdi-generate.nix { nvidia-driver = config.hardware.nvidia.package; };

          in

          lib.getExe script;

        Type = "oneshot";

      };

    };



  };



}

nixos/modules/services/hardware/nvidia-container-toolkit-cdi-generator/cdi-generate.nix→nixos/modules/services/hardware/nvidia-container-toolkit/cdi-generate.nix

+35 −0
Original line number Diff line number Diff line 
{

  addDriverRunpath,

  glibc,

  jq,

  lib,

  mounts,

  nvidia-container-toolkit,

  nvidia-driver,

  runtimeShell,

  writeScriptBin,

}:

let

  mountOptions = { options = ["ro" "nosuid" "nodev" "bind"]; };

  mounts = [

    # FIXME: Making /usr mounts optional

    { hostPath = lib.getExe' nvidia-driver "nvidia-cuda-mps-control";

      containerPath = "/usr/bin/nvidia-cuda-mps-control"; }

    { hostPath = lib.getExe' nvidia-driver "nvidia-cuda-mps-server";

      containerPath = "/usr/bin/nvidia-cuda-mps-server"; }

    { hostPath = lib.getExe' nvidia-driver "nvidia-debugdump";

      containerPath = "/usr/bin/nvidia-debugdump"; }

    { hostPath = lib.getExe' nvidia-driver "nvidia-powerd";

      containerPath = "/usr/bin/nvidia-powerd"; }

    { hostPath = lib.getExe' nvidia-driver "nvidia-smi";

      containerPath = "/usr/bin/nvidia-smi"; }

    { hostPath = lib.getExe' nvidia-container-toolkit "nvidia-ctk";

      containerPath = "/usr/bin/nvidia-ctk"; }

    { hostPath = "${lib.getLib glibc}/lib";

      containerPath = "${lib.getLib glibc}/lib"; }



    # FIXME: use closureinfo

    {

      hostPath = addDriverRunpath.driverLink;

      containerPath = addDriverRunpath.driverLink;

    }

    { hostPath = "${lib.getLib glibc}/lib";

      containerPath = "${lib.getLib glibc}/lib"; }

    { hostPath = "${lib.getLib glibc}/lib64";

      containerPath = "${lib.getLib glibc}/lib64"; }

  ];

}: let

  mkMount = {hostPath, containerPath, mountOptions}: {

    inherit hostPath containerPath;

    options = mountOptions;

  };

  jqAddMountExpression = ".containerEdits.mounts[.containerEdits.mounts | length] |= . +";

  mountsToJq = lib.concatMap

  allJqMounts = lib.concatMap

    (mount:

      ["${lib.getExe jq} '${jqAddMountExpression} ${builtins.toJSON (mount // mountOptions)}'"])

      ["${lib.getExe jq} '${jqAddMountExpression} ${builtins.toJSON (mkMount mount)}'"])

    mounts;

in

writeScriptBin "nvidia-cdi-generator"
@@ -56,5 +31,5 @@ function cdiGenerate { 
}



cdiGenerate | \

  ${lib.concatStringsSep " | " mountsToJq} > $RUNTIME_DIRECTORY/nvidia-container-toolkit.json

  ${lib.concatStringsSep " | " allJqMounts} > $RUNTIME_DIRECTORY/nvidia-container-toolkit.json

''

nixos/modules/services/hardware/nvidia-container-toolkit/default.nix

0 → 100644
+121 −0
Original line number Diff line number Diff line 
{ config, lib, pkgs, ... }:



{

  imports = [

    (lib.mkRenamedOptionModule

      [ "virtualisation" "containers" "cdi" "dynamic" "nvidia" "enable" ]

      [ "hardware" "nvidia-container-toolkit" "enable" ])

  ];



  options = let

    mountType = {

      options = {

        hostPath = lib.mkOption {

          type = lib.types.str;

          description = "Host path.";

        };

        containerPath = lib.mkOption {

          type = lib.types.str;

          description = "Container path.";

        };

        mountOptions = lib.mkOption {

          default = [ "ro" "nosuid" "nodev" "bind" ];

          type = lib.types.listOf lib.types.str;

          description = "Mount options.";

        };

      };

    };

  in {



    hardware.nvidia-container-toolkit = {

      enable = lib.mkOption {

        default = false;

        type = lib.types.bool;

        description = ''

          Enable dynamic CDI configuration for NVidia devices by running

          nvidia-container-toolkit on boot.

        '';

      };



      mounts = lib.mkOption {

        type = lib.types.listOf (lib.types.submodule mountType);

        default = [];

        description = "Mounts to be added to every container under the Nvidia CDI profile.";

      };



      mount-nvidia-executables = lib.mkOption {

        default = true;

        type = lib.types.bool;

        description = ''

          Mount executables nvidia-smi, nvidia-cuda-mps-control, nvidia-cuda-mps-server,

          nvidia-debugdump, nvidia-powerd and nvidia-ctk on containers.

        '';

      };



      mount-nvidia-docker-1-directories = lib.mkOption {

        default = true;

        type = lib.types.bool;

        description = ''

          Mount nvidia-docker-1 directories on containers: /usr/local/nvidia/lib and

          /usr/local/nvidia/lib64.

        '';

      };

    };



  };



  config = {



    hardware.nvidia-container-toolkit.mounts = let

      nvidia-driver = config.hardware.nvidia.package;

    in (lib.mkMerge [

      [{ hostPath = pkgs.addDriverRunpath.driverLink;

         containerPath = pkgs.addDriverRunpath.driverLink; }

       { hostPath = "${lib.getLib pkgs.glibc}/lib";

         containerPath = "${lib.getLib pkgs.glibc}/lib"; }

       { hostPath = "${lib.getLib pkgs.glibc}/lib64";

         containerPath = "${lib.getLib pkgs.glibc}/lib64"; }]

      (lib.mkIf config.hardware.nvidia-container-toolkit.mount-nvidia-executables

        [{ hostPath = lib.getExe' nvidia-driver "nvidia-cuda-mps-control";

           containerPath = "/usr/bin/nvidia-cuda-mps-control"; }

         { hostPath = lib.getExe' nvidia-driver "nvidia-cuda-mps-server";

           containerPath = "/usr/bin/nvidia-cuda-mps-server"; }

         { hostPath = lib.getExe' nvidia-driver "nvidia-debugdump";

           containerPath = "/usr/bin/nvidia-debugdump"; }

         { hostPath = lib.getExe' nvidia-driver "nvidia-powerd";

           containerPath = "/usr/bin/nvidia-powerd"; }

         { hostPath = lib.getExe' nvidia-driver "nvidia-smi";

           containerPath = "/usr/bin/nvidia-smi"; }])

      # nvidia-docker 1.0 uses /usr/local/nvidia/lib{,64}

      #   e.g.

      #     - https://gitlab.com/nvidia/container-images/cuda/-/blob/e3ff10eab3a1424fe394899df0e0f8ca5a410f0f/dist/12.3.1/ubi9/base/Dockerfile#L44

      #     - https://github.com/NVIDIA/nvidia-docker/blob/01d2c9436620d7dde4672e414698afe6da4a282f/src/nvidia/volumes.go#L104-L173

      (lib.mkIf config.hardware.nvidia-container-toolkit.mount-nvidia-docker-1-directories

        [{ hostPath = "${lib.getLib nvidia-driver}/lib";

           containerPath = "/usr/local/nvidia/lib"; }

         { hostPath = "${lib.getLib nvidia-driver}/lib";

           containerPath = "/usr/local/nvidia/lib64"; }])

    ]);



    systemd.services.nvidia-container-toolkit-cdi-generator = lib.mkIf config.hardware.nvidia-container-toolkit.enable {

      description = "Container Device Interface (CDI) for Nvidia generator";

      wantedBy = [ "multi-user.target" ];

      after = [ "systemd-udev-settle.service" ];

      serviceConfig = {

        RuntimeDirectory = "cdi";

        RemainAfterExit = true;

        ExecStart =

          let

            script = pkgs.callPackage ./cdi-generate.nix {

              inherit (config.hardware.nvidia-container-toolkit) mounts;

              nvidia-driver = config.hardware.nvidia.package;

            };

          in

          lib.getExe script;

        Type = "oneshot";

      };

    };



  };



}