Unverified Commit 117bbae4 authored by Rafael Fernández López's avatar Rafael Fernández López
Browse files

nvidia-container-toolkit: reintroduce nvidia runtime wrappers 

This allows users to keep using `docker run --gpus`. Despite CDI is
the recommended way to expose GPU's to containers nowadays, allow
users to keep using the old `--gpus` method.
parent c804ea5a

nixos/modules/services/hardware/nvidia-container-toolkit/default.nix

+173 −120
Original line number Diff line number Diff line
@@ -39,7 +39,6 @@ 
      };

    in

    {



      hardware.nvidia-container-toolkit = {

        enable = lib.mkOption {

          default = false;
@@ -129,10 +128,44 @@ 
          '';

        };

      };

    };



  config = lib.mkMerge [

    (lib.mkIf config.virtualisation.docker.enableNvidia {

      environment.etc."nvidia-container-runtime/config.toml".text = ''

        disable-require = true

        supported-driver-capabilities = "compat32,compute,display,graphics,ngx,utility,video"

        [nvidia-container-cli]

        environment = []

        ldconfig = "@${lib.getExe' pkgs.glibc "ldconfig"}"

        load-kmods = true

        no-cgroups = false

        path = "${lib.getExe' pkgs.libnvidia-container "nvidia-container-cli"}"

        [nvidia-container-runtime]

        mode = "auto"

        runtimes = ["docker-runc", "runc", "crun"]

        [nvidia-container-runtime-hook]

        path = "${lib.getOutput "tools" config.hardware.nvidia-container-toolkit.package}/bin/nvidia-container-runtime-hook"

        skip-mode-detection = false

        [nvidia-ctk]

        path = "${lib.getExe' config.hardware.nvidia-container-toolkit.package "nvidia-ctk"}"

      '';



      virtualisation.docker = {

        daemon.settings = {

          default-runtime = "nvidia";

          runtimes.nvidia = {

            path = "${lib.getOutput "tools" config.hardware.nvidia-container-toolkit.package}/bin/nvidia-container-runtime";

            args = [ ];

          };

        };



  config = lib.mkIf config.hardware.nvidia-container-toolkit.enable {

        extraPackages = [

          (lib.getOutput "tools" config.hardware.nvidia-container-toolkit.package)

        ];

      };

    })

    (lib.mkIf config.hardware.nvidia-container-toolkit.enable {

      assertions = [

        {

          assertion =
@@ -149,20 +182,40 @@ 
        }

      ];



    virtualisation.docker = {

      daemon.settings = lib.mkIf (lib.versionAtLeast config.virtualisation.docker.package.version "25") {

      warnings = lib.mkMerge [

        (lib.mkIf config.virtualisation.podman.enableNvidia [

          "Setting virtualisation.podman.enableNvidia has no effect and will be removed soon."

        ])

      ];



      virtualisation = {

        containers.containersConf.settings = {

          engine = {

            cdi_spec_dirs = [

              "/etc/cdi"

              "/var/run/cdi"

            ];

          };

        };

        docker =

          let

            dockerVersion = config.virtualisation.docker.package.version;

          in

          {

            daemon.settings = lib.mkIf (lib.versionAtLeast dockerVersion "25") {

              features.cdi = true;

            };



      rootless.daemon.settings =

        lib.mkIf

          (

            config.virtualisation.docker.rootless.enable

            && (lib.versionAtLeast config.virtualisation.docker.package.version "25")

          )

          {

            rootless = {

              daemon.settings = lib.mkIf (lib.versionAtLeast dockerVersion "25") {

                features.cdi = true;

              };



              extraPackages = [

                (lib.getOutput "tools" config.hardware.nvidia-container-toolkit.package)

              ];

            };

          };

      };



      hardware = {
@@ -259,7 +312,7 @@ 
          Type = "oneshot";

        };

      };



  };

    })

  ];



}

nixos/modules/virtualisation/docker-rootless.nix

+9 −1
Original line number Diff line number Diff line
@@ -50,6 +50,14 @@ in 
    };



    package = lib.mkPackageOption pkgs "docker" { };



    extraPackages = lib.mkOption {

      type = lib.types.listOf lib.types.package;

      default = [ ];

      description = ''

        Extra packages to add to PATH for the docker daemon process.

      '';

    };

  };



  ###### implementation
@@ -68,7 +76,7 @@ in 
      wantedBy = [ "default.target" ];

      description = "Docker Application Container Engine (Rootless)";

      # needs newuidmap from pkgs.shadow

      path = [ "/run/wrappers" ];

      path = [ "/run/wrappers" ] ++ cfg.extraPackages;

      environment = proxy_env;

      unitConfig = {

        # docker-rootless doesn't support running as root.

nixos/modules/virtualisation/docker.nix

+3 −6
Original line number Diff line number Diff line
@@ -88,7 +88,7 @@ in 
      description = ''

        **Deprecated**, please use hardware.nvidia-container-toolkit.enable instead.



        Enable nvidia-docker wrapper, supporting NVIDIA GPUs inside docker containers.

        Enable Nvidia GPU support inside docker containers.

      '';

    };
@@ -246,7 +246,7 @@ in 
        "net.ipv4.conf.all.forwarding" = mkOverride 98 true;

        "net.ipv4.conf.default.forwarding" = mkOverride 98 true;

      };

      environment.systemPackages = [ cfg.package ] ++ optional cfg.enableNvidia pkgs.nvidia-docker;

      environment.systemPackages = [ cfg.package ];

      users.groups.docker.gid = config.ids.gids.docker;

      systemd.packages = [ cfg.package ];
@@ -287,10 +287,7 @@ in 
        };



        path =

          [ pkgs.kmod ]

          ++ optional (cfg.storageDriver == "zfs") pkgs.zfs

          ++ optional cfg.enableNvidia pkgs.nvidia-docker

          ++ cfg.extraPackages;

          [ pkgs.kmod ] ++ optional (cfg.storageDriver == "zfs") config.boot.zfs.package ++ cfg.extraPackages;

      };



      systemd.sockets.docker = {

pkgs/by-name/nv/nvidia-container-toolkit/package.nix

+4 −30
Original line number Diff line number Diff line
@@ -5,23 +5,11 @@ 
  makeWrapper,

  buildGoModule,

  formats,

  configTemplate ? null,

  configTemplatePath ? null,

  libnvidia-container,

  autoAddDriverRunpath,

}:



assert configTemplate != null -> (lib.isAttrs configTemplate && configTemplatePath == null);

assert

  configTemplatePath != null -> (lib.isStringLike configTemplatePath && configTemplate == null);



let

  configToml =

    if configTemplatePath != null then

      configTemplatePath

    else

      (formats.toml { }).generate "config.toml" configTemplate;



  # From https://gitlab.com/nvidia/container-toolkit/container-toolkit/-/blob/03cbf9c6cd26c75afef8a2dd68e0306aace80401/Makefile#L54

  cliVersionPackage = "github.com/NVIDIA/nvidia-container-toolkit/internal/info";

in
@@ -97,23 +85,9 @@ buildGoModule (finalAttrs: { 
    in

    [ "-skip=^${builtins.concatStringsSep "$|^" skippedTests}$" ];



  postInstall =

    ''

  postInstall = ''

    mkdir -p $tools/bin

    mv $out/bin/{nvidia-cdi-hook,nvidia-container-runtime,nvidia-container-runtime.cdi,nvidia-container-runtime-hook,nvidia-container-runtime.legacy} $tools/bin



      for bin in nvidia-container-runtime-hook nvidia-container-runtime; do

        wrapProgram $tools/bin/$bin \

          --prefix PATH : ${libnvidia-container}/bin:$out/bin

      done

    ''

    + lib.optionalString (configTemplate != null || configTemplatePath != null) ''

      mkdir -p $out/etc/nvidia-container-runtime



      cp ${configToml} $out/etc/nvidia-container-runtime/config.toml



      substituteInPlace $out/etc/nvidia-container-runtime/config.toml \

        --subst-var-by glibcbin ${lib.getBin glibc}

  '';



  meta = {

pkgs/by-name/nv/nvidia-container-toolkit/packages.nix

+1 −22
Original line number Diff line number Diff line
@@ -6,28 +6,7 @@ 


# Note this scope isn't recursed into, at the time of writing.

lib.makeScope newScope (self: {



  # The config is only exposed as an attrset so that the user may reach the

  # default values, for inspectability purposes.

  dockerConfig = {

    disable-require = false;

    #swarm-resource = "DOCKER_RESOURCE_GPU"



    nvidia-container-cli = {

      #root = "/run/nvidia/driver";

      #path = "/usr/bin/nvidia-container-cli";

      environment = [ ];

      #debug = "/var/log/nvidia-container-runtime-hook.log";

      ldcache = "/tmp/ld.so.cache";

      load-kmods = true;

      #no-cgroups = false;

      #user = "root:video";

      ldconfig = "@@glibcbin@/bin/ldconfig";

    };

  };

  nvidia-container-toolkit-docker = self.callPackage ./package.nix {

    configTemplate = self.dockerConfig;

  };

  nvidia-container-toolkit-docker = self.callPackage ./package.nix { };



  nvidia-docker = symlinkJoin {

    name = "nvidia-docker";