Unverified Commit 0b6fa5ee authored by Gabriella Gonzalez's avatar Gabriella Gonzalez Committed by GitHub
Browse files

virtualisation.oci-containers: Add new `imageStream` option (#335430) 

This adds a new `imageStream` option that can be used in conjunction
with `pkgs.dockerTools.streamLayeredImage` so that the image archive
never needs to be materialized in the `/nix/store`.  This greatly
improves the disk utilization for systems that use container images
built using Nix because they only need to store image layers instead of
the full image.  Additionally, when deploying the new system and only
new layers need to be built/copied.
parent 51f0e92a

nixos/modules/virtualisation/oci-containers.nix

+36 −3
Original line number Diff line number Diff line
@@ -33,6 +33,25 @@ let 
          example = literalExpression "pkgs.dockerTools.buildImage {...};";

        };



        imageStream = mkOption {

          type = with types; nullOr package;

          default = null;

          description = ''

            Path to a script that streams the desired image on standard output.



            This option is mainly intended for use with

            `pkgs.dockerTools.streamLayeredImage` so that the intermediate

            image archive does not need to be stored in the Nix store.  For

            larger images this optimization can significantly reduce Nix store

            churn compared to using the `imageFile` option, because you don't

            have to store a new copy of the image archive in the Nix store

            every time you change the image.  Instead, if you stream the image

            then you only need to build and store the layers that differ from

            the previous image.

          '';

          example = literalExpression "pkgs.dockerTools.streamLayeredImage {...};";

        };



        login = {



          username = mkOption {
@@ -275,6 +294,9 @@ let 
        ${optionalString (container.imageFile != null) ''

          ${cfg.backend} load -i ${container.imageFile}

        ''}

        ${optionalString (container.imageStream != null) ''

          ${container.imageStream} | ${cfg.backend} load

        ''}

        ${optionalString (cfg.backend == "podman") ''

          rm -f /run/podman-${escapedName}.ctr-id

        ''}
@@ -282,10 +304,10 @@ let 
    };

  in {

    wantedBy = [] ++ optional (container.autoStart) "multi-user.target";

    wants = lib.optional (container.imageFile == null)  "network-online.target";

    wants = lib.optional (container.imageFile == null && container.imageStream == null)  "network-online.target";

    after = lib.optionals (cfg.backend == "docker") [ "docker.service" "docker.socket" ]

            # if imageFile is not set, the service needs the network to download the image from the registry

            ++ lib.optionals (container.imageFile == null) [ "network-online.target" ]

            # if imageFile or imageStream is not set, the service needs the network to download the image from the registry

            ++ lib.optionals (container.imageFile == null && container.imageStream == null) [ "network-online.target" ]

            ++ dependsOn;

    requires = dependsOn;

    environment = proxy_env;
@@ -393,6 +415,17 @@ in { 
  config = lib.mkIf (cfg.containers != {}) (lib.mkMerge [

    {

      systemd.services = mapAttrs' (n: v: nameValuePair "${cfg.backend}-${n}" (mkService n v)) cfg.containers;



      assertions =

        let

          toAssertion = _: { imageFile, imageStream, ... }:

            { assertion = imageFile == null || imageStream == null;



              message = "You can only define one of imageFile and imageStream";

            };



        in

          lib.mapAttrsToList toAssertion cfg.containers;

    }

    (lib.mkIf (cfg.backend == "podman") {

      virtualisation.podman.enable = true;

nixos/tests/cntr.nix

+1 −1
Original line number Diff line number Diff line
@@ -18,7 +18,7 @@ let 
            inherit backend;

            containers.nginx = {

              image = "nginx-container";

              imageFile = pkgs.dockerTools.examples.nginx;

              imageStream = pkgs.dockerTools.examples.nginxStream;

              ports = [ "8181:80" ];

            };

          };

nixos/tests/oci-containers.nix

+1 −1
Original line number Diff line number Diff line
@@ -20,7 +20,7 @@ let 
          inherit backend;

          containers.nginx = {

            image = "nginx-container";

            imageFile = pkgs.dockerTools.examples.nginx;

            imageStream = pkgs.dockerTools.examples.nginxStream;

            ports = ["8181:80"];

          };

        };

nixos/tests/traefik.nix

+1 −1
Original line number Diff line number Diff line
@@ -23,7 +23,7 @@ import ./make-test-python.nix ({ pkgs, ... }: { 
            "traefik.http.routers.nginx.rule=Host(`nginx.traefik.test`)"

          ];

          image = "nginx-container";

          imageFile = pkgs.dockerTools.examples.nginx;

          imageStream = pkgs.dockerTools.examples.nginxStream;

        };

      };

pkgs/build-support/docker/examples.nix

+49 −45
Original line number Diff line number Diff line
@@ -17,6 +17,52 @@ let 
  };

  evalMinimalConfig = module: nixosLib.evalModules { modules = [ module ]; };



  nginxArguments = let

    nginxPort = "80";

    nginxConf = pkgs.writeText "nginx.conf" ''

      user nobody nobody;

      daemon off;

      error_log /dev/stdout info;

      pid /dev/null;

      events {}

      http {

        access_log /dev/stdout;

        server {

          listen ${nginxPort};

          index index.html;

          location / {

            root ${nginxWebRoot};

          }

        }

      }

    '';

    nginxWebRoot = pkgs.writeTextDir "index.html" ''

      <html><body><h1>Hello from NGINX</h1></body></html>

    '';

  in

  { name = "nginx-container";

    tag = "latest";

    contents = [

      fakeNss

      pkgs.nginx

    ];



    extraCommands = ''

      mkdir -p tmp/nginx_client_body



      # nginx still tries to read this directory even if error_log

      # directive is specifying another file :/

      mkdir -p var/log/nginx

    '';



    config = {

      Cmd = [ "nginx" "-c" nginxConf ];

      ExposedPorts = {

        "${nginxPort}/tcp" = {};

      };

    };

  };



in



rec {
@@ -60,52 +106,10 @@ rec { 
  };



  # 3. another service example

  nginx = let

    nginxPort = "80";

    nginxConf = pkgs.writeText "nginx.conf" ''

      user nobody nobody;

      daemon off;

      error_log /dev/stdout info;

      pid /dev/null;

      events {}

      http {

        access_log /dev/stdout;

        server {

          listen ${nginxPort};

          index index.html;

          location / {

            root ${nginxWebRoot};

          }

        }

      }

    '';

    nginxWebRoot = pkgs.writeTextDir "index.html" ''

      <html><body><h1>Hello from NGINX</h1></body></html>

    '';

  in

  buildLayeredImage {

    name = "nginx-container";

    tag = "latest";

    contents = [

      fakeNss

      pkgs.nginx

    ];

  nginx = buildLayeredImage nginxArguments;



    extraCommands = ''

      mkdir -p tmp/nginx_client_body



      # nginx still tries to read this directory even if error_log

      # directive is specifying another file :/

      mkdir -p var/log/nginx

    '';



    config = {

      Cmd = [ "nginx" "-c" nginxConf ];

      ExposedPorts = {

        "${nginxPort}/tcp" = {};

      };

    };

  };

  # Used to demonstrate how virtualisation.oci-containers.imageStream works

  nginxStream = pkgs.dockerTools.streamLayeredImage nginxArguments;



  # 4. example of pulling an image. could be used as a base for other images

  nixFromDockerHub = pullImage {