Commit aae85881 authored by Louis Blin's avatar Louis Blin
Browse files

dockerTools.buildLayeredImage: support fromImage 

It is now possible to pass a `fromImage` to `buildLayeredImage` and
`streamLayeredImage`, similar to what `buildImage` currently supports.

This will prepend the layers of the given base image to the resulting
image, while ensuring that at most `maxLayers` are used. It will also
ensure that environment variables from the base image are propagated
to the final image.
parent 148e6860

doc/builders/images/dockertools.section.md

+6 −0
Original line number Diff line number Diff line
@@ -111,6 +111,12 @@ Create a Docker image with many of the store paths being on their own layer to i 


    *Default:* the output path's hash



`fromImage` _optional_



: The repository tarball containing the base image. It must be a valid Docker image, such as one exported by `docker save`.



    *Default:* `null`, which can be seen as equivalent to `FROM scratch` of a `Dockerfile`.



`contents` _optional_



: Top level paths in the container. Either a single derivation, or a list of derivations.

nixos/tests/docker-tools.nix

+29 −1
Original line number Diff line number Diff line
@@ -161,12 +161,18 @@ import ./make-test-python.nix ({ pkgs, ... }: { 
            "docker run --rm ${examples.layered-image.imageName} cat extraCommands",

        )



    with subtest("Ensure building an image on top of a layered Docker images work"):

    with subtest("Ensure images built on top of layered Docker images work"):

        docker.succeed(

            "docker load --input='${examples.layered-on-top}'",

            "docker run --rm ${examples.layered-on-top.imageName}",

        )



    with subtest("Ensure layered images built on top of layered Docker images work"):

        docker.succeed(

            "docker load --input='${examples.layered-on-top-layered}'",

            "docker run --rm ${examples.layered-on-top-layered.imageName}",

        )





    def set_of_layers(image_name):

        return set(
@@ -205,6 +211,16 @@ import ./make-test-python.nix ({ pkgs, ... }: { 
        assert "FROM_CHILD=true" in env, "envvars from the child should be preserved"

        assert "LAST_LAYER=child" in env, "envvars from the child should take priority"



    with subtest("Ensure environment variables of layered images are correctly inherited"):

        docker.succeed(

            "docker load --input='${examples.environmentVariablesLayered}'"

        )

        out = docker.succeed("docker run --rm ${examples.environmentVariablesLayered.imageName} env")

        env = out.splitlines()

        assert "FROM_PARENT=true" in env, "envvars from the parent should be preserved"

        assert "FROM_CHILD=true" in env, "envvars from the child should be preserved"

        assert "LAST_LAYER=child" in env, "envvars from the child should take priority"



    with subtest("Ensure image with only 2 layers can be loaded"):

        docker.succeed(

            "docker load --input='${examples.two-layered-image}'"
@@ -219,6 +235,18 @@ import ./make-test-python.nix ({ pkgs, ... }: { 
            "docker run bulk-layer ls /bin/hello",

        )



    with subtest(

        "Ensure the bulk layer with a base image respects the number of maxLayers"

    ):

        docker.succeed(

            "docker load --input='${pkgs.dockerTools.examples.layered-bulk-layer}'",

            # Ensure the image runs correctly

            "docker run layered-bulk-layer ls /bin/hello",

        )



        # Ensure the image has the correct number of layers

        assert len(set_of_layers("layered-bulk-layer")) == 4



    with subtest("Ensure correct behavior when no store is needed"):

        # This check tests that buildLayeredImage can build images that don't need a store.

        docker.succeed(

pkgs/build-support/docker/default.nix

+27 −2
Original line number Diff line number Diff line
@@ -729,6 +729,8 @@ rec { 
    name,

    # Image tag, the Nix's output hash will be used if null

    tag ? null,

    # Parent image, to append to.

    fromImage ? null,

    # Files to put on the image (a nix store path or list of paths).

    contents ? [],

    # Docker config; e.g. what command to run on the container.
@@ -791,7 +793,7 @@ rec { 
      unnecessaryDrvs = [ baseJson overallClosure ];



      conf = runCommand "${baseName}-conf.json" {

        inherit maxLayers created;

        inherit fromImage maxLayers created;

        imageName = lib.toLower name;

        passthru.imageTag =

          if tag != null
@@ -821,6 +823,27 @@ rec { 
                         unnecessaryDrvs}

        }



        # Compute the number of layers that are already used by a potential

        # 'fromImage' as well as the customization layer. Ensure that there is

        # still at least one layer available to store the image contents.

        usedLayers=0



        # subtract number of base image layers

        if [[ -n "$fromImage" ]]; then

          (( usedLayers += $(tar -xOf "$fromImage" manifest.json | jq '.[0].Layers | length') ))

        fi



        # one layer will be taken up by the customisation layer

        (( usedLayers += 1 ))



        if ! (( $usedLayers < $maxLayers )); then

          echo >&2 "Error: usedLayers $usedLayers layers to store 'fromImage' and" \

                    "'extraCommands', but only maxLayers=$maxLayers were" \

                    "allowed. At least 1 layer is required to store contents."

          exit 1

        fi

        availableLayers=$(( maxLayers - usedLayers ))



        # Create $maxLayers worth of Docker Layers, one layer per store path

        # unless there are more paths than $maxLayers. In that case, create

        # $maxLayers-1 for the most popular layers, and smush the remainaing
@@ -838,18 +861,20 @@ rec { 
                | (.[:$maxLayers-1] | map([.])) + [ .[$maxLayers-1:] ]

                | map(select(length > 0))

            ' \

              --argjson maxLayers "$(( maxLayers - 1 ))" # one layer will be taken up by the customisation layer

              --argjson maxLayers "$availableLayers"

        )"



        cat ${baseJson} | jq '

          . + {

            "store_dir": $store_dir,

            "from_image": $from_image,

            "store_layers": $store_layers,

            "customisation_layer", $customisation_layer,

            "repo_tag": $repo_tag,

            "created": $created

          }

          ' --arg store_dir "${storeDir}" \

            --argjson from_image ${if fromImage == null then "null" else "'\"${fromImage}\"'"} \

            --argjson store_layers "$store_layers" \

            --arg customisation_layer ${customisationLayer} \

            --arg repo_tag "$imageName:$imageTag" \

pkgs/build-support/docker/examples.nix

+64 −21
Original line number Diff line number Diff line
@@ -188,7 +188,25 @@ rec { 
    };

  };



  # 12. example of running something as root on top of a parent image

  # 12 Create a layered image on top of a layered image

  layered-on-top-layered = pkgs.dockerTools.buildLayeredImage {

    name = "layered-on-top-layered";

    tag = "latest";

    fromImage = layered-image;

    extraCommands = ''

      mkdir ./example-output

      chmod 777 ./example-output

    '';

    config = {

      Env = [ "PATH=${pkgs.coreutils}/bin/" ];

      WorkingDir = "/example-output";

      Cmd = [

        "${pkgs.bash}/bin/bash" "-c" "echo hello > foo; cat foo"

      ];

    };

  };



  # 13. example of running something as root on top of a parent image

  # Regression test related to PR #52109

  runAsRootParentImage = buildImage {

    name = "runAsRootParentImage";
@@ -197,7 +215,7 @@ rec { 
    fromImage = bash;

  };



  # 13. example of 3 layers images This image is used to verify the

  # 14. example of 3 layers images This image is used to verify the

  # order of layers is correct.

  # It allows to validate

  # - the layer of parent are below
@@ -235,11 +253,10 @@ rec { 
    '';

  };



  # 14. Environment variable inheritance.

  # 15. Environment variable inheritance.

  # Child image should inherit parents environment variables,

  # optionally overriding them.

  environmentVariables = let

    parent = pkgs.dockerTools.buildImage {

  environmentVariablesParent = pkgs.dockerTools.buildImage {

    name = "parent";

    tag = "latest";

    config = {
@@ -249,9 +266,10 @@ rec { 
      ];

    };

  };

  in pkgs.dockerTools.buildImage {



  environmentVariables = pkgs.dockerTools.buildImage {

    name = "child";

    fromImage = parent;

    fromImage = environmentVariablesParent;

    tag = "latest";

    contents = [ pkgs.coreutils ];

    config = {
@@ -262,14 +280,27 @@ rec { 
    };

  };



  # 15. Create another layered image, for comparing layers with image 10.

  environmentVariablesLayered = pkgs.dockerTools.buildLayeredImage {

    name = "child";

    fromImage = environmentVariablesParent;

    tag = "latest";

    contents = [ pkgs.coreutils ];

    config = {

      Env = [

        "FROM_CHILD=true"

        "LAST_LAYER=child"

      ];

    };

  };



  # 16. Create another layered image, for comparing layers with image 10.

  another-layered-image = pkgs.dockerTools.buildLayeredImage {

    name = "another-layered-image";

    tag = "latest";

    config.Cmd = [ "${pkgs.hello}/bin/hello" ];

  };



  # 16. Create a layered image with only 2 layers

  # 17. Create a layered image with only 2 layers

  two-layered-image = pkgs.dockerTools.buildLayeredImage {

    name = "two-layered-image";

    tag = "latest";
@@ -278,7 +309,7 @@ rec { 
    maxLayers = 2;

  };



  # 17. Create a layered image with more packages than max layers.

  # 18. Create a layered image with more packages than max layers.

  # coreutils and hello are part of the same layer

  bulk-layer = pkgs.dockerTools.buildLayeredImage {

    name = "bulk-layer";
@@ -289,7 +320,19 @@ rec { 
    maxLayers = 2;

  };



  # 18. Create a "layered" image without nix store layers. This is not

  # 19. Create a layered image with a base image and more packages than max

  # layers. coreutils and hello are part of the same layer

  layered-bulk-layer = pkgs.dockerTools.buildLayeredImage {

    name = "layered-bulk-layer";

    tag = "latest";

    fromImage = two-layered-image;

    contents = with pkgs; [

      coreutils hello

    ];

    maxLayers = 4;

  };



  # 20. Create a "layered" image without nix store layers. This is not

  # recommended, but can be useful for base images in rare cases.

  no-store-paths = pkgs.dockerTools.buildLayeredImage {

    name = "no-store-paths";
@@ -321,7 +364,7 @@ rec { 
    };

  };



  # 19. Support files in the store on buildLayeredImage

  # 21. Support files in the store on buildLayeredImage

  # See: https://github.com/NixOS/nixpkgs/pull/91084#issuecomment-653496223

  filesInStore = pkgs.dockerTools.buildLayeredImageWithNixDb {

    name = "file-in-store";
@@ -341,7 +384,7 @@ rec { 
    };

  };



  # 20. Ensure that setting created to now results in a date which

  # 22. Ensure that setting created to now results in a date which

  # isn't the epoch + 1 for layered images.

  unstableDateLayered = pkgs.dockerTools.buildLayeredImage {

    name = "unstable-date-layered";

pkgs/build-support/docker/stream_layered_image.py

+87 −7
Original line number Diff line number Diff line
@@ -33,6 +33,7 @@ function does all this. 


import io

import os

import re

import sys

import json

import hashlib
@@ -126,10 +127,85 @@ class ExtractChecksum: 
        return (self._digest.hexdigest(), self._size)





FromImage = namedtuple("FromImage", ["tar", "manifest_json", "image_json"])

# Some metadata for a layer

LayerInfo = namedtuple("LayerInfo", ["size", "checksum", "path", "paths"])





def load_from_image(from_image_str):

    """

    Loads the given base image, if any.



    from_image_str: Path to the base image archive.



    Returns: A 'FromImage' object with references to the loaded base image,

             or 'None' if no base image was provided.

    """

    if from_image_str is None:

        return None



    base_tar = tarfile.open(from_image_str)



    manifest_json_tarinfo = base_tar.getmember("manifest.json")

    with base_tar.extractfile(manifest_json_tarinfo) as f:

        manifest_json = json.load(f)



    image_json_tarinfo = base_tar.getmember(manifest_json[0]["Config"])

    with base_tar.extractfile(image_json_tarinfo) as f:

        image_json = json.load(f)



    return FromImage(base_tar, manifest_json, image_json)





def add_base_layers(tar, from_image):

    """

    Adds the layers from the given base image to the final image.



    tar: 'tarfile.TarFile' object for new layers to be added to.

    from_image: 'FromImage' object with references to the loaded base image.

    """

    if from_image is None:

        print("No 'fromImage' provided", file=sys.stderr)

        return []



    layers = from_image.manifest_json[0]["Layers"]

    checksums = from_image.image_json["rootfs"]["diff_ids"]

    layers_checksums = zip(layers, checksums)



    for num, (layer, checksum) in enumerate(layers_checksums, start=1):

        layer_tarinfo = from_image.tar.getmember(layer)

        checksum = re.sub(r"^sha256:", "", checksum)



        tar.addfile(layer_tarinfo, from_image.tar.extractfile(layer_tarinfo))

        path = layer_tarinfo.path

        size = layer_tarinfo.size



        print("Adding base layer", num, "from", path, file=sys.stderr)

        yield LayerInfo(size=size, checksum=checksum, path=path, paths=[path])



    from_image.tar.close()





def overlay_base_config(from_image, final_config):

    """

    Overlays the final image 'config' JSON on top of selected defaults from the

    base image 'config' JSON.



    from_image: 'FromImage' object with references to the loaded base image.

    final_config: 'dict' object of the final image 'config' JSON.

    """

    if from_image is None:

        return final_config



    base_config = from_image.image_json["config"]



    # Preserve environment from base image

    final_env = base_config.get("Env", []) + final_config.get("Env", [])

    if final_env:

        final_config["Env"] = final_env

    return final_config





def add_layer_dir(tar, paths, store_dir, mtime):

    """

    Appends given store paths to a TarFile object as a new layer.
@@ -248,17 +324,21 @@ def main(): 
    mtime = int(created.timestamp())

    store_dir = conf["store_dir"]



    from_image = load_from_image(conf["from_image"])



    with tarfile.open(mode="w|", fileobj=sys.stdout.buffer) as tar:

        layers = []

        for num, store_layer in enumerate(conf["store_layers"]):

            print(

              "Creating layer", num,

              "from paths:", store_layer,

        layers.extend(add_base_layers(tar, from_image))



        start = len(layers) + 1

        for num, store_layer in enumerate(conf["store_layers"], start=start):

            print("Creating layer", num, "from paths:", store_layer,

                  file=sys.stderr)

            info = add_layer_dir(tar, store_layer, store_dir, mtime=mtime)

            layers.append(info)



        print("Creating the customisation layer...", file=sys.stderr)

        print("Creating layer", len(layers) + 1, "with customisation...",

              file=sys.stderr)

        layers.append(

          add_customisation_layer(

            tar,
@@ -273,7 +353,7 @@ def main(): 
            "created": datetime.isoformat(created),

            "architecture": conf["architecture"],

            "os": "linux",

            "config": conf["config"],

            "config": overlay_base_config(from_image, conf["config"]),

            "rootfs": {

                "diff_ids": [f"sha256:{layer.checksum}" for layer in layers],

                "type": "layers",