Commit 4b603ad9 authored by Joel Höner's avatar Joel Höner
Browse files

dockerTools: configurable compression schema 

This commit adds support for swapping out the compression algorithm
used in all major docker-tools commands that generate images. The
default algorithm remains unchanged (gzip).
parent af435645

doc/build-helpers/images/dockertools.section.md

+7 −0
Original line number Diff line number Diff line
@@ -178,6 +178,13 @@ Similarly, if you encounter errors similar to `Error_Protocol ("certificate has 


  _Default value:_ 0.



`compressor` (String; _optional_)



: Selects the algorithm used to compress the image.



  _Default value:_ `"gz"`.\

  _Possible values:_ `"none"`, `"gz"`, `"zstd"`.



`contents` **DEPRECATED**



: This attribute is deprecated, and users are encouraged to use `copyToRoot` instead.

nixos/tests/docker-tools.nix

+21 −0
Original line number Diff line number Diff line
@@ -128,6 +128,15 @@ in { 
        docker.succeed("docker images --format '{{.Tag}}' | grep -F '${examples.nixLayered.imageTag}'")

        docker.succeed("docker rmi ${examples.nixLayered.imageName}")



    with subtest("Check that images with alternative compression schemas load"):

        docker.succeed(

            "docker load --input='${examples.bashZstdCompressed}'",

            "docker rmi ${examples.bashZstdCompressed.imageName}",

        )

        docker.succeed(

            "docker load --input='${examples.bashUncompressed}'",

            "docker rmi ${examples.bashUncompressed.imageName}",

        )



    with subtest(

        "Check if the nix store is correctly initialized by listing "
@@ -449,6 +458,18 @@ in { 
            "docker run --rm ${examples.layeredImageWithFakeRootCommands.imageName} /hello/bin/layeredImageWithFakeRootCommands-hello"

        )



    with subtest("mergeImage correctly deals with varying compression schemas in inputs"):

        docker.succeed("docker load --input='${examples.mergeVaryingCompressor}'")



        for sub_image, tag in [

            ("${examples.redis.imageName}", "${examples.redis.imageTag}"),

            ("${examples.bashUncompressed.imageName}", "${examples.bashUncompressed.imageTag}"),

            ("${examples.bashZstdCompressed.imageName}", "${examples.bashZstdCompressed.imageTag}"),

        ]:

            docker.succeed(f"docker images --format '{{{{.Repository}}}}-{{{{.Tag}}}}' | grep -F '{sub_image}-{tag}'")

            docker.succeed(f"docker rmi {sub_image}")





    with subtest("exportImage produces a valid tarball"):

        docker.succeed(

            "tar -tf ${examples.exportBash} | grep '\./bin/bash' > /dev/null"

pkgs/build-support/docker/default.nix

+65 −14
Original line number Diff line number Diff line
@@ -8,6 +8,7 @@ 
, proot

, fakeNss

, fakeroot

, file

, go

, jq

, jshon
@@ -34,6 +35,7 @@ 
, writeText

, writeTextDir

, writePython3

, zstd

}:



let
@@ -76,6 +78,30 @@ let 
  # mapping from the go package.

  defaultArchitecture = go.GOARCH;



  compressors = {

    none = {

      ext = "";

      nativeInputs = [ ];

      compress = "cat";

      decompress = "cat";

    };

    gz = {

      ext = ".gz";

      nativeInputs = [ pigz ];

      compress = "pigz -p$NIX_BUILD_CORES -nTR";

      decompress = "pigz -d -p$NIX_BUILD_CORES";

    };

    zstd = {

      ext = ".zst";

      nativeInputs = [ zstd ];

      compress = "zstd -T$NIX_BUILD_CORES";

      decompress = "zstd -d -T$NIX_BUILD_CORES";

    };

  };



  compressorForImage = compressor: imageName: compressors.${compressor} or

    (throw "in docker image ${imageName}: compressor must be one of: [${toString builtins.attrNames compressors}]");



in

rec {

  examples = callPackage ./examples.nix {
@@ -487,16 +513,17 @@ rec { 
      '';

    };



  buildLayeredImage = lib.makeOverridable ({ name, ... }@args:

  buildLayeredImage = lib.makeOverridable ({ name, compressor ? "gz", ... }@args:

    let

      stream = streamLayeredImage args;

      compress = compressorForImage compressor name;

    in

    runCommand "${baseNameOf name}.tar.gz"

    runCommand "${baseNameOf name}.tar${compress.ext}"

      {

        inherit (stream) imageName;

        passthru = { inherit (stream) imageTag; };

        nativeBuildInputs = [ pigz ];

      } "${stream} | pigz -nTR > $out"

        nativeBuildInputs = compress.nativeInputs;

      } "${stream} | ${compress.compress} > $out"

  );



  # 1. extract the base image
@@ -539,6 +566,8 @@ rec { 
      buildVMMemorySize ? 512

    , # Time of creation of the image.

      created ? "1970-01-01T00:00:01Z"

    , # Compressor to use. One of: none, gz, zstd.

      compressor ? "gz"

    , # Deprecated.

      contents ? null

    ,
@@ -574,6 +603,8 @@ rec { 
        in

        if created == "now" then impure else pure;



      compress = compressorForImage compressor name;



      layer =

        if runAsRoot == null

        then
@@ -590,9 +621,9 @@ rec { 
              extraCommands;

            copyToRoot = rootContents;

          };

      result = runCommand "docker-image-${baseName}.tar.gz"

      result = runCommand "docker-image-${baseName}.tar${compress.ext}"

        {

          nativeBuildInputs = [ jshon pigz jq moreutils ];

          nativeBuildInputs = [ jshon jq moreutils ] ++ compress.nativeInputs;

          # Image name must be lowercase

          imageName = lib.toLower name;

          imageTag = lib.optionalString (tag != null) tag;
@@ -746,7 +777,7 @@ rec { 
        chmod -R a-w image



        echo "Cooking the image..."

        tar -C image --hard-dereference --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=0 --group=0 --xform s:'^./':: -c . | pigz -nTR > $out

        tar -C image --hard-dereference --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=0 --group=0 --xform s:'^./':: -c . | ${compress.compress} > $out



        echo "Finished."

      '';
@@ -761,16 +792,28 @@ rec { 
  mergeImages = images: runCommand "merge-docker-images"

    {

      inherit images;

      nativeBuildInputs = [ pigz jq ];

      nativeBuildInputs = [ file jq ]

        ++ compressors.none.nativeInputs

        ++ compressors.gz.nativeInputs

        ++ compressors.zstd.nativeInputs;

    } ''

    mkdir image inputs

    # Extract images

    repos=()

    manifests=()

    last_image_mime="application/gzip"

    for item in $images; do

      name=$(basename $item)

      mkdir inputs/$name

      tar -I pigz -xf $item -C inputs/$name



      last_image_mime=$(file --mime-type -b $item)

      case $last_image_mime in

        "application/x-tar") ${compressors.none.decompress};;

        "application/zstd") ${compressors.zstd.decompress};;

        "application/gzip") ${compressors.gz.decompress};;

        *) echo "error: unexpected layer type $last_image_mime" >&2; exit 1;;

      esac < $item | tar -xC inputs/$name



      if [ -f inputs/$name/repositories ]; then

        repos+=(inputs/$name/repositories)

      fi
@@ -787,7 +830,14 @@ rec { 
    mv repositories image/repositories

    mv manifest.json image/manifest.json

    # Create tarball and gzip

    tar -C image --hard-dereference --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=0 --group=0 --xform s:'^./':: -c . | pigz -nTR > $out

    tar -C image --hard-dereference --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=0 --group=0 --xform s:'^./':: -c . | (

      case $last_image_mime in

        "application/x-tar") ${compressors.none.compress};;

        "application/zstd") ${compressors.zstd.compress};;

        "application/gzip") ${compressors.gz.compress};;

        # `*)` not needed; already checked.

      esac

    ) > $out

  '';
@@ -1238,14 +1288,15 @@ rec { 
      };



  # Wrapper around streamNixShellImage to build an image from the result

  buildNixShellImage = { drv, ... }@args:

  buildNixShellImage = { drv, compressor ? "gz", ... }@args:

    let

      stream = streamNixShellImage args;

      compress = compressorForImage compressor drv.name;

    in

    runCommand "${drv.name}-env.tar.gz"

    runCommand "${drv.name}-env.tar${compress.ext}"

      {

        inherit (stream) imageName;

        passthru = { inherit (stream) imageTag; };

        nativeBuildInputs = [ pigz ];

      } "${stream} | pigz -nTR > $out";

        nativeBuildInputs = compress.nativeInputs;

      } "${stream} | ${compress.compress} > $out";

}

pkgs/build-support/docker/examples.nix

+22 −0
Original line number Diff line number Diff line
@@ -480,6 +480,22 @@ rec { 
    layerC = layerOnTopOf layerB "c";

  in layerC;



  bashUncompressed = pkgs.dockerTools.buildImage {

    name = "bash-uncompressed";

    tag = "latest";

    compressor = "none";

    # Not recommended. Use `buildEnv` between copy and packages to avoid file duplication.

    copyToRoot = pkgs.bashInteractive;

  };



  bashZstdCompressed = pkgs.dockerTools.buildImage {

    name = "bash-zstd";

    tag = "latest";

    compressor = "zstd";

    # Not recommended. Use `buildEnv` between copy and packages to avoid file duplication.

    copyToRoot = pkgs.bashInteractive;

  };



  # buildImage without explicit tag

  bashNoTag = pkgs.dockerTools.buildImage {

    name = "bash-no-tag";
@@ -614,6 +630,12 @@ rec { 
    layeredImageWithFakeRootCommands

  ];



  mergeVaryingCompressor = pkgs.dockerTools.mergeImages [

    redis

    bashUncompressed

    bashZstdCompressed

  ];



  helloOnRoot = pkgs.dockerTools.streamLayeredImage {

    name = "hello";

    tag = "latest";