pkgs.dockertools.buildLayeredImage: customisable layering strategy (#122608)

    , includeStorePaths ? true

    , includeNixDB ? false

    , passthru ? {}

    ,

    , # Pipeline used to produce docker layers. If not set, popularity contest

      # algorithm is used. If set, maxLayers is ignored as the author of the

      # pipeline can use one of the available functions (like "limit_layers")

      # to control the amount of layers.

      # See: pkgs/build-support/flatten-references-graph/src/flatten_references_graph/pipe.py

      # for available functions, and it's test for how to use them.

      # WARNING!! this interface is highly experimental and subject to change.

      layeringPipeline ? null

    , # Enables debug logging for the layering pipeline.

      debug ? false

    }:

      assert

      (lib.assertMsg (maxLayers > 1)

      (lib.assertMsg (layeringPipeline == null -> maxLayers > 1)

        "the maxLayers argument of dockerTools.buildLayeredImage function must be greather than 1 (current value: ${toString maxLayers})");

      assert

      (lib.assertMsg (enableFakechroot -> !stdenv.hostPlatform.isDarwin) ''

          '';

        };

        closureRoots = lib.optionals includeStorePaths /* normally true */ (

          [ baseJson customisationLayer ]

        );

        overallClosure = writeText "closure" (lib.concatStringsSep " " closureRoots);

        # These derivations are only created as implementation details of docker-tools,

        # so they'll be excluded from the created images.

        unnecessaryDrvs = [ baseJson overallClosure customisationLayer ];

        layersJsonFile = buildPackages.dockerMakeLayers {

          inherit debug;

          closureRoots = optionals includeStorePaths [ baseJson customisationLayer ];

          excludePaths = [ baseJson customisationLayer ];

          pipeline =

            if layeringPipeline != null

            then layeringPipeline

            else import

              ./popularity-contest-layering-pipeline.nix

               { inherit lib jq runCommand; }

               { inherit fromImage maxLayers; }

          ;

        };

        conf = runCommand "${baseName}-conf.json"

          {

            inherit fromImage maxLayers created mtime uid gid uname gname;

            inherit fromImage created mtime uid gid uname gname layersJsonFile;

            imageName = lib.toLower name;

            preferLocalBuild = true;

            passthru.imageTag =

              then tag

              else

                lib.head (lib.strings.splitString "-" (baseNameOf (builtins.unsafeDiscardStringContext conf.outPath)));

            paths = buildPackages.referencesByPopularity overallClosure;

            nativeBuildInputs = [ jq ];

          } ''

          ${if (tag == null) then ''

              mtime="$(date -Iseconds -d "$mtime")"

          fi

          paths() {

            cat $paths ${lib.concatMapStringsSep " "

                           (path: "| (grep -v ${path} || true)")

                           unnecessaryDrvs}

          }

          # Compute the number of layers that are already used by a potential

          # 'fromImage' as well as the customization layer. Ensure that there is

          # still at least one layer available to store the image contents.

          usedLayers=0

          # subtract number of base image layers

          if [[ -n "$fromImage" ]]; then

            (( usedLayers += $(tar -xOf "$fromImage" manifest.json | jq '.[0].Layers | length') ))

          fi

          # one layer will be taken up by the customisation layer

          (( usedLayers += 1 ))

          if ! (( $usedLayers < $maxLayers )); then

            echo >&2 "Error: usedLayers $usedLayers layers to store 'fromImage' and" \

                      "'extraCommands', but only maxLayers=$maxLayers were" \

                      "allowed. At least 1 layer is required to store contents."

            exit 1

          fi

          availableLayers=$(( maxLayers - usedLayers ))

          # Create $maxLayers worth of Docker Layers, one layer per store path

          # unless there are more paths than $maxLayers. In that case, create

          # $maxLayers-1 for the most popular layers, and smush the remainaing

          # store paths in to one final layer.

          #

          # The following code is fiddly w.r.t. ensuring every layer is

          # created, and that no paths are missed. If you change the

          # following lines, double-check that your code behaves properly

          # when the number of layers equals:

          #      maxLayers-1, maxLayers, and maxLayers+1, 0

          paths |

            jq -sR '

              rtrimstr("\n") | split("\n")

                | (.[:$maxLayers-1] | map([.])) + [ .[$maxLayers-1:] ]

                | map(select(length > 0))

              ' \

              --argjson maxLayers "$availableLayers" > store_layers.json

          # The index on $store_layers is necessary because the --slurpfile

          # automatically reads the file as an array.

          cat ${baseJson} | jq '

          jq '

            . + {

              "store_dir": $store_dir,

              "from_image": $from_image,

            }

            ' --arg store_dir "${storeDir}" \

              --argjson from_image ${if fromImage == null then "null" else "'\"${fromImage}\"'"} \

              --slurpfile store_layers store_layers.json \

              --slurpfile store_layers "$layersJsonFile" \

              --arg customisation_layer ${customisationLayer} \

              --arg repo_tag "$imageName:$imageTag" \

              --arg created "$created" \

              --arg uid "$uid" \

              --arg gid "$gid" \

              --arg uname "$uname" \

              --arg gname "$gname" |

            tee $out

              --arg gname "$gname" \

              ${baseJson} \

                | tee $out

        '';

        result = runCommand "stream-${baseName}"

{

  coreutils,

  flattenReferencesGraph,

  lib,

  jq,

  runCommand,

}:

{

  closureRoots,

  excludePaths ? [ ],

  # This could be a path to (or a derivation producing a path to)

  # a json file containing the pipeline

  pipeline ? [ ],

  debug ? false,

}:

if closureRoots == [ ] then

  builtins.toFile "docker-layers-empty" "[]"

else

  runCommand "docker-layers"

    {

      __structuredAttrs = true;

      # graph, exclude_paths and pipeline are expected by the

      # flatten_references_graph executable.

      exportReferencesGraph.graph = closureRoots;

      exclude_paths = excludePaths;

      inherit pipeline;

      nativeBuildInputs = [

        coreutils

        flattenReferencesGraph

        jq

      ];

    }

    ''

      . .attrs.sh

      flatten_references_graph_arg=.attrs.json

      echo "pipeline: $pipeline"

      if jq -e '.pipeline | type == "string"' .attrs.json; then

        jq '. + { "pipeline": $pipeline[0] }' \

          --slurpfile pipeline "$pipeline" \

          .attrs.json > flatten_references_graph_arg.json

        flatten_references_graph_arg=flatten_references_graph_arg.json

      fi

      ${lib.optionalString debug "export DEBUG=True"}

      flatten_references_graph "$flatten_references_graph_arg" > ''${outputs[out]}

    ''

{

  lib,

  runCommand,

  jq,

}:

{

  maxLayers,

  fromImage ? null,

}:

runCommand "popularity-contest-layering-pipeline.json" { inherit maxLayers; } ''

  # Compute the number of layers that are already used by a potential

  # 'fromImage' as well as the customization layer. Ensure that there is

  # still at least one layer available to store the image contents.

  # one layer will be taken up by the customisation layer

  usedLayers=1

  ${lib.optionalString (fromImage != null) ''

    # subtract number of base image layers

    baseImageLayersCount=$(tar -xOf "${fromImage}" manifest.json | ${lib.getExe jq} '.[0].Layers | length')

    (( usedLayers += baseImageLayersCount ))

  ''}

  if ! (( $usedLayers < $maxLayers )); then

    echo >&2 "Error: usedLayers $usedLayers layers to store 'fromImage' and" \

              "'extraCommands', but only maxLayers=$maxLayers were" \

              "allowed. At least 1 layer is required to store contents."

    exit 1

  fi

  availableLayers=$(( maxLayers - usedLayers ))

  # Produce pipeline which uses popularity_contest algo.

  echo '[["popularity_contest"],["limit_layers",'$availableLayers']]' > $out

''

# Start this shell with:

# nix-shell path/to/root/of/nixpkgs -A flattenReferencesGraph.dev-shell

{

  mkShell,

  callPackage,

  python3Packages,

}:

let

  helpers = callPackage (import ./helpers.nix) { };

in

mkShell {

  inputsFrom = [ (callPackage (import ./package.nix) { }) ];

  buildInputs = [

    helpers.format

    helpers.lint

    helpers.unittest

    # This is needed to plot graphs when DEBUG_PLOT is set to True.

    python3Packages.pycairo

    # This can be used on linux to display the graphs.

    # On other platforms the image viewer needs to be set with

    # DEBUG_PLOT_IMAGE_VIEWER env var.

    # pkgs.gwenview

  ];

  shellHook = ''

    echo '

    **********************************************************************

    **********************************************************************

      Commands useful for development (should be executed from scr dir):

      format

        * formats all files in place using autopep8

      lint

        * lints all files using flake8

      unittest

        * runs all unit tests

          following env vars can be set to enable extra output in tests:

          - DEBUG=True - enable debug logging

          - DEBUG_PLOT=True - plot graphs processed by split_paths.py and

              subcomponent.py

          - DEBUG_PLOT_IMAGE_VIEWER=$PATH_OF_IMAGE_VIEWER_APP - app used to

              display plots (default: gwenview)

          - DEBUG_PLOT_SAVE_BASE_NAME=$SOME_NAME - if set, plots will be saved

              to files instead of displayed with image viewer

    **********************************************************************

    **********************************************************************

    '

  '';

}

{

  bash,

  writers,

  python3Packages,

}:

let

  writeCheckedBashBin =

    name:

    let

      interpreter = "${bash}/bin/bash";

    in

    writers.makeScriptWriter {

      inherit interpreter;

      check = "${interpreter} -n $1";

    } "/bin/${name}";

  # Helpers used during build/development.

  lint = writeCheckedBashBin "lint" ''

    ${python3Packages.flake8}/bin/flake8 --show-source ''${@}

  '';

  unittest = writeCheckedBashBin "unittest" ''

    if [ "$#" -eq 0 ]; then

      set -- discover -p '*_test.py'

    fi

    ${python3Packages.python}/bin/python -m unittest "''${@}"

  '';

  format = writeCheckedBashBin "format" ''

    ${python3Packages.autopep8}/bin/autopep8 -r -i . "''${@}"

  '';

in

{

  inherit format lint unittest;

}