Merge pull request #289584 from athre0z/docker-zstd (d2dfcfcf) · Commits · nix / nixpkgs

doc/build-helpers/images/dockertools.section.md

+7 −0

Original line number	Diff line number	Diff line
		@@ -178,6 +178,13 @@ Similarly, if you encounter errors similar to `Error_Protocol ("certificate has

		_Default value:_ 0.

		`compressor` (String; _optional_)

		: Selects the algorithm used to compress the image.

		_Default value:_ `"gz"`.\
		_Possible values:_ `"none"`, `"gz"`, `"zstd"`.

		`contents` DEPRECATED

		: This attribute is deprecated, and users are encouraged to use `copyToRoot` instead.

nixos/tests/docker-tools.nix

+21 −0

Original line number	Diff line number	Diff line
		@@ -155,6 +155,15 @@ in {
		docker.succeed("docker images --format '{{.Tag}}' \| grep -F '${examples.nixLayered.imageTag}'")
		docker.succeed("docker rmi ${examples.nixLayered.imageName}")

		with subtest("Check that images with alternative compression schemas load"):
		docker.succeed(
		"docker load --input='${examples.bashZstdCompressed}'",
		"docker rmi ${examples.bashZstdCompressed.imageName}",
		)
		docker.succeed(
		"docker load --input='${examples.bashUncompressed}'",
		"docker rmi ${examples.bashUncompressed.imageName}",
		)

		with subtest(
		"Check if the nix store is correctly initialized by listing "
		@@ -476,6 +485,18 @@ in {
		"docker run --rm ${examples.layeredImageWithFakeRootCommands.imageName} /hello/bin/layeredImageWithFakeRootCommands-hello"
		)

		with subtest("mergeImage correctly deals with varying compression schemas in inputs"):
		docker.succeed("docker load --input='${examples.mergeVaryingCompressor}'")

		for sub_image, tag in [
		("${examples.redis.imageName}", "${examples.redis.imageTag}"),
		("${examples.bashUncompressed.imageName}", "${examples.bashUncompressed.imageTag}"),
		("${examples.bashZstdCompressed.imageName}", "${examples.bashZstdCompressed.imageTag}"),
		]:
		docker.succeed(f"docker images --format '{{{{.Repository}}}}-{{{{.Tag}}}}' \| grep -F '{sub_image}-{tag}'")
		docker.succeed(f"docker rmi {sub_image}")


		with subtest("exportImage produces a valid tarball"):
		docker.succeed(
		"tar -tf ${examples.exportBash} \| grep '\./bin/bash' > /dev/null"

pkgs/build-support/docker/default.nix

+65 −14

Original line number	Diff line number	Diff line
		@@ -8,6 +8,7 @@
		, proot
		, fakeNss
		, fakeroot
		, file
		, go
		, jq
		, jshon
		@@ -34,6 +35,7 @@
		, writeText
		, writeTextDir
		, writePython3
		, zstd
		}:

		let
		@@ -76,6 +78,30 @@ let
		# mapping from the go package.
		defaultArchitecture = go.GOARCH;

		compressors = {
		none = {
		ext = "";
		nativeInputs = [ ];
		compress = "cat";
		decompress = "cat";
		};
		gz = {
		ext = ".gz";
		nativeInputs = [ pigz ];
		compress = "pigz -p$NIX_BUILD_CORES -nTR";
		decompress = "pigz -d -p$NIX_BUILD_CORES";
		};
		zstd = {
		ext = ".zst";
		nativeInputs = [ zstd ];
		compress = "zstd -T$NIX_BUILD_CORES";
		decompress = "zstd -d -T$NIX_BUILD_CORES";
		};
		};

		compressorForImage = compressor: imageName: compressors.${compressor} or
		(throw "in docker image ${imageName}: compressor must be one of: [${toString builtins.attrNames compressors}]");

		in
		rec {
		examples = callPackage ./examples.nix {
		@@ -487,16 +513,17 @@ rec {
		'';
		};

		buildLayeredImage = lib.makeOverridable ({ name, ... }@args:
		buildLayeredImage = lib.makeOverridable ({ name, compressor ? "gz", ... }@args:
		let
		stream = streamLayeredImage args;
		compress = compressorForImage compressor name;
		in
		runCommand "${baseNameOf name}.tar.gz"
		runCommand "${baseNameOf name}.tar${compress.ext}"
		{
		inherit (stream) imageName;
		passthru = { inherit (stream) imageTag; };
		nativeBuildInputs = [ pigz ];
		} "${stream} \| pigz -nTR > $out"
		nativeBuildInputs = compress.nativeInputs;
		} "${stream} \| ${compress.compress} > $out"
		);

		# 1. extract the base image
		@@ -539,6 +566,8 @@ rec {
		buildVMMemorySize ? 512
		, # Time of creation of the image.
		created ? "1970-01-01T00:00:01Z"
		, # Compressor to use. One of: none, gz, zstd.
		compressor ? "gz"
		, # Deprecated.
		contents ? null
		,
		@@ -574,6 +603,8 @@ rec {
		in
		if created == "now" then impure else pure;

		compress = compressorForImage compressor name;

		layer =
		if runAsRoot == null
		then
		@@ -590,9 +621,9 @@ rec {
		extraCommands;
		copyToRoot = rootContents;
		};
		result = runCommand "docker-image-${baseName}.tar.gz"
		result = runCommand "docker-image-${baseName}.tar${compress.ext}"
		{
		nativeBuildInputs = [ jshon pigz jq moreutils ];
		nativeBuildInputs = [ jshon jq moreutils ] ++ compress.nativeInputs;
		# Image name must be lowercase
		imageName = lib.toLower name;
		imageTag = lib.optionalString (tag != null) tag;
		@@ -746,7 +777,7 @@ rec {
		chmod -R a-w image

		echo "Cooking the image..."
		tar -C image --hard-dereference --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=0 --group=0 --xform s:'^./':: -c . \| pigz -nTR > $out
		tar -C image --hard-dereference --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=0 --group=0 --xform s:'^./':: -c . \| ${compress.compress} > $out

		echo "Finished."
		'';
		@@ -761,16 +792,28 @@ rec {
		mergeImages = images: runCommand "merge-docker-images"
		{
		inherit images;
		nativeBuildInputs = [ pigz jq ];
		nativeBuildInputs = [ file jq ]
		++ compressors.none.nativeInputs
		++ compressors.gz.nativeInputs
		++ compressors.zstd.nativeInputs;
		} ''
		mkdir image inputs
		# Extract images
		repos=()
		manifests=()
		last_image_mime="application/gzip"
		for item in $images; do
		name=$(basename $item)
		mkdir inputs/$name
		tar -I pigz -xf $item -C inputs/$name

		last_image_mime=$(file --mime-type -b $item)
		case $last_image_mime in
		"application/x-tar") ${compressors.none.decompress};;
		"application/zstd") ${compressors.zstd.decompress};;
		"application/gzip") ${compressors.gz.decompress};;
		*) echo "error: unexpected layer type $last_image_mime" >&2; exit 1;;
		esac < $item \| tar -xC inputs/$name

		if [ -f inputs/$name/repositories ]; then
		repos+=(inputs/$name/repositories)
		fi
		@@ -787,7 +830,14 @@ rec {
		mv repositories image/repositories
		mv manifest.json image/manifest.json
		# Create tarball and gzip
		tar -C image --hard-dereference --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=0 --group=0 --xform s:'^./':: -c . \| pigz -nTR > $out
		tar -C image --hard-dereference --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=0 --group=0 --xform s:'^./':: -c . \| (
		case $last_image_mime in
		"application/x-tar") ${compressors.none.compress};;
		"application/zstd") ${compressors.zstd.compress};;
		"application/gzip") ${compressors.gz.compress};;
		# `*)` not needed; already checked.
		esac
		) > $out
		'';


		@@ -1239,14 +1289,15 @@ rec {
		};

		# Wrapper around streamNixShellImage to build an image from the result
		buildNixShellImage = { drv, ... }@args:
		buildNixShellImage = { drv, compressor ? "gz", ... }@args:
		let
		stream = streamNixShellImage args;
		compress = compressorForImage compressor drv.name;
		in
		runCommand "${drv.name}-env.tar.gz"
		runCommand "${drv.name}-env.tar${compress.ext}"
		{
		inherit (stream) imageName;
		passthru = { inherit (stream) imageTag; };
		nativeBuildInputs = [ pigz ];
		} "${stream} \| pigz -nTR > $out";
		nativeBuildInputs = compress.nativeInputs;
		} "${stream} \| ${compress.compress} > $out";
		}

pkgs/build-support/docker/examples.nix

+22 −0

Original line number	Diff line number	Diff line
		@@ -480,6 +480,22 @@ rec {
		layerC = layerOnTopOf layerB "c";
		in layerC;

		bashUncompressed = pkgs.dockerTools.buildImage {
		name = "bash-uncompressed";
		tag = "latest";
		compressor = "none";
		# Not recommended. Use `buildEnv` between copy and packages to avoid file duplication.
		copyToRoot = pkgs.bashInteractive;
		};

		bashZstdCompressed = pkgs.dockerTools.buildImage {
		name = "bash-zstd";
		tag = "latest";
		compressor = "zstd";
		# Not recommended. Use `buildEnv` between copy and packages to avoid file duplication.
		copyToRoot = pkgs.bashInteractive;
		};

		# buildImage without explicit tag
		bashNoTag = pkgs.dockerTools.buildImage {
		name = "bash-no-tag";
		@@ -614,6 +630,12 @@ rec {
		layeredImageWithFakeRootCommands
		];

		mergeVaryingCompressor = pkgs.dockerTools.mergeImages [
		redis
		bashUncompressed
		bashZstdCompressed
		];

		helloOnRoot = pkgs.dockerTools.streamLayeredImage {
		name = "hello";
		tag = "latest";

Admin message