nixos/k3s: add `autoDeployCharts` option and use systemd-tmpfiles for content activation (#374017) (d07ebbab) · Commits · nix / nixpkgs

nixos/doc/manual/release-notes/rl-2505.section.md

+2 −0

Original line number	Diff line number	Diff line
		@@ -597,6 +597,8 @@
		- New options for the declarative configuration of the user space part of ALSA have been introduced under [hardware.alsa](options.html#opt-hardware.alsa.enable), including setting the default capture and playback device, defining sound card aliases and volume controls.
		Note: these are intended for users not running a sound server like PulseAudio or PipeWire, but having ALSA as their only sound system.

		- `services.k3s` now provides the `autoDeployCharts` option that allows to automatically deploy Helm charts via the k3s Helm controller.

		- Caddy can now be built with plugins by using `caddy.withPlugins`, a `passthru` function that accepts an attribute set as a parameter. The `plugins` argument represents a list of Caddy plugins, with each Caddy plugin being a versioned module. The `hash` argument represents the `vendorHash` of the resulting Caddy source code with the plugins added.

		Example:

nixos/modules/services/cluster/k3s/default.nix

+509 −164

Original line number	Diff line number	Diff line
		@@ -20,13 +20,289 @@ let
		chartDir = "/var/lib/rancher/k3s/server/static/charts";
		imageDir = "/var/lib/rancher/k3s/agent/images";
		containerdConfigTemplateFile = "/var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl";
		yamlFormat = pkgs.formats.yaml { };
		yamlDocSeparator = builtins.toFile "yaml-doc-separator" "\n---\n";
		# Manifests need a valid YAML suffix to be respected by k3s
		mkManifestTarget =
		name: if (lib.hasSuffix ".yaml" name \|\| lib.hasSuffix ".yml" name) then name else name + ".yaml";
		# Produces a list containing all duplicate manifest names
		duplicateManifests =
		with builtins;
		lib.intersectLists (attrNames cfg.autoDeployCharts) (attrNames cfg.manifests);
		# Produces a list containing all duplicate chart names
		duplicateCharts =
		with builtins;
		lib.intersectLists (attrNames cfg.autoDeployCharts) (attrNames cfg.charts);

		# Converts YAML -> JSON -> Nix
		fromYaml =
		path:
		with builtins;
		fromJSON (
		readFile (
		pkgs.runCommand "${path}-converted.json" { nativeBuildInputs = [ yq-go ]; } ''
		yq --no-colors --output-format json ${path} > $out
		''
		)
		);

		# Replace characters that are problematic in file names
		cleanHelmChartName =
		lib.replaceStrings
		[
		"/"
		":"
		]
		[
		"-"
		"-"
		];

		# Fetch a Helm chart from a public registry. This only supports a basic Helm pull.
		fetchHelm =
		{
		name,
		repo,
		version,
		hash ? lib.fakeHash,
		}:
		pkgs.runCommand (cleanHelmChartName "${lib.removePrefix "https://" repo}-${name}-${version}.tgz")
		{
		inherit (lib.fetchers.normalizeHash { } { inherit hash; }) outputHash outputHashAlgo;
		impureEnvVars = lib.fetchers.proxyImpureEnvVars;
		nativeBuildInputs = with pkgs; [
		kubernetes-helm
		cacert
		];
		}
		''
		export HOME="$PWD"
		helm repo add repository ${repo}
		helm pull repository/${name} --version ${version}
		mv ./*.tgz $out
		'';

		manifestModule =
		# Returns the path to a YAML manifest file
		mkExtraDeployManifest =
		x:
		# x is a derivation that provides a YAML file
		if lib.isDerivation x then
		x.outPath
		# x is an attribute set that needs to be converted to a YAML file
		else if builtins.isAttrs x then
		(yamlFormat.generate "extra-deploy-chart-manifest" x)
		# assume x is a path to a YAML file
		else
		x;

		# Generate a HelmChart custom resource.
		mkHelmChartCR =
		name: value:
		let
		mkTarget =
		name: if (lib.hasSuffix ".yaml" name \|\| lib.hasSuffix ".yml" name) then name else name + ".yaml";
		chartValues = if (lib.isPath value.values) then fromYaml value.values else value.values;
		# use JSON for values as it's a subset of YAML and understood by the k3s Helm controller
		valuesContent = builtins.toJSON chartValues;
		in
		lib.types.submodule (
		# merge with extraFieldDefinitions to allow setting advanced values and overwrite generated
		# values
		lib.recursiveUpdate {
		apiVersion = "helm.cattle.io/v1";
		kind = "HelmChart";
		metadata = {
		inherit name;
		namespace = "kube-system";
		};
		spec = {
		inherit valuesContent;
		inherit (value) targetNamespace createNamespace;
		chart = "https://%{KUBERNETES_API}%/static/charts/${name}.tgz";
		};
		} value.extraFieldDefinitions;

		# Generate a HelmChart custom resource together with extraDeploy manifests. This
		# generates possibly a multi document YAML file that the auto deploy mechanism of k3s
		# deploys.
		mkAutoDeployChartManifest = name: value: {
		# target is the final name of the link created for the manifest file
		target = mkManifestTarget name;
		inherit (value) enable package;
		# source is a store path containing the complete manifest file
		source = pkgs.concatText "auto-deploy-chart-${name}.yaml" (
		[
		(yamlFormat.generate "helm-chart-manifest-${name}.yaml" (mkHelmChartCR name value))
		]
		# alternate the YAML doc seperator (---) and extraDeploy manifests to create
		# multi document YAMLs
		++ (lib.concatMap (x: [
		yamlDocSeparator
		(mkExtraDeployManifest x)
		]) value.extraDeploy)
		);
		};

		autoDeployChartsModule = lib.types.submodule (
		{ config, ... }:
		{
		options = {
		enable = lib.mkOption {
		type = lib.types.bool;
		default = true;
		example = false;
		description = ''
		Whether to enable the installation of this Helm chart. Note that setting
		this option to `false` will not uninstall the chart from the cluster, if
		it was previously installed. Please use the the `--disable` flag or `.skip`
		files to delete/disable Helm charts, as mentioned in the
		[docs](https://docs.k3s.io/installation/packaged-components#disabling-manifests).
		'';
		};

		repo = lib.mkOption {
		type = lib.types.nonEmptyStr;
		example = "https://kubernetes.github.io/ingress-nginx";
		description = ''
		The repo of the Helm chart. Only has an effect if `package` is not set.
		The Helm chart is fetched during build time and placed as a `.tgz` archive on the
		filesystem.
		'';
		};

		name = lib.mkOption {
		type = lib.types.nonEmptyStr;
		example = "ingress-nginx";
		description = ''
		The name of the Helm chart. Only has an effect if `package` is not set.
		The Helm chart is fetched during build time and placed as a `.tgz` archive on the
		filesystem.
		'';
		};

		version = lib.mkOption {
		type = lib.types.nonEmptyStr;
		example = "4.7.0";
		description = ''
		The version of the Helm chart. Only has an effect if `package` is not set.
		The Helm chart is fetched during build time and placed as a `.tgz` archive on the
		filesystem.
		'';
		};

		hash = lib.mkOption {
		type = lib.types.str;
		example = "sha256-ej+vpPNdiOoXsaj1jyRpWLisJgWo8EqX+Z5VbpSjsPA=";
		description = ''
		The hash of the packaged Helm chart. Only has an effect if `package` is not set.
		The Helm chart is fetched during build time and placed as a `.tgz` archive on the
		filesystem.
		'';
		};

		package = lib.mkOption {
		type = with lib.types; either path package;
		example = lib.literalExpression "../my-helm-chart.tgz";
		description = ''
		The packaged Helm chart. Overwrites the options `repo`, `name`, `version`
		and `hash` in case of conflicts.
		'';
		};

		targetNamespace = lib.mkOption {
		type = lib.types.nonEmptyStr;
		default = "default";
		example = "kube-system";
		description = "The namespace in which the Helm chart gets installed.";
		};

		createNamespace = lib.mkOption {
		type = lib.types.bool;
		default = false;
		example = true;
		description = "Whether to create the target namespace if not present.";
		};

		values = lib.mkOption {
		type = with lib.types; either path attrs;
		default = { };
		example = {
		replicaCount = 3;
		hostName = "my-host";
		server = {
		name = "nginx";
		port = 80;
		};
		};
		description = ''
		Override default chart values via Nix expressions. This is equivalent to setting
		values in a `values.yaml` file.

		WARNING: The values (including secrets!) specified here are exposed unencrypted
		in the world-readable nix store.
		'';
		};

		extraDeploy = lib.mkOption {
		type = with lib.types; listOf (either path attrs);
		default = [ ];
		example = lib.literalExpression ''
		[
		../manifests/my-extra-deployment.yaml
		{
		apiVersion = "v1";
		kind = "Service";
		metadata = {
		name = "app-service";
		};
		spec = {
		selector = {
		"app.kubernetes.io/name" = "MyApp";
		};
		ports = [
		{
		name = "name-of-service-port";
		protocol = "TCP";
		port = 80;
		targetPort = "http-web-svc";
		}
		];
		};
		}
		];
		'';
		description = "List of extra Kubernetes manifests to deploy with this Helm chart.";
		};

		extraFieldDefinitions = lib.mkOption {
		inherit (yamlFormat) type;
		default = { };
		example = {
		spec = {
		bootstrap = true;
		helmVersion = "v2";
		backOffLimit = 3;
		jobImage = "custom-helm-controller:v0.0.1";
		};
		};
		description = ''
		Extra HelmChart field definitions that are merged with the rest of the HelmChart
		custom resource. This can be used to set advanced fields or to overwrite
		generated fields. See https://docs.k3s.io/helm#helmchart-field-definitions
		for possible fields.
		'';
		};
		};

		config.package = lib.mkDefault (fetchHelm {
		inherit (config)
		repo
		name
		version
		hash
		;
		});
		}
		);

		manifestModule = lib.types.submodule (
		{
		name,
		config,
		@@ -43,7 +319,7 @@ let

		target = lib.mkOption {
		type = lib.types.nonEmptyStr;
		example = lib.literalExpression "manifest.yaml";
		example = "manifest.yaml";
		description = ''
		Name of the symlink (relative to {file}`${manifestDir}`).
		Defaults to the attribute name.
		@@ -71,59 +347,28 @@ let
		};

		config = {
		target = lib.mkDefault (mkTarget name);
		target = lib.mkDefault (mkManifestTarget name);
		source = lib.mkIf (config.content != null) (
		let
		name' = "k3s-manifest-" + builtins.baseNameOf name;
		docName = "k3s-manifest-doc-" + builtins.baseNameOf name;
		yamlDocSeparator = builtins.toFile "yaml-doc-separator" "\n---\n";
		mkYaml = name: x: (pkgs.formats.yaml { }).generate name x;
		mkSource =
		value:
		if builtins.isList value then
		pkgs.concatText name' (
		lib.concatMap (x: [
		yamlDocSeparator
		(mkYaml docName x)
		(yamlFormat.generate docName x)
		]) value
		)
		else
		mkYaml name' value;
		yamlFormat.generate name' value;
		in
		lib.mkDerivedConfig options.content mkSource
		);
		};
		}
		);

		enabledManifests = lib.filter (m: m.enable) (lib.attrValues cfg.manifests);
		linkManifestEntry = m: "${pkgs.coreutils-full}/bin/ln -sfn ${m.source} ${manifestDir}/${m.target}";
		linkImageEntry = image: "${pkgs.coreutils-full}/bin/ln -sfn ${image} ${imageDir}/${image.name}";
		linkChartEntry =
		let
		mkTarget = name: if (lib.hasSuffix ".tgz" name) then name else name + ".tgz";
		in
		name: value:
		"${pkgs.coreutils-full}/bin/ln -sfn ${value} ${chartDir}/${mkTarget (builtins.baseNameOf name)}";

		activateK3sContent = pkgs.writeShellScript "activate-k3s-content" ''
		${lib.optionalString (
		builtins.length enabledManifests > 0
		) "${pkgs.coreutils-full}/bin/mkdir -p ${manifestDir}"}
		${lib.optionalString (cfg.charts != { }) "${pkgs.coreutils-full}/bin/mkdir -p ${chartDir}"}
		${lib.optionalString (
		builtins.length cfg.images > 0
		) "${pkgs.coreutils-full}/bin/mkdir -p ${imageDir}"}

		${builtins.concatStringsSep "\n" (map linkManifestEntry enabledManifests)}
		${builtins.concatStringsSep "\n" (lib.mapAttrsToList linkChartEntry cfg.charts)}
		${builtins.concatStringsSep "\n" (map linkImageEntry cfg.images)}

		${lib.optionalString (cfg.containerdConfigTemplate != null) ''
		mkdir -p $(dirname ${containerdConfigTemplateFile})
		${pkgs.coreutils-full}/bin/ln -sfn ${pkgs.writeText "config.toml.tmpl" cfg.containerdConfigTemplate} ${containerdConfigTemplateFile}
		''}
		'';
		in
		{
		imports = [ (removeOption [ "docker" ] "k3s docker option is no longer supported.") ];
		@@ -242,6 +487,7 @@ in
		type = lib.types.attrsOf manifestModule;
		default = { };
		example = lib.literalExpression ''
		{
		deployment.source = ../manifests/deployment.yaml;
		my-service = {
		enable = false;
		@@ -265,7 +511,7 @@ in
		}
		];
		};
		}
		};
		};

		nginx.content = [
		@@ -314,6 +560,7 @@ in
		};
		}
		];
		};
		'';
		description = ''
		Auto-deploying manifests that are linked to {file}`${manifestDir}` before k3s starts.
		@@ -337,10 +584,9 @@ in
		Packaged Helm charts that are linked to {file}`${chartDir}` before k3s starts.
		The attribute name will be used as the link target (relative to {file}`${chartDir}`).
		The specified charts will only be placed on the file system and made available to the
		Kubernetes APIServer from within the cluster, you may use the
		[k3s Helm controller](https://docs.k3s.io/helm#using-the-helm-controller)
		to deploy the charts. This option only makes sense on server nodes
		(`role = server`).
		Kubernetes APIServer from within the cluster. See the [](#opt-services.k3s.autoDeployCharts)
		option and the [k3s Helm controller docs](https://docs.k3s.io/helm#using-the-helm-controller)
		to deploy Helm charts. This option only makes sense on server nodes (`role = server`).
		'';
		};

		@@ -450,6 +696,53 @@ in
		set the `clientConnection.kubeconfig` if you want to use `extraKubeProxyConfig`.
		'';
		};

		autoDeployCharts = lib.mkOption {
		type = lib.types.attrsOf autoDeployChartsModule;
		apply = lib.mapAttrs mkAutoDeployChartManifest;
		default = { };
		example = lib.literalExpression ''
		{
		harbor = {
		name = "harbor";
		repo = "https://helm.goharbor.io";
		version = "1.14.0";
		hash = "sha256-fMP7q1MIbvzPGS9My91vbQ1d3OJMjwc+o8YE/BXZaYU=";
		values = {
		existingSecretAdminPassword = "harbor-admin";
		expose = {
		tls = {
		enabled = true;
		certSource = "secret";
		secret.secretName = "my-tls-secret";
		};
		ingress = {
		hosts.core = "example.com";
		className = "nginx";
		};
		};
		};
		};

		custom-chart = {
		package = ../charts/my-chart.tgz;
		values = ../values/my-values.yaml;
		extraFieldDefinitions = {
		spec.timeout = "60s";
		};
		};
		}
		'';
		description = ''
		Auto deploying Helm charts that are installed by the k3s Helm controller. Avoid to use
		attribute names that are also used in the [](#opt-services.k3s.manifests) and
		[](#opt-services.k3s.charts) options. Manifests with the same name will override
		auto deploying charts with the same name. Similiarly, charts with the same name will
		overwrite the Helm chart contained in auto deploying charts. This option only makes
		sense on server nodes (`role = server`). See the
		[k3s Helm documentation](https://docs.k3s.io/helm) for further information.
		'';
		};
		};

		# implementation
		@@ -462,6 +755,15 @@ in
		++ (lib.optional (cfg.role != "server" && cfg.charts != { })
		"k3s: Helm charts are only made available to the cluster on server nodes (role == server), they will be ignored by this node."
		)
		++ (lib.optional (cfg.role != "server" && cfg.autoDeployCharts != { })
		"k3s: Auto deploying Helm charts are only installed on server nodes (role == server), they will be ignored by this node."
		)
		++ (lib.optional (duplicateManifests != [ ])
		"k3s: The following auto deploying charts are overriden by manifests of the same name: ${toString duplicateManifests}."
		)
		++ (lib.optional (duplicateCharts != [ ])
		"k3s: The following auto deploying charts are overriden by charts of the same name: ${toString duplicateCharts}."
		)
		++ (lib.optional (
		cfg.disableAgent && cfg.images != [ ]
		) "k3s: Images are only imported on nodes with an enabled agent, they will be ignored by this node")
		@@ -486,6 +788,50 @@ in

		environment.systemPackages = [ config.services.k3s.package ];

		# Use systemd-tmpfiles to activate k3s content
		systemd.tmpfiles.settings."10-k3s" =
		let
		# Merge manifest with manifests generated from auto deploying charts, keep only enabled manifests
		enabledManifests = lib.filterAttrs (_: v: v.enable) (cfg.autoDeployCharts // cfg.manifests);
		# Merge charts with charts contained in enabled auto deploying charts
		helmCharts =
		(lib.concatMapAttrs (n: v: { ${n} = v.package; }) (
		lib.filterAttrs (_: v: v.enable) cfg.autoDeployCharts
		))
		// cfg.charts;
		# Make a systemd-tmpfiles rule for a manifest
		mkManifestRule = manifest: {
		name = "${manifestDir}/${manifest.target}";
		value = {
		"L+".argument = "${manifest.source}";
		};
		};
		# Ensure that all chart targets have a .tgz suffix
		mkChartTarget = name: if (lib.hasSuffix ".tgz" name) then name else name + ".tgz";
		# Make a systemd-tmpfiles rule for a chart
		mkChartRule = target: source: {
		name = "${chartDir}/${mkChartTarget target}";
		value = {
		"L+".argument = "${source}";
		};
		};
		# Make a systemd-tmpfiles rule for a container image
		mkImageRule = image: {
		name = "${imageDir}/${image.name}";
		value = {
		"L+".argument = "${image}";
		};
		};
		in
		(lib.mapAttrs' (_: v: mkManifestRule v) enabledManifests)
		// (lib.mapAttrs' (n: v: mkChartRule n v) helmCharts)
		// (builtins.listToAttrs (map mkImageRule cfg.images))
		// (lib.optionalAttrs (cfg.containerdConfigTemplate != null) {
		${containerdConfigTemplateFile} = {
		"L+".argument = "${pkgs.writeText "config.toml.tmpl" cfg.containerdConfigTemplate}";
		};
		});

		systemd.services.k3s =
		let
		kubeletParams =
		@@ -533,7 +879,6 @@ in
		LimitCORE = "infinity";
		TasksMax = "infinity";
		EnvironmentFile = cfg.environmentFile;
		ExecStartPre = activateK3sContent;
		ExecStart = lib.concatStringsSep " \\\n " (
		[ "${cfg.package}/bin/k3s ${cfg.role}" ]
		++ (lib.optional cfg.clusterInit "--cluster-init")

nixos/tests/k3s/auto-deploy-charts.nix

0 → 100644

+135 −0

File added.

Preview size limit exceeded, changes collapsed.

nixos/tests/k3s/default.nix

+3 −0

Original line number	Diff line number	Diff line
		@@ -11,6 +11,9 @@ in
		_: k3s: import ./airgap-images.nix { inherit system pkgs k3s; }
		) allK3s;
		auto-deploy = lib.mapAttrs (_: k3s: import ./auto-deploy.nix { inherit system pkgs k3s; }) allK3s;
		auto-deploy-charts = lib.mapAttrs (
		_: k3s: import ./auto-deploy-charts.nix { inherit system pkgs k3s; }
		) allK3s;
		containerd-config = lib.mapAttrs (
		_: k3s: import ./containerd-config.nix { inherit system pkgs k3s; }
		) allK3s;

nixos/tests/k3s/k3s-test-chart/Chart.yaml

0 → 100644

+24 −0

Original line number	Diff line number	Diff line
		apiVersion: v2
		name: k3s-test-chart
		description: A Helm chart that is used in k3s NixOS tests.

		# A chart can be either an 'application' or a 'library' chart.
		#
		# Application charts are a collection of templates that can be packaged into versioned archives
		# to be deployed.
		#
		# Library charts provide useful utilities or functions for the chart developer. They're included as
		# a dependency of application charts to inject those utilities and functions into the rendering
		# pipeline. Library charts do not define any templates and therefore cannot be deployed.
		type: application

		# This is the chart version. This version number should be incremented each time you make changes
		# to the chart and its templates, including the app version.
		# Versions are expected to follow Semantic Versioning (https://semver.org/)
		version: 0.1.0

		# This is the version number of the application being deployed. This version number should be
		# incremented each time you make changes to the application. Versions are not expected to
		# follow Semantic Versioning. They should reflect the version the application is using.
		# It is recommended to use it with quotes.
		appVersion: "1.16.0"

Admin message