Merge pull request #275180 from rorosen/extend-k3s-module (72249a0d) · Commits · nix / nixpkgs

nixos/modules/services/cluster/k3s/default.nix

+241 −0

Original line number	Diff line number	Diff line
		@@ -17,6 +17,109 @@ let
		]
		++ config
		) instruction;

		manifestDir = "/var/lib/rancher/k3s/server/manifests";
		chartDir = "/var/lib/rancher/k3s/server/static/charts";
		imageDir = "/var/lib/rancher/k3s/agent/images";

		manifestModule =
		let
		mkTarget =
		name: if (lib.hasSuffix ".yaml" name \|\| lib.hasSuffix ".yml" name) then name else name + ".yaml";
		in
		lib.types.submodule (
		{
		name,
		config,
		options,
		...
		}:
		{
		options = {
		enable = lib.mkOption {
		type = lib.types.bool;
		default = true;
		description = "Whether this manifest file should be generated.";
		};

		target = lib.mkOption {
		type = lib.types.nonEmptyStr;
		example = lib.literalExpression "manifest.yaml";
		description = ''
		Name of the symlink (relative to {file}`${manifestDir}`).
		Defaults to the attribute name.
		'';
		};

		content = lib.mkOption {
		type = with lib.types; nullOr (either attrs (listOf attrs));
		default = null;
		description = ''
		Content of the manifest file. A single attribute set will
		generate a single document YAML file. A list of attribute sets
		will generate multiple documents separated by `---` in a single
		YAML file.
		'';
		};

		source = lib.mkOption {
		type = lib.types.path;
		example = lib.literalExpression "./manifests/app.yaml";
		description = ''
		Path of the source `.yaml` file.
		'';
		};
		};

		config = {
		target = lib.mkDefault (mkTarget name);
		source = lib.mkIf (config.content != null) (
		let
		name' = "k3s-manifest-" + builtins.baseNameOf name;
		docName = "k3s-manifest-doc-" + builtins.baseNameOf name;
		yamlDocSeparator = builtins.toFile "yaml-doc-separator" "\n---\n";
		mkYaml = name: x: (pkgs.formats.yaml { }).generate name x;
		mkSource =
		value:
		if builtins.isList value then
		pkgs.concatText name' (
		lib.concatMap (x: [
		yamlDocSeparator
		(mkYaml docName x)
		]) value
		)
		else
		mkYaml name' value;
		in
		lib.mkDerivedConfig options.content mkSource
		);
		};
		}
		);

		enabledManifests = with builtins; filter (m: m.enable) (attrValues cfg.manifests);
		linkManifestEntry = m: "${pkgs.coreutils-full}/bin/ln -sfn ${m.source} ${manifestDir}/${m.target}";
		linkImageEntry = image: "${pkgs.coreutils-full}/bin/ln -sfn ${image} ${imageDir}/${image.name}";
		linkChartEntry =
		let
		mkTarget = name: if (lib.hasSuffix ".tgz" name) then name else name + ".tgz";
		in
		name: value:
		"${pkgs.coreutils-full}/bin/ln -sfn ${value} ${chartDir}/${mkTarget (builtins.baseNameOf name)}";

		activateK3sContent = pkgs.writeShellScript "activate-k3s-content" ''
		${lib.optionalString (
		builtins.length enabledManifests > 0
		) "${pkgs.coreutils-full}/bin/mkdir -p ${manifestDir}"}
		${lib.optionalString (cfg.charts != { }) "${pkgs.coreutils-full}/bin/mkdir -p ${chartDir}"}
		${lib.optionalString (
		builtins.length cfg.images > 0
		) "${pkgs.coreutils-full}/bin/mkdir -p ${imageDir}"}

		${builtins.concatStringsSep "\n" (map linkManifestEntry enabledManifests)}
		${builtins.concatStringsSep "\n" (lib.mapAttrsToList linkChartEntry cfg.charts)}
		${builtins.concatStringsSep "\n" (map linkImageEntry cfg.images)}
		'';
		in
		{
		imports = [ (removeOption [ "docker" ] "k3s docker option is no longer supported.") ];
		@@ -127,11 +230,148 @@ in
		default = null;
		description = "File path containing the k3s YAML config. This is useful when the config is generated (for example on boot).";
		};

		manifests = mkOption {
		type = types.attrsOf manifestModule;
		default = { };
		example = lib.literalExpression ''
		deployment.source = ../manifests/deployment.yaml;
		my-service = {
		enable = false;
		target = "app-service.yaml";
		content = {
		apiVersion = "v1";
		kind = "Service";
		metadata = {
		name = "app-service";
		};
		spec = {
		selector = {
		"app.kubernetes.io/name" = "MyApp";
		};
		ports = [
		{
		name = "name-of-service-port";
		protocol = "TCP";
		port = 80;
		targetPort = "http-web-svc";
		}
		];
		};
		}
		};

		nginx.content = [
		{
		apiVersion = "v1";
		kind = "Pod";
		metadata = {
		name = "nginx";
		labels = {
		"app.kubernetes.io/name" = "MyApp";
		};
		};
		spec = {
		containers = [
		{
		name = "nginx";
		image = "nginx:1.14.2";
		ports = [
		{
		containerPort = 80;
		name = "http-web-svc";
		}
		];
		}
		];
		};
		}
		{
		apiVersion = "v1";
		kind = "Service";
		metadata = {
		name = "nginx-service";
		};
		spec = {
		selector = {
		"app.kubernetes.io/name" = "MyApp";
		};
		ports = [
		{
		name = "name-of-service-port";
		protocol = "TCP";
		port = 80;
		targetPort = "http-web-svc";
		}
		];
		};
		}
		];
		'';
		description = ''
		Auto-deploying manifests that are linked to {file}`${manifestDir}` before k3s starts.
		Note that deleting manifest files will not remove or otherwise modify the resources
		it created. Please use the the `--disable` flag or `.skip` files to delete/disable AddOns,
		as mentioned in the [docs](https://docs.k3s.io/installation/packaged-components#disabling-manifests).
		This option only makes sense on server nodes (`role = server`).
		Read the [auto-deploying manifests docs](https://docs.k3s.io/installation/packaged-components#auto-deploying-manifests-addons)
		for further information.
		'';
		};

		charts = mkOption {
		type = with types; attrsOf (either path package);
		default = { };
		example = lib.literalExpression ''
		nginx = ../charts/my-nginx-chart.tgz;
		redis = ../charts/my-redis-chart.tgz;
		'';
		description = ''
		Packaged Helm charts that are linked to {file}`${chartDir}` before k3s starts.
		The attribute name will be used as the link target (relative to {file}`${chartDir}`).
		The specified charts will only be placed on the file system and made available to the
		Kubernetes APIServer from within the cluster, you may use the
		[k3s Helm controller](https://docs.k3s.io/helm#using-the-helm-controller)
		to deploy the charts. This option only makes sense on server nodes
		(`role = server`).
		'';
		};

		images = mkOption {
		type = with types; listOf package;
		default = [ ];
		example = lib.literalExpression ''
		[
		(pkgs.dockerTools.pullImage {
		imageName = "docker.io/bitnami/keycloak";
		imageDigest = "sha256:714dfadc66a8e3adea6609bda350345bd3711657b7ef3cf2e8015b526bac2d6b";
		sha256 = "0imblp0kw9vkcr7sp962jmj20fpmb3hvd3hmf4cs4x04klnq3k90";
		finalImageTag = "21.1.2-debian-11-r0";
		})
		]
		'';
		description = ''
		List of derivations that provide container images.
		All images are linked to {file}`${imageDir}` before k3s starts and consequently imported
		by the k3s agent. This option only makes sense on nodes with an enabled agent.
		'';
		};
		};

		# implementation

		config = mkIf cfg.enable {
		warnings =
		(lib.optional (cfg.role != "server" && cfg.manifests != { })
		"k3s: Auto deploying manifests are only installed on server nodes (role == server), they will be ignored by this node."
		)
		++ (lib.optional (cfg.role != "server" && cfg.charts != { })
		"k3s: Helm charts are only made available to the cluster on server nodes (role == server), they will be ignored by this node."
		)
		++ (lib.optional (cfg.disableAgent && cfg.images != [ ])
		"k3s: Images are only imported on nodes with an enabled agent, they will be ignored by this node"
		);

		assertions = [
		{
		assertion = cfg.role == "agent" -> (cfg.configPath != null \|\| cfg.serverAddr != "");
		@@ -178,6 +418,7 @@ in
		LimitCORE = "infinity";
		TasksMax = "infinity";
		EnvironmentFile = cfg.environmentFile;
		ExecStartPre = activateK3sContent;
		ExecStart = concatStringsSep " \\\n " (
		[ "${cfg.package}/bin/k3s ${cfg.role}" ]
		++ (optional cfg.clusterInit "--cluster-init")

nixos/tests/k3s/auto-deploy.nix

0 → 100644

+122 −0

Original line number	Diff line number	Diff line
		import ../make-test-python.nix (
		{
		pkgs,
		lib,
		k3s,
		...
		}:
		let
		pauseImageEnv = pkgs.buildEnv {
		name = "k3s-pause-image-env";
		paths = with pkgs; [
		tini
		(hiPrio coreutils)
		busybox
		];
		};
		pauseImage = pkgs.dockerTools.buildImage {
		name = "test.local/pause";
		tag = "local";
		copyToRoot = pauseImageEnv;
		config.Entrypoint = [
		"/bin/tini"
		"--"
		"/bin/sleep"
		"inf"
		];
		};
		helloImage = pkgs.dockerTools.buildImage {
		name = "test.local/hello";
		tag = "local";
		copyToRoot = pkgs.hello;
		config.Entrypoint = [ "${pkgs.hello}/bin/hello" ];
		};
		in
		{
		name = "${k3s.name}-auto-deploy";

		nodes.machine =
		{ pkgs, ... }:
		{
		environment.systemPackages = [ k3s ];

		# k3s uses enough resources the default vm fails.
		virtualisation.memorySize = 1536;
		virtualisation.diskSize = 4096;

		services.k3s.enable = true;
		services.k3s.role = "server";
		services.k3s.package = k3s;
		# Slightly reduce resource usage
		services.k3s.extraFlags = builtins.toString [
		"--disable coredns"
		"--disable local-storage"
		"--disable metrics-server"
		"--disable servicelb"
		"--disable traefik"
		"--pause-image test.local/pause:local"
		];
		services.k3s.images = [
		pauseImage
		helloImage
		];
		services.k3s.manifests = {
		absent = {
		enable = false;
		content = {
		apiVersion = "v1";
		kind = "Namespace";
		metadata.name = "absent";
		};
		};

		present = {
		target = "foo-namespace.yaml";
		content = {
		apiVersion = "v1";
		kind = "Namespace";
		metadata.name = "foo";
		};
		};

		hello.content = {
		apiVersion = "batch/v1";
		kind = "Job";
		metadata.name = "hello";
		spec = {
		template.spec = {
		containers = [
		{
		name = "hello";
		image = "test.local/hello:local";
		}
		];
		restartPolicy = "OnFailure";
		};
		};
		};
		};
		};

		testScript = ''
		start_all()

		machine.wait_for_unit("k3s")
		# check existence of the manifest files
		machine.fail("ls /var/lib/rancher/k3s/server/manifests/absent.yaml")
		machine.succeed("ls /var/lib/rancher/k3s/server/manifests/foo-namespace.yaml")
		machine.succeed("ls /var/lib/rancher/k3s/server/manifests/hello.yaml")

		# check if container images got imported
		machine.succeed("crictl img \| grep 'test\.local/pause'")
		machine.succeed("crictl img \| grep 'test\.local/hello'")

		# check if resources of manifests got created
		machine.wait_until_succeeds("kubectl get ns foo")
		machine.wait_until_succeeds("kubectl wait --for=condition=complete job/hello")
		machine.fail("kubectl get ns absent")

		machine.shutdown()
		'';
		}
		)

nixos/tests/k3s/default.nix

+2 −0

Original line number	Diff line number	Diff line
		@@ -19,4 +19,6 @@ in
		single-node = lib.mapAttrs (_: k3s: import ./single-node.nix { inherit system pkgs k3s; }) allK3s;
		# Run a multi-node k3s cluster and verify pod networking works across nodes
		multi-node = lib.mapAttrs (_: k3s: import ./multi-node.nix { inherit system pkgs k3s; }) allK3s;
		# Test wether container images are imported and auto deploying manifests work
		auto-deploy = lib.mapAttrs (_: k3s: import ./auto-deploy.nix { inherit system pkgs k3s; }) allK3s;
		}

Admin message