Merge pull request #328385 from rorosen/k3s-kubelet-config (5f2a6de6) · Commits · nix / nixpkgs

nixos/modules/services/cluster/k3s/default.nix

+98 −35

Original line number	Diff line number	Diff line
		@@ -359,6 +359,53 @@ in
		by the k3s agent. This option only makes sense on nodes with an enabled agent.
		'';
		};

		gracefulNodeShutdown = {
		enable = lib.mkEnableOption ''
		graceful node shutdowns where the kubelet attempts to detect
		node system shutdown and terminates pods running on the node. See the
		[documentation](https://kubernetes.io/docs/concepts/cluster-administration/node-shutdown/#graceful-node-shutdown)
		for further information.
		'';

		shutdownGracePeriod = lib.mkOption {
		type = lib.types.nonEmptyStr;
		default = "30s";
		example = "1m30s";
		description = ''
		Specifies the total duration that the node should delay the shutdown by. This is the total
		grace period for pod termination for both regular and critical pods.
		'';
		};

		shutdownGracePeriodCriticalPods = lib.mkOption {
		type = lib.types.nonEmptyStr;
		default = "10s";
		example = "15s";
		description = ''
		Specifies the duration used to terminate critical pods during a node shutdown. This should be
		less than `shutdownGracePeriod`.
		'';
		};
		};

		extraKubeletConfig = lib.mkOption {
		type = with lib.types; attrsOf anything;
		default = { };
		example = {
		podsPerCore = 3;
		memoryThrottlingFactor = 0.69;
		containerLogMaxSize = "5Mi";
		};
		description = ''
		Extra configuration to add to the kubelet's configuration file. The subset of the kubelet's
		configuration that can be configured via a file is defined by the
		[KubeletConfiguration](https://kubernetes.io/docs/reference/config-api/kubelet-config.v1beta1/)
		struct. See the
		[documentation](https://kubernetes.io/docs/tasks/administer-cluster/kubelet-config-file/)
		for further information.
		'';
		};
		};

		# implementation
		@@ -397,7 +444,22 @@ in

		environment.systemPackages = [ config.services.k3s.package ];

		systemd.services.k3s = {
		systemd.services.k3s =
		let
		kubeletParams =
		(lib.optionalAttrs (cfg.gracefulNodeShutdown.enable) {
		inherit (cfg.gracefulNodeShutdown) shutdownGracePeriod shutdownGracePeriodCriticalPods;
		})
		// cfg.extraKubeletConfig;
		kubeletConfig = (pkgs.formats.yaml { }).generate "k3s-kubelet-config" (
		{
		apiVersion = "kubelet.config.k8s.io/v1beta1";
		kind = "KubeletConfiguration";
		}
		// kubeletParams
		);
		in
		{
		description = "k3s service";
		after = [
		"firewall.service"
		@@ -430,6 +492,7 @@ in
		++ (optional (cfg.token != "") "--token ${cfg.token}")
		++ (optional (cfg.tokenFile != null) "--token-file ${cfg.tokenFile}")
		++ (optional (cfg.configPath != null) "--config ${cfg.configPath}")
		++ (optional (kubeletParams != { }) "--kubelet-arg=config=${kubeletConfig}")
		++ (lib.flatten cfg.extraFlags)
		);
		};

nixos/tests/k3s/default.nix

+4 −1

Original line number	Diff line number	Diff line
		@@ -15,6 +15,9 @@ in
		inherit (pkgs) etcd;
		}
		) allK3s;
		single-node = lib.mapAttrs (_: k3s: import ./single-node.nix { inherit system pkgs k3s; }) allK3s;
		kubelet-config = lib.mapAttrs (
		_: k3s: import ./kubelet-config.nix { inherit system pkgs k3s; }
		) allK3s;
		multi-node = lib.mapAttrs (_: k3s: import ./multi-node.nix { inherit system pkgs k3s; }) allK3s;
		single-node = lib.mapAttrs (_: k3s: import ./single-node.nix { inherit system pkgs k3s; }) allK3s;
		}

nixos/tests/k3s/kubelet-config.nix

0 → 100644

+80 −0

Original line number	Diff line number	Diff line
		# A test that sets extra kubelet configuration and enables graceful node shutdown
		import ../make-test-python.nix (
		{
		pkgs,
		lib,
		k3s,
		...
		}:
		let
		nodeName = "test";
		shutdownGracePeriod = "1m13s";
		shutdownGracePeriodCriticalPods = "13s";
		podsPerCore = 3;
		memoryThrottlingFactor = 0.69;
		containerLogMaxSize = "5Mi";
		in
		{
		name = "${k3s.name}-kubelet-config";
		nodes.machine =
		{ pkgs, ... }:
		{
		environment.systemPackages = [ pkgs.jq ];

		# k3s uses enough resources the default vm fails.
		virtualisation.memorySize = 1536;
		virtualisation.diskSize = 4096;

		services.k3s = {
		enable = true;
		package = k3s;
		# Slightly reduce resource usage
		extraFlags = [
		"--disable coredns"
		"--disable local-storage"
		"--disable metrics-server"
		"--disable servicelb"
		"--disable traefik"
		"--node-name ${nodeName}"
		];
		gracefulNodeShutdown = {
		enable = true;
		inherit shutdownGracePeriod shutdownGracePeriodCriticalPods;
		};
		extraKubeletConfig = {
		inherit podsPerCore memoryThrottlingFactor containerLogMaxSize;
		};
		};
		};

		testScript = ''
		import json

		start_all()
		machine.wait_for_unit("k3s")
		# wait until the node is ready
		machine.wait_until_succeeds(r"""kubectl wait --for='jsonpath={.status.conditions[?(@.type=="Ready")].status}=True' nodes/${nodeName}""")
		# test whether the kubelet registered an inhibitor lock
		machine.succeed("systemd-inhibit --list --no-legend \| grep \"kubelet.k3s-server.shutdown\"")
		# run kubectl proxy in the background, close stdout through redirection to not wait for the command to finish
		machine.execute("kubectl proxy --address 127.0.0.1 --port=8001 >&2 &")
		machine.wait_until_succeeds("nc -z 127.0.0.1 8001")
		# get the kubeletconfig
		kubelet_config=json.loads(machine.succeed("curl http://127.0.0.1:8001/api/v1/nodes/${nodeName}/proxy/configz \| jq '.kubeletconfig'"))

		with subtest("Kubelet config values are set correctly"):
		assert kubelet_config["shutdownGracePeriod"] == "${shutdownGracePeriod}", \
		f"unexpected value for shutdownGracePeriod: {kubelet_config["shutdownGracePeriod"]}"
		assert kubelet_config["shutdownGracePeriodCriticalPods"] == "${shutdownGracePeriodCriticalPods}", \
		f"unexpected value for shutdownGracePeriodCriticalPods: {kubelet_config["shutdownGracePeriodCriticalPods"]}"
		assert kubelet_config["podsPerCore"] == ${toString podsPerCore}, \
		f"unexpected value for podsPerCore: {kubelet_config["podsPerCore"]}"
		assert kubelet_config["memoryThrottlingFactor"] == ${toString memoryThrottlingFactor}, \
		f"unexpected value for memoryThrottlingFactor: {kubelet_config["memoryThrottlingFactor"]}"
		assert kubelet_config["containerLogMaxSize"] == "${containerLogMaxSize}", \
		f"unexpected value for containerLogMaxSize: {kubelet_config["containerLogMaxSize"]}"
		'';

		meta.maintainers = lib.teams.k3s.members;
		}
		)