nixos/k3s: factor out generic functionality

  ...

}:

let

  cfg = config.services.k3s;

  removeOption =

    config: instruction:

    lib.mkRemovedOptionModule (

      [

        "services"

        "k3s"

      ]

      ++ config

    ) instruction;

  mkRancherModule =

    {

      # name used in paths/names, e.g. k3s

      name ? null,

      # extra flags to pass to the binary before user-defined extraFlags

      extraBinFlags ? [ ],

    }:

    let

      cfg = config.services.${name};

      manifestDir = "/var/lib/rancher/${name}/server/manifests";

      imageDir = "/var/lib/rancher/${name}/agent/images";

      containerdConfigTemplateFile = "/var/lib/rancher/${name}/agent/etc/containerd/config.toml.tmpl";

  manifestDir = "/var/lib/rancher/k3s/server/manifests";

  chartDir = "/var/lib/rancher/k3s/server/static/charts";

  imageDir = "/var/lib/rancher/k3s/agent/images";

  containerdConfigTemplateFile = "/var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl";

      yamlFormat = pkgs.formats.yaml { };

      yamlDocSeparator = builtins.toFile "yaml-doc-separator" "\n---\n";

  # Manifests need a valid YAML suffix to be respected by k3s

      # Manifests need a valid YAML suffix to be respected

      mkManifestTarget =

        name: if (lib.hasSuffix ".yaml" name || lib.hasSuffix ".yml" name) then name else name + ".yaml";

      # Produces a list containing all duplicate manifest names

      duplicateManifests = lib.intersectLists (builtins.attrNames cfg.autoDeployCharts) (

        builtins.attrNames cfg.manifests

      );

  # Produces a list containing all duplicate chart names

  duplicateCharts = lib.intersectLists (builtins.attrNames cfg.autoDeployCharts) (

    builtins.attrNames cfg.charts

  );

      # Converts YAML -> JSON -> Nix

      fromYaml =

        name: value:

        let

          chartValues = if (lib.isPath value.values) then fromYaml value.values else value.values;

      # use JSON for values as it's a subset of YAML and understood by the k3s Helm controller

          # use JSON for values as it's a subset of YAML and understood by the rancher Helm controller

          valuesContent = builtins.toJSON chartValues;

        in

        # merge with extraFieldDefinitions to allow setting advanced values and overwrite generated

        } value.extraFieldDefinitions;

      # Generate a HelmChart custom resource together with extraDeploy manifests. This

  # generates possibly a multi document YAML file that the auto deploy mechanism of k3s

      # generates possibly a multi document YAML file that the auto deploy mechanism

      # deploys.

      mkAutoDeployChartManifest = name: value: {

        # target is the final name of the link created for the manifest file

            target = lib.mkDefault (mkManifestTarget name);

            source = lib.mkIf (config.content != null) (

              let

            name' = "k3s-manifest-" + builtins.baseNameOf name;

            docName = "k3s-manifest-doc-" + builtins.baseNameOf name;

                name' = "${name}-manifest-" + builtins.baseNameOf name;

                docName = "${name}-manifest-doc-" + builtins.baseNameOf name;

                mkSource =

                  value:

                  if builtins.isList value then

      );

    in

    {

  imports = [ (removeOption [ "docker" ] "k3s docker option is no longer supported.") ];

      paths = { inherit manifestDir imageDir containerdConfigTemplateFile; };

      # interface

  options.services.k3s = {

    enable = lib.mkEnableOption "k3s";

    package = lib.mkPackageOption pkgs "k3s" { };

    role = lib.mkOption {

      description = ''

        Whether k3s should run as a server or agent.

        If it's a server:

        - By default it also runs workloads as an agent.

        - Starts by default as a standalone server using an embedded sqlite datastore.

        - Configure `clusterInit = true` to switch over to embedded etcd datastore and enable HA mode.

        - Configure `serverAddr` to join an already-initialized HA cluster.

      options = {

        enable = lib.mkEnableOption name;

        If it's an agent:

        package = lib.mkPackageOption pkgs name { };

        - `serverAddr` is required.

      '';

        role = lib.mkOption {

          description = "Whether ${name} should run as a server or agent.";

          default = "server";

          type = lib.types.enum [

            "server"

        serverAddr = lib.mkOption {

          type = lib.types.str;

      description = ''

        The k3s server to connect to.

        Servers and agents need to communicate each other. Read

        [the networking docs](https://rancher.com/docs/k3s/latest/en/installation/installation-requirements/#networking)

        to know how to configure the firewall.

      '';

          description = "The ${name} server to connect to, used to join a cluster.";

          example = "https://10.0.0.10:6443";

          default = "";

        };

    clusterInit = lib.mkOption {

      type = lib.types.bool;

      default = false;

      description = ''

        Initialize HA cluster using an embedded etcd datastore.

        If this option is `false` and `role` is `server`

        On a server that was using the default embedded sqlite backend,

        enabling this option will migrate to an embedded etcd DB.

        If an HA cluster using the embedded etcd datastore was already initialized,

        this option has no effect.

        This option only makes sense in a server that is not connecting to another server.

        If you are configuring an HA cluster with an embedded etcd,

        the 1st server must have `clusterInit = true`

        and other servers must connect to it using `serverAddr`.

      '';

    };

        token = lib.mkOption {

          type = lib.types.str;

          description = ''

        The k3s token to use when connecting to a server.

            The ${name} token to use when connecting to a server.

        WARNING: This option will expose store your token unencrypted world-readable in the nix store.

            WARNING: This option will expose your token unencrypted in the world-readable nix store.

            If this is undesired use the tokenFile option instead.

          '';

          default = "";

        tokenFile = lib.mkOption {

          type = lib.types.nullOr lib.types.path;

      description = "File path containing k3s token to use when connecting to the server.";

          description = "File path containing ${name} token to use when connecting to the server.";

          default = null;

        };

        extraFlags = lib.mkOption {

      description = "Extra flags to pass to the k3s command.";

          description = "Extra flags to pass to the ${name} command.";

          type = with lib.types; either str (listOf str);

          default = [ ];

          example = [

        "--disable traefik"

            "--etcd-expose-metrics"

            "--cluster-cidr 10.24.0.0/16"

          ];

        };

    disableAgent = lib.mkOption {

      type = lib.types.bool;

      default = false;

      description = "Only run the server. This option only makes sense for a server.";

    };

        environmentFile = lib.mkOption {

          type = lib.types.nullOr lib.types.path;

          description = ''

        File path containing environment variables for configuring the k3s service in the format of an EnvironmentFile. See {manpage}`systemd.exec(5)`.

            File path containing environment variables for configuring the ${name} service in the format of an EnvironmentFile. See {manpage}`systemd.exec(5)`.

          '';

          default = null;

        };

        configPath = lib.mkOption {

          type = lib.types.nullOr lib.types.path;

          default = null;

      description = "File path containing the k3s YAML config. This is useful when the config is generated (for example on boot).";

          description = "File path containing the ${name} YAML config. This is useful when the config is generated (for example on boot).";

        };

        manifests = lib.mkOption {

            };

          '';

          description = ''

        Auto-deploying manifests that are linked to {file}`${manifestDir}` before k3s starts.

            Auto-deploying manifests that are linked to {file}`${manifestDir}` before ${name} starts.

            Note that deleting manifest files will not remove or otherwise modify the resources

            it created. Please use the the `--disable` flag or `.skip` files to delete/disable AddOns,

            as mentioned in the [docs](https://docs.k3s.io/installation/packaged-components#disabling-manifests).

          '';

        };

    charts = lib.mkOption {

      type = with lib.types; attrsOf (either path package);

      default = { };

      example = lib.literalExpression ''

        nginx = ../charts/my-nginx-chart.tgz;

        redis = ../charts/my-redis-chart.tgz;

      '';

      description = ''

        Packaged Helm charts that are linked to {file}`${chartDir}` before k3s starts.

        The attribute name will be used as the link target (relative to {file}`${chartDir}`).

        The specified charts will only be placed on the file system and made available to the

        Kubernetes APIServer from within the cluster. See the [](#opt-services.k3s.autoDeployCharts)

        option and the [k3s Helm controller docs](https://docs.k3s.io/helm#using-the-helm-controller)

        to deploy Helm charts. This option only makes sense on server nodes (`role = server`).

      '';

    };

        containerdConfigTemplate = lib.mkOption {

          type = lib.types.nullOr lib.types.str;

          default = null;

          example = lib.literalExpression ''

        # Base K3s config

            # Base config

            {{ template "base" . }}

            # Add a custom runtime

          '';

          description = ''

            Config template for containerd, to be placed at

        `/var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl`.

        See the K3s docs on [configuring containerd](https://docs.k3s.io/advanced#configuring-containerd).

            `/var/lib/rancher/${name}/agent/etc/containerd/config.toml.tmpl`.

            See the docs on [configuring containerd](https://docs.${name}.io/advanced#configuring-containerd).

          '';

        };

                hash = "sha256-IM2BLZ0EdKIZcRWOtuFY9TogZJXCpKtPZnMnPsGlq0Y=";

                finalImageTag = "21.1.2-debian-11-r0";

              })

          config.services.k3s.package.airgap-images

            ]

          '';

          description = ''

            List of derivations that provide container images.

        All images are linked to {file}`${imageDir}` before k3s starts and consequently imported

        by the k3s agent. Consider importing the k3s airgap images archive of the k3s package in

        use, if you want to pre-provision this node with all k3s container images. This option

        only makes sense on nodes with an enabled agent.

            All images are linked to {file}`${imageDir}` before ${name} starts and are consequently imported

            by the ${name} agent. This option only makes sense on nodes with an enabled agent.

          '';

        };

          default = { };

          example = {

            mode = "nftables";

        clientConnection.kubeconfig = "/var/lib/rancher/k3s/agent/kubeproxy.kubeconfig";

            clientConnection.kubeconfig = "/var/lib/rancher/${name}/agent/kubeproxy.kubeconfig";

          };

          description = ''

            Extra configuration to add to the kube-proxy's configuration file. The subset of the kube-proxy's

            configuration that can be configured via a file is defined by the

            [KubeProxyConfiguration](https://kubernetes.io/docs/reference/config-api/kube-proxy-config.v1alpha1/)

        struct. Note that the kubeconfig param will be override by `clientConnection.kubeconfig`, so you must

        set the `clientConnection.kubeconfig` if you want to use `extraKubeProxyConfig`.

            struct. Note that the kubeconfig param will be overriden by `clientConnection.kubeconfig`, so you must

            set the `clientConnection.kubeconfig` option if you want to use `extraKubeProxyConfig`.

          '';

        };

            }

          '';

          description = ''

        Auto deploying Helm charts that are installed by the k3s Helm controller. Avoid to use

        attribute names that are also used in the [](#opt-services.k3s.manifests) and

        [](#opt-services.k3s.charts) options. Manifests with the same name will override

        auto deploying charts with the same name. Similiarly, charts with the same name will

        overwrite the Helm chart contained in auto deploying charts. This option only makes

        sense on server nodes (`role = server`). See the

        [k3s Helm documentation](https://docs.k3s.io/helm) for further information.

            Auto deploying Helm charts that are installed by the ${name} Helm controller. Avoid using

            attribute names that are also used in the [](#opt-services.${name}.manifests) option.

            Manifests with the same name will override auto deploying charts with the same name.

            This option only makes sense on server nodes (`role = server`). See the

            [${name} Helm documentation](https://docs.${name}.io/helm) for further information.

          '';

        };

      };

      # implementation

  config = lib.mkIf cfg.enable {

      config = {

        warnings =

          (lib.optional (cfg.role != "server" && cfg.manifests != { })

        "k3s: Auto deploying manifests are only installed on server nodes (role == server), they will be ignored by this node."

      )

      ++ (lib.optional (cfg.role != "server" && cfg.charts != { })

        "k3s: Helm charts are only made available to the cluster on server nodes (role == server), they will be ignored by this node."

            "${name}: Auto deploying manifests are only installed on server nodes (role == server), they will be ignored by this node."

          )

          ++ (lib.optional (cfg.role != "server" && cfg.autoDeployCharts != { })

        "k3s: Auto deploying Helm charts are only installed on server nodes (role == server), they will be ignored by this node."

            "${name}: Auto deploying Helm charts are only installed on server nodes (role == server), they will be ignored by this node."

          )

          ++ (lib.optional (duplicateManifests != [ ])

        "k3s: The following auto deploying charts are overriden by manifests of the same name: ${toString duplicateManifests}."

      )

      ++ (lib.optional (duplicateCharts != [ ])

        "k3s: The following auto deploying charts are overriden by charts of the same name: ${toString duplicateCharts}."

            "${name}: The following auto deploying charts are overriden by manifests of the same name: ${toString duplicateManifests}."

          )

      ++ (lib.optional (

        cfg.disableAgent && cfg.images != [ ]

      ) "k3s: Images are only imported on nodes with an enabled agent, they will be ignored by this node")

          ++ (lib.optional (

            cfg.role == "agent" && cfg.configPath == null && cfg.serverAddr == ""

      ) "k3s: serverAddr or configPath (with 'server' key) should be set if role is 'agent'")

          ) "${name}: serverAddr or configPath (with 'server' key) should be set if role is 'agent'")

          ++ (lib.optional

            (cfg.role == "agent" && cfg.configPath == null && cfg.tokenFile == null && cfg.token == "")

        "k3s: Token or tokenFile or configPath (with 'token' or 'token-file' keys) should be set if role is 'agent'"

            "${name}: Token or tokenFile or configPath (with 'token' or 'token-file' keys) should be set if role is 'agent'"

          );

    assertions = [

      {

        assertion = cfg.role == "agent" -> !cfg.disableAgent;

        message = "k3s: disableAgent must be false if role is 'agent'";

      }

      {

        assertion = cfg.role == "agent" -> !cfg.clusterInit;

        message = "k3s: clusterInit must be false if role is 'agent'";

      }

    ];

    environment.systemPackages = [ config.services.k3s.package ];

        environment.systemPackages = [ config.services.${name}.package ];

    # Use systemd-tmpfiles to activate k3s content

    systemd.tmpfiles.settings."10-k3s" =

        # Use systemd-tmpfiles to activate content

        systemd.tmpfiles.settings."10-${name}" =

          let

            # Merge manifest with manifests generated from auto deploying charts, keep only enabled manifests

            enabledManifests = lib.filterAttrs (_: v: v.enable) (cfg.autoDeployCharts // cfg.manifests);

        # Merge charts with charts contained in enabled auto deploying charts

        helmCharts =

          (lib.concatMapAttrs (n: v: { ${n} = v.package; }) (

            lib.filterAttrs (_: v: v.enable) cfg.autoDeployCharts

          ))

          // cfg.charts;

            # Make a systemd-tmpfiles rule for a manifest

            mkManifestRule = manifest: {

              name = "${manifestDir}/${manifest.target}";

                "L+".argument = "${manifest.source}";

              };

            };

        # Ensure that all chart targets have a .tgz suffix

        mkChartTarget = name: if (lib.hasSuffix ".tgz" name) then name else name + ".tgz";

        # Make a systemd-tmpfiles rule for a chart

        mkChartRule = target: source: {

          name = "${chartDir}/${mkChartTarget target}";

          value = {

            "L+".argument = "${source}";

          };

        };

            # Make a systemd-tmpfiles rule for a container image

            mkImageRule = image: {

              name = "${imageDir}/${image.name}";

            };

          in

          (lib.mapAttrs' (_: v: mkManifestRule v) enabledManifests)

      // (lib.mapAttrs' (n: v: mkChartRule n v) helmCharts)

          // (builtins.listToAttrs (map mkImageRule cfg.images))

          // (lib.optionalAttrs (cfg.containerdConfigTemplate != null) {

            ${containerdConfigTemplateFile} = {

            };

          });

    systemd.services.k3s =

        systemd.services.${name} =

          let

            kubeletParams =

              (lib.optionalAttrs (cfg.gracefulNodeShutdown.enable) {

                inherit (cfg.gracefulNodeShutdown) shutdownGracePeriod shutdownGracePeriodCriticalPods;

              })

              // cfg.extraKubeletConfig;

        kubeletConfig = (pkgs.formats.yaml { }).generate "k3s-kubelet-config" (

            kubeletConfig = (pkgs.formats.yaml { }).generate "${name}-kubelet-config" (

              {

                apiVersion = "kubelet.config.k8s.io/v1beta1";

                kind = "KubeletConfiguration";

              // kubeletParams

            );

        kubeProxyConfig = (pkgs.formats.yaml { }).generate "k3s-kubeProxy-config" (

            kubeProxyConfig = (pkgs.formats.yaml { }).generate "${name}-kubeProxy-config" (

              {

                apiVersion = "kubeproxy.config.k8s.io/v1alpha1";

                kind = "KubeProxyConfiguration";

            );

          in

          {

        description = "k3s service";

            description = "${name} service";

            after = [

              "firewall.service"

              "network-online.target"

              TasksMax = "infinity";

              EnvironmentFile = cfg.environmentFile;

              ExecStart = lib.concatStringsSep " \\\n " (

            [ "${cfg.package}/bin/k3s ${cfg.role}" ]

            ++ (lib.optional cfg.clusterInit "--cluster-init")

            ++ (lib.optional cfg.disableAgent "--disable-agent")

                [ "${cfg.package}/bin/${name} ${cfg.role}" ]

                ++ (lib.optional (cfg.serverAddr != "") "--server ${cfg.serverAddr}")

                ++ (lib.optional (cfg.token != "") "--token ${cfg.token}")

                ++ (lib.optional (cfg.tokenFile != null) "--token-file ${cfg.tokenFile}")

                ++ (lib.optional (cfg.configPath != null) "--config ${cfg.configPath}")

                ++ (lib.optional (kubeletParams != { }) "--kubelet-arg=config=${kubeletConfig}")

                ++ (lib.optional (cfg.extraKubeProxyConfig != { }) "--kube-proxy-arg=config=${kubeProxyConfig}")

                ++ extraBinFlags

                ++ (lib.flatten cfg.extraFlags)

              );

            };

          };

      };

    };

in

{

  imports =

    # pass mkRancherModule explicitly instead of via

    # _modules.args to prevent infinite recursion

    builtins.map (

      f:

      import f {

        inherit config lib;

        inherit mkRancherModule;

      }

    ) [ ./k3s.nix ];

  meta.maintainers = lib.teams.k3s.members;

  meta.maintainers =

    with lib.maintainers;

    [ azey7f ] # modules only

    ++ lib.teams.k3s.members;

}