Commit a64423c7 authored by Robert Rose's avatar Robert Rose
Browse files

nixos/k3s: extend k3s module 

This contribution extends the k3s module to
enable the usage of Helm charts and container
images in air-gapped environments. Additionally,
the manifests option allows to specify arbitrary
manifests that are deployed by k3s automatically.
It is now possible to deploy Kubernetes workloads
using the k3s module.
parent 6c030876

nixos/modules/services/cluster/k3s/default.nix

+241 −0
Original line number Diff line number Diff line
@@ -17,6 +17,109 @@ let 
      ]

      ++ config

    ) instruction;



  manifestDir = "/var/lib/rancher/k3s/server/manifests";

  chartDir = "/var/lib/rancher/k3s/server/static/charts";

  imageDir = "/var/lib/rancher/k3s/agent/images";



  manifestModule =

    let

      mkTarget =

        name: if (lib.hasSuffix ".yaml" name || lib.hasSuffix ".yml" name) then name else name + ".yaml";

    in

    lib.types.submodule (

      {

        name,

        config,

        options,

        ...

      }:

      {

        options = {

          enable = lib.mkOption {

            type = lib.types.bool;

            default = true;

            description = "Whether this manifest file should be generated.";

          };



          target = lib.mkOption {

            type = lib.types.nonEmptyStr;

            example = lib.literalExpression "manifest.yaml";

            description = ''

              Name of the symlink (relative to {file}`${manifestDir}`).

              Defaults to the attribute name.

            '';

          };



          content = lib.mkOption {

            type = with lib.types; nullOr (either attrs (listOf attrs));

            default = null;

            description = ''

              Content of the manifest file. A single attribute set will

              generate a single document YAML file. A list of attribute sets

              will generate multiple documents separated by `---` in a single

              YAML file.

            '';

          };



          source = lib.mkOption {

            type = lib.types.path;

            example = lib.literalExpression "./manifests/app.yaml";

            description = ''

              Path of the source `.yaml` file.

            '';

          };

        };



        config = {

          target = lib.mkDefault (mkTarget name);

          source = lib.mkIf (config.content != null) (

            let

              name' = "k3s-manifest-" + builtins.baseNameOf name;

              docName = "k3s-manifest-doc-" + builtins.baseNameOf name;

              yamlDocSeparator = builtins.toFile "yaml-doc-separator" "\n---\n";

              mkYaml = name: x: (pkgs.formats.yaml { }).generate name x;

              mkSource =

                value:

                if builtins.isList value then

                  pkgs.concatText name' (

                    lib.concatMap (x: [

                      yamlDocSeparator

                      (mkYaml docName x)

                    ]) value

                  )

                else

                  mkYaml name' value;

            in

            lib.mkDerivedConfig options.content mkSource

          );

        };

      }

    );



  enabledManifests = with builtins; filter (m: m.enable) (attrValues cfg.manifests);

  linkManifestEntry = m: "${pkgs.coreutils-full}/bin/ln -sfn ${m.source} ${manifestDir}/${m.target}";

  linkImageEntry = image: "${pkgs.coreutils-full}/bin/ln -sfn ${image} ${imageDir}/${image.name}";

  linkChartEntry =

    let

      mkTarget = name: if (lib.hasSuffix ".tgz" name) then name else name + ".tgz";

    in

    name: value:

    "${pkgs.coreutils-full}/bin/ln -sfn ${value} ${chartDir}/${mkTarget (builtins.baseNameOf name)}";



  activateK3sContent = pkgs.writeShellScript "activate-k3s-content" ''

    ${lib.optionalString (

      builtins.length enabledManifests > 0

    ) "${pkgs.coreutils-full}/bin/mkdir -p ${manifestDir}"}

    ${lib.optionalString (cfg.charts != { }) "${pkgs.coreutils-full}/bin/mkdir -p ${chartDir}"}

    ${lib.optionalString (

      builtins.length cfg.images > 0

    ) "${pkgs.coreutils-full}/bin/mkdir -p ${imageDir}"}



    ${builtins.concatStringsSep "\n" (map linkManifestEntry enabledManifests)}

    ${builtins.concatStringsSep "\n" (lib.mapAttrsToList linkChartEntry cfg.charts)}

    ${builtins.concatStringsSep "\n" (map linkImageEntry cfg.images)}

  '';

in

{

  imports = [ (removeOption [ "docker" ] "k3s docker option is no longer supported.") ];
@@ -127,11 +230,148 @@ in 
      default = null;

      description = "File path containing the k3s YAML config. This is useful when the config is generated (for example on boot).";

    };



    manifests = mkOption {

      type = types.attrsOf manifestModule;

      default = { };

      example = lib.literalExpression ''

        deployment.source = ../manifests/deployment.yaml;

        my-service = {

          enable = false;

          target = "app-service.yaml";

          content = {

            apiVersion = "v1";

            kind = "Service";

            metadata = {

              name = "app-service";

            };

            spec = {

              selector = {

                "app.kubernetes.io/name" = "MyApp";

              };

              ports = [

                {

                  name = "name-of-service-port";

                  protocol = "TCP";

                  port = 80;

                  targetPort = "http-web-svc";

                }

              ];

            };

          }

        };



        nginx.content = [

          {

            apiVersion = "v1";

            kind = "Pod";

            metadata = {

              name = "nginx";

              labels = {

                "app.kubernetes.io/name" = "MyApp";

              };

            };

            spec = {

              containers = [

                {

                  name = "nginx";

                  image = "nginx:1.14.2";

                  ports = [

                    {

                      containerPort = 80;

                      name = "http-web-svc";

                    }

                  ];

                }

              ];

            };

          }

          {

            apiVersion = "v1";

            kind = "Service";

            metadata = {

              name = "nginx-service";

            };

            spec = {

              selector = {

                "app.kubernetes.io/name" = "MyApp";

              };

              ports = [

                {

                  name = "name-of-service-port";

                  protocol = "TCP";

                  port = 80;

                  targetPort = "http-web-svc";

                }

              ];

            };

          }

        ];

      '';

      description = ''

        Auto-deploying manifests that are linked to {file}`${manifestDir}` before k3s starts.

        Note that deleting manifest files will not remove or otherwise modify the resources

        it created. Please use the the `--disable` flag or `.skip` files to delete/disable AddOns,

        as mentioned in the [docs](https://docs.k3s.io/installation/packaged-components#disabling-manifests).

        This option only makes sense on server nodes (`role = server`).

        Read the [auto-deploying manifests docs](https://docs.k3s.io/installation/packaged-components#auto-deploying-manifests-addons)

        for further information.

      '';

    };



    charts = mkOption {

      type = with types; attrsOf (either path package);

      default = { };

      example = lib.literalExpression ''

        nginx = ../charts/my-nginx-chart.tgz;

        redis = ../charts/my-redis-chart.tgz;

      '';

      description = ''

        Packaged Helm charts that are linked to {file}`${chartDir}` before k3s starts.

        The attribute name will be used as the link target (relative to {file}`${chartDir}`).

        The specified charts will only be placed on the file system and made available to the

        Kubernetes APIServer from within the cluster, you may use the

        [k3s Helm controller](https://docs.k3s.io/helm#using-the-helm-controller)

        to deploy the charts. This option only makes sense on server nodes

        (`role = server`).

      '';

    };



    images = mkOption {

      type = with types; listOf package;

      default = [ ];

      example = lib.literalExpression ''

        [

          (pkgs.dockerTools.pullImage {

            imageName = "docker.io/bitnami/keycloak";

            imageDigest = "sha256:714dfadc66a8e3adea6609bda350345bd3711657b7ef3cf2e8015b526bac2d6b";

            sha256 = "0imblp0kw9vkcr7sp962jmj20fpmb3hvd3hmf4cs4x04klnq3k90";

            finalImageTag = "21.1.2-debian-11-r0";

          })

        ]

      '';

      description = ''

        List of derivations that provide container images.

        All images are linked to {file}`${imageDir}` before k3s starts and consequently imported

        by the k3s agent. This option only makes sense on nodes with an enabled agent.

      '';

    };

  };



  # implementation



  config = mkIf cfg.enable {

    warnings =

      (lib.optional (cfg.role != "server" && cfg.manifests != { })

        "k3s: Auto deploying manifests are only installed on server nodes (role == server), they will be ignored by this node."

      )

      ++ (lib.optional (cfg.role != "server" && cfg.charts != { })

        "k3s: Helm charts are only made available to the cluster on server nodes (role == server), they will be ignored by this node."

      )

      ++ (lib.optional (cfg.disableAgent && cfg.images != [ ])

        "k3s: Images are only imported on nodes with an enabled agent, they will be ignored by this node"

      );



    assertions = [

      {

        assertion = cfg.role == "agent" -> (cfg.configPath != null || cfg.serverAddr != "");
@@ -178,6 +418,7 @@ in 
        LimitCORE = "infinity";

        TasksMax = "infinity";

        EnvironmentFile = cfg.environmentFile;

        ExecStartPre = activateK3sContent;

        ExecStart = concatStringsSep " \\\n " (

          [ "${cfg.package}/bin/k3s ${cfg.role}" ]

          ++ (optional cfg.clusterInit "--cluster-init")

nixos/tests/k3s/auto-deploy.nix

0 → 100644
+122 −0
Original line number Diff line number Diff line 
import ../make-test-python.nix (

  {

    pkgs,

    lib,

    k3s,

    ...

  }:

  let

    pauseImageEnv = pkgs.buildEnv {

      name = "k3s-pause-image-env";

      paths = with pkgs; [

        tini

        (hiPrio coreutils)

        busybox

      ];

    };

    pauseImage = pkgs.dockerTools.buildImage {

      name = "test.local/pause";

      tag = "local";

      copyToRoot = pauseImageEnv;

      config.Entrypoint = [

        "/bin/tini"

        "--"

        "/bin/sleep"

        "inf"

      ];

    };

    helloImage = pkgs.dockerTools.buildImage {

      name = "test.local/hello";

      tag = "local";

      copyToRoot = pkgs.hello;

      config.Entrypoint = [ "${pkgs.hello}/bin/hello" ];

    };

  in

  {

    name = "${k3s.name}-auto-deploy";



    nodes.machine =

      { pkgs, ... }:

      {

        environment.systemPackages = [ k3s ];



        # k3s uses enough resources the default vm fails.

        virtualisation.memorySize = 1536;

        virtualisation.diskSize = 4096;



        services.k3s.enable = true;

        services.k3s.role = "server";

        services.k3s.package = k3s;

        # Slightly reduce resource usage

        services.k3s.extraFlags = builtins.toString [

          "--disable coredns"

          "--disable local-storage"

          "--disable metrics-server"

          "--disable servicelb"

          "--disable traefik"

          "--pause-image test.local/pause:local"

        ];

        services.k3s.images = [

          pauseImage

          helloImage

        ];

        services.k3s.manifests = {

          absent = {

            enable = false;

            content = {

              apiVersion = "v1";

              kind = "Namespace";

              metadata.name = "absent";

            };

          };



          present = {

            target = "foo-namespace.yaml";

            content = {

              apiVersion = "v1";

              kind = "Namespace";

              metadata.name = "foo";

            };

          };



          hello.content = {

            apiVersion = "batch/v1";

            kind = "Job";

            metadata.name = "hello";

            spec = {

              template.spec = {

                containers = [

                  {

                    name = "hello";

                    image = "test.local/hello:local";

                  }

                ];

                restartPolicy = "OnFailure";

              };

            };

          };

        };

      };



    testScript = ''

      start_all()



      machine.wait_for_unit("k3s")

      # check existence of the manifest files

      machine.fail("ls /var/lib/rancher/k3s/server/manifests/absent.yaml")

      machine.succeed("ls /var/lib/rancher/k3s/server/manifests/foo-namespace.yaml")

      machine.succeed("ls /var/lib/rancher/k3s/server/manifests/hello.yaml")



      # check if container images got imported

      machine.succeed("crictl img | grep 'test\.local/pause'")

      machine.succeed("crictl img | grep 'test\.local/hello'")



      # check if resources of manifests got created

      machine.wait_until_succeeds("kubectl get ns foo")

      machine.wait_until_succeeds("kubectl wait --for=condition=complete job/hello")

      machine.fail("kubectl get ns absent")



      machine.shutdown()

    '';

  }

)

nixos/tests/k3s/default.nix

+2 −0
Original line number Diff line number Diff line
@@ -19,4 +19,6 @@ in 
  single-node = lib.mapAttrs (_: k3s: import ./single-node.nix { inherit system pkgs k3s; }) allK3s;

  # Run a multi-node k3s cluster and verify pod networking works across nodes

  multi-node = lib.mapAttrs (_: k3s: import ./multi-node.nix { inherit system pkgs k3s; }) allK3s;

  # Test wether container images are imported and auto deploying manifests work

  auto-deploy = lib.mapAttrs (_: k3s: import ./auto-deploy.nix { inherit system pkgs k3s; }) allK3s;

}