Unverified Commit 2f5bd8e6 authored by azey's avatar azey
Browse files

nixos/tests/k3s: generalize to rancher 

Similarly to 2ce16ee6, prepares for merge with nixos/tests/rke2.
parent bf83174d

.github/labeler.yml

+1 −1
Original line number Diff line number Diff line
@@ -263,7 +263,7 @@ 
          - any-glob-to-any-file:

              - nixos/modules/services/cluster/rancher/default.nix

              - nixos/modules/services/cluster/rancher/k3s.nix

              - nixos/tests/k3s/**/*

              - nixos/tests/rancher/**/*

              - pkgs/applications/networking/cluster/k3s/**/*



"6.topic: kernel":

nixos/tests/all-tests.nix

+1 −1
Original line number Diff line number Diff line
@@ -800,7 +800,7 @@ in 
  jitsi-meet = runTest ./jitsi-meet.nix;

  jool = import ./jool.nix { inherit pkgs runTest; };

  jotta-cli = runTest ./jotta-cli.nix;

  k3s = handleTest ./k3s { };

  k3s = handleTest ./rancher { rancherDistro = "k3s"; };

  kafka = handleTest ./kafka { };

  kanboard = runTest ./web-apps/kanboard.nix;

  kanidm = runTest ./kanidm.nix;

nixos/tests/k3s/default.nix

deleted100644 → 0
+0 −34
Original line number Diff line number Diff line 
{

  system ? builtins.currentSystem,

  pkgs ? import ../../.. { inherit system; },

  lib ? pkgs.lib,

}:

let

  allK3s = lib.filterAttrs (

    n: _: lib.strings.hasPrefix "k3s_" n && (builtins.tryEval pkgs.${n}).success

  ) pkgs;

in

{

  airgap-images = lib.mapAttrs (

    _: k3s: import ./airgap-images.nix { inherit system pkgs k3s; }

  ) allK3s;

  auto-deploy = lib.mapAttrs (_: k3s: import ./auto-deploy.nix { inherit system pkgs k3s; }) allK3s;

  auto-deploy-charts = lib.mapAttrs (

    _: k3s: import ./auto-deploy-charts.nix { inherit system pkgs k3s; }

  ) allK3s;

  containerd-config = lib.mapAttrs (

    _: k3s: import ./containerd-config.nix { inherit system pkgs k3s; }

  ) allK3s;

  etcd = lib.mapAttrs (

    _: k3s:

    import ./etcd.nix {

      inherit system pkgs k3s;

      inherit (pkgs) etcd;

    }

  ) allK3s;

  kubelet-config = lib.mapAttrs (

    _: k3s: import ./kubelet-config.nix { inherit system pkgs k3s; }

  ) allK3s;

  multi-node = lib.mapAttrs (_: k3s: import ./multi-node.nix { inherit system pkgs k3s; }) allK3s;

  single-node = lib.mapAttrs (_: k3s: import ./single-node.nix { inherit system pkgs k3s; }) allK3s;

}

nixos/tests/k3s/airgap-images.nix→nixos/tests/rancher/airgap-images.nix

+38 −0
Original line number Diff line number Diff line 
# A test that imports k3s airgapped images and verifies that all expected images are present

import ../make-test-python.nix (

  { lib, k3s, ... }:

  {

    name = "${k3s.name}-airgap-images";

    lib,

    rancherDistro,

    rancherPackage,

    serviceName,

    disabledComponents,

    ...

  }:

  {

    name = "${rancherPackage.name}-airgap-images";

    meta.maintainers = lib.teams.k3s.members;



    nodes.machine = _: {
@@ -10,25 +17,22 @@ import ../make-test-python.nix ( 
      virtualisation.memorySize = 1536;

      virtualisation.diskSize = 4096;



      services.k3s = {

      services.${rancherDistro} = {

        enable = true;

        role = "server";

        package = k3s;

        # Slightly reduce resource usage

        extraFlags = [

          "--disable coredns"

          "--disable local-storage"

          "--disable metrics-server"

          "--disable servicelb"

          "--disable traefik"

        ];

        images = [ k3s.airgap-images ];

        package = rancherPackage;

        disable = disabledComponents;

        images =

          {

            k3s = [ rancherPackage.airgap-images ];

          }

          .${rancherDistro};

      };

    };



    testScript = ''

      machine.wait_for_unit("k3s")

      machine.wait_until_succeeds("journalctl -r --no-pager -u k3s | grep \"Imported images from /var/lib/rancher/k3s/agent/images/\"")

      machine.wait_for_unit("${serviceName}")

      machine.wait_until_succeeds("journalctl -r --no-pager -u ${serviceName} | grep \"Imported images from /var/lib/rancher/${rancherDistro}/agent/images/\"")

    '';

  }

)

nixos/tests/k3s/auto-deploy-charts.nix→nixos/tests/rancher/auto-deploy-charts.nix

+41 −27
Original line number Diff line number Diff line 
# Tests whether container images are imported and auto deploying Helm charts,

# including the bundled traefik, work

# including the bundled traefik or ingress-nginx, work

import ../make-test-python.nix (

  {

    k3s,

    lib,

    pkgs,

    lib,

    rancherDistro,

    rancherPackage,

    serviceName,

    disabledComponents,

    ...

  }:

  let

    testImageEnv = pkgs.buildEnv {

      name = "k3s-pause-image-env";

      name = "${rancherDistro}-pause-image-env";

      paths = with pkgs; [

        busybox

        hello
@@ -24,11 +27,11 @@ import ../make-test-python.nix ( 
    };

    # pack the test helm chart as a .tgz archive

    package =

      pkgs.runCommand "k3s-test-chart.tgz"

      pkgs.runCommand "${rancherDistro}-test-chart.tgz"

        {

          nativeBuildInputs = [ pkgs.kubernetes-helm ];

          chart = builtins.toJSON {

            name = "k3s-test-chart";

            name = "${rancherDistro}-test-chart";

            version = "0.1.0";

          };

          values = builtins.toJSON {
@@ -93,7 +96,7 @@ import ../make-test-python.nix ( 
    };

  in

  {

    name = "${k3s.name}-auto-deploy-helm";

    name = "${rancherPackage.name}-auto-deploy-helm";

    meta.maintainers = lib.teams.k3s.members;

    nodes.machine =

      { pkgs, ... }:
@@ -104,19 +107,23 @@ import ../make-test-python.nix ( 
          diskSize = 4096;

        };

        environment.systemPackages = [ pkgs.yq-go ];

        services.k3s = {

        services.${rancherDistro} = {

          enable = true;

          package = k3s;

          # Slightly reduce resource usage

          extraFlags = [

            "--disable coredns"

            "--disable local-storage"

            "--disable metrics-server"

            "--disable servicelb"

          ];

          package = rancherPackage;

          disable =

            {

              k3s = lib.remove "traefik" disabledComponents;

              rke2 = lib.remove "rke2-ingress-nginx" disabledComponents;

            }

            .${rancherDistro};

          images = [

            {

              # Provides the k3s Helm controller

            k3s.airgap-images

              k3s = rancherPackage.airgap-images;

              rke2 = rancherPackage.images-core-linux-amd64-tar-zst;

            }

            .${rancherDistro}



            testImage

          ];

          autoDeployCharts = {
@@ -133,7 +140,7 @@ import ../make-test-python.nix ( 
              values =

                /.

                + builtins.unsafeDiscardStringContext (

                  builtins.toFile "k3s-test-chart-values.yaml" ''

                  builtins.toFile "${rancherDistro}-test-chart-values.yaml" ''

                    runCommand: "echo 'Hello, file!'"

                    image:

                      repository: test.local/test
@@ -176,14 +183,14 @@ import ../make-test-python.nix ( 
      ''

        import json



        machine.wait_for_unit("k3s")

        machine.wait_for_unit("${serviceName}")

        # check existence/absence of chart manifest files

        machine.succeed("test -e /var/lib/rancher/k3s/server/manifests/hello.yaml")

        machine.succeed("test ! -e /var/lib/rancher/k3s/server/manifests/disabled.yaml")

        machine.succeed("test -e /var/lib/rancher/k3s/server/manifests/values-file.yaml")

        machine.succeed("test -e /var/lib/rancher/k3s/server/manifests/advanced.yaml")

        machine.succeed("test -e /var/lib/rancher/${rancherDistro}/server/manifests/hello.yaml")

        machine.succeed("test ! -e /var/lib/rancher/${rancherDistro}/server/manifests/disabled.yaml")

        machine.succeed("test -e /var/lib/rancher/${rancherDistro}/server/manifests/values-file.yaml")

        machine.succeed("test -e /var/lib/rancher/${rancherDistro}/server/manifests/advanced.yaml")

        # check that the timeout is set correctly, select only the first doc in advanced.yaml

        advancedManifest = json.loads(machine.succeed("yq -o json '.items[0]' /var/lib/rancher/k3s/server/manifests/advanced.yaml"))

        advancedManifest = json.loads(machine.succeed("yq -o json '.items[0]' /var/lib/rancher/${rancherDistro}/server/manifests/advanced.yaml"))

        t.assertEqual(advancedManifest["spec"]["timeout"], "69s", "unexpected value for spec.timeout")

        # wait for test jobs to complete

        machine.wait_until_succeeds("kubectl wait --for=condition=complete job/hello", timeout=180)
@@ -197,8 +204,15 @@ import ../make-test-python.nix ( 
        t.assertEqual(hello_output.rstrip(), "Hello, world!", "unexpected output of hello job")

        t.assertEqual(values_file_output.rstrip(), "Hello, file!", "unexpected output of values file job")

        t.assertEqual(advanced_output.rstrip(), "advanced hello", "unexpected output of advanced job")

        # wait for bundled traefik deployment

        # wait for bundled ingress deployment

        ${

          {

            k3s = ''

              machine.wait_until_succeeds("kubectl -n kube-system rollout status deployment traefik", timeout=180)

            '';

          }

          .${rancherDistro}

        }

      '';

  }

)