Unverified Commit a14741d2 authored by rorosen's avatar rorosen Committed by GitHub
Browse files

nixos/tests/{k3s,rke2}: merge & cleanup (#469788)

parents b6d3df5c 49bd3e5e
Loading
Loading
Loading
Loading
+1 −1
Original line number Diff line number Diff line
@@ -263,7 +263,7 @@
          - any-glob-to-any-file:
              - nixos/modules/services/cluster/rancher/default.nix
              - nixos/modules/services/cluster/rancher/k3s.nix
              - nixos/tests/k3s/**/*
              - nixos/tests/rancher/**/*
              - pkgs/applications/networking/cluster/k3s/**/*

"6.topic: kernel":
+14 −2
Original line number Diff line number Diff line
@@ -802,7 +802,11 @@ in
  jitsi-meet = runTest ./jitsi-meet.nix;
  jool = import ./jool.nix { inherit pkgs runTest; };
  jotta-cli = runTest ./jotta-cli.nix;
  k3s = handleTest ./k3s { };
  k3s = import ./rancher {
    inherit pkgs runTest;
    inherit (pkgs) lib;
    rancherDistro = "k3s";
  };
  kafka = handleTest ./kafka { };
  kaidan = runTest ./kaidan;
  kanboard = runTest ./web-apps/kanboard.nix;
@@ -1343,7 +1347,15 @@ in
  restic-rest-server = runTest ./restic-rest-server.nix;
  retroarch = runTest ./retroarch.nix;
  ringboard = runTest ./ringboard.nix;
  rke2 = handleTestOn [ "aarch64-linux" "x86_64-linux" ] ./rke2 { };
  rke2 = import ./rancher {
    inherit pkgs;
    inherit (pkgs) lib;
    runTest = runTestOn [
      "aarch64-linux"
      "x86_64-linux"
    ];
    rancherDistro = "rke2";
  };
  rkvm = handleTest ./rkvm { };
  rmfakecloud = runTest ./rmfakecloud.nix;
  robustirc-bridge = runTest ./robustirc-bridge.nix;

nixos/tests/k3s/airgap-images.nix

deleted100644 → 0
+0 −34
Original line number Diff line number Diff line
# A test that imports k3s airgapped images and verifies that all expected images are present
import ../make-test-python.nix (
  { lib, k3s, ... }:
  {
    name = "${k3s.name}-airgap-images";
    meta.maintainers = lib.teams.k3s.members;

    nodes.machine = _: {
      # k3s uses enough resources the default vm fails.
      virtualisation.memorySize = 1536;
      virtualisation.diskSize = 4096;

      services.k3s = {
        enable = true;
        role = "server";
        package = k3s;
        # Slightly reduce resource usage
        extraFlags = [
          "--disable coredns"
          "--disable local-storage"
          "--disable metrics-server"
          "--disable servicelb"
          "--disable traefik"
        ];
        images = [ k3s.airgap-images ];
      };
    };

    testScript = ''
      machine.wait_for_unit("k3s")
      machine.wait_until_succeeds("journalctl -r --no-pager -u k3s | grep \"Imported images from /var/lib/rancher/k3s/agent/images/\"")
    '';
  }
)
+0 −204
Original line number Diff line number Diff line
# Tests whether container images are imported and auto deploying Helm charts,
# including the bundled traefik, work
import ../make-test-python.nix (
  {
    k3s,
    lib,
    pkgs,
    ...
  }:
  let
    testImageEnv = pkgs.buildEnv {
      name = "k3s-pause-image-env";
      paths = with pkgs; [
        busybox
        hello
      ];
    };
    testImage = pkgs.dockerTools.buildImage {
      name = "test.local/test";
      tag = "local";
      # Slightly reduces the time needed to import image
      compressor = "zstd";
      copyToRoot = testImageEnv;
    };
    # pack the test helm chart as a .tgz archive
    package =
      pkgs.runCommand "k3s-test-chart.tgz"
        {
          nativeBuildInputs = [ pkgs.kubernetes-helm ];
          chart = builtins.toJSON {
            name = "k3s-test-chart";
            version = "0.1.0";
          };
          values = builtins.toJSON {
            restartPolicy = "Never";
            runCommand = "";
            image = {
              repository = "foo";
              tag = "1.0.0";
            };
          };
          job = builtins.toJSON {
            apiVersion = "batch/v1";
            kind = "Job";
            metadata = {
              name = "{{ .Release.Name }}";
              namespace = "{{ .Release.Namespace }}";
            };
            spec = {
              template = {
                spec = {
                  containers = [
                    {
                      name = "test";
                      image = "{{ .Values.image.repository }}:{{ .Values.image.tag }}";
                      command = [ "sh" ];
                      args = [
                        "-c"
                        "{{ .Values.runCommand }}"
                      ];
                    }
                  ];
                  restartPolicy = "{{ .Values.restartPolicy }}";
                };
              };
            };
          };
          passAsFile = [
            "values"
            "chart"
            "job"
          ];
        }
        ''
          mkdir -p chart/templates
          cp "$chartPath" chart/Chart.yaml
          cp "$valuesPath" chart/values.yaml
          cp "$jobPath" chart/templates/job.json

          helm package chart
          mv ./*.tgz $out
        '';
    # The common Helm chart that is used in this test
    testChart = {
      inherit package;
      values = {
        runCommand = "hello";
        image = {
          repository = testImage.imageName;
          tag = testImage.imageTag;
        };
      };
    };
  in
  {
    name = "${k3s.name}-auto-deploy-helm";
    meta.maintainers = lib.teams.k3s.members;
    nodes.machine =
      { pkgs, ... }:
      {
        # k3s uses enough resources the default vm fails.
        virtualisation = {
          memorySize = 1536;
          diskSize = 4096;
        };
        environment.systemPackages = [ pkgs.yq-go ];
        services.k3s = {
          enable = true;
          package = k3s;
          # Slightly reduce resource usage
          extraFlags = [
            "--disable coredns"
            "--disable local-storage"
            "--disable metrics-server"
            "--disable servicelb"
          ];
          images = [
            # Provides the k3s Helm controller
            k3s.airgap-images
            testImage
          ];
          autoDeployCharts = {
            # regular test chart that should get installed
            hello = testChart;
            # disabled chart that should not get installed
            disabled = testChart // {
              enable = false;
            };
            # chart with values set via YAML file
            values-file = testChart // {
              # Remove unsafeDiscardStringContext workaround when Nix can convert a string to a path
              # https://github.com/NixOS/nix/issues/12407
              values =
                /.
                + builtins.unsafeDiscardStringContext (
                  builtins.toFile "k3s-test-chart-values.yaml" ''
                    runCommand: "echo 'Hello, file!'"
                    image:
                      repository: test.local/test
                      tag: local
                  ''
                );
            };
            # advanced chart that should get installed in the "test" namespace with a custom
            # timeout and overridden values
            advanced = testChart // {
              # create the "test" namespace via extraDeploy for testing
              extraDeploy = [
                {
                  apiVersion = "v1";
                  kind = "Namespace";
                  metadata.name = "test";
                }
              ];
              extraFieldDefinitions = {
                spec = {
                  # overwrite chart values
                  valuesContent = ''
                    runCommand: "echo 'advanced hello'"
                    image:
                      repository: ${testImage.imageName}
                      tag: ${testImage.imageTag}
                  '';
                  # overwrite the chart namespace
                  targetNamespace = "test";
                  # set a custom timeout
                  timeout = "69s";
                };
              };
            };
          };
        };
      };

    testScript = # python
      ''
        import json

        machine.wait_for_unit("k3s")
        # check existence/absence of chart manifest files
        machine.succeed("test -e /var/lib/rancher/k3s/server/manifests/hello.yaml")
        machine.succeed("test ! -e /var/lib/rancher/k3s/server/manifests/disabled.yaml")
        machine.succeed("test -e /var/lib/rancher/k3s/server/manifests/values-file.yaml")
        machine.succeed("test -e /var/lib/rancher/k3s/server/manifests/advanced.yaml")
        # check that the timeout is set correctly, select only the first doc in advanced.yaml
        advancedManifest = json.loads(machine.succeed("yq -o json '.items[0]' /var/lib/rancher/k3s/server/manifests/advanced.yaml"))
        t.assertEqual(advancedManifest["spec"]["timeout"], "69s", "unexpected value for spec.timeout")
        # wait for test jobs to complete
        machine.wait_until_succeeds("kubectl wait --for=condition=complete job/hello", timeout=180)
        machine.wait_until_succeeds("kubectl wait --for=condition=complete job/values-file", timeout=180)
        machine.wait_until_succeeds("kubectl -n test wait --for=condition=complete job/advanced", timeout=180)
        # check output of test jobs
        hello_output = machine.succeed("kubectl logs -l batch.kubernetes.io/job-name=hello")
        values_file_output = machine.succeed("kubectl logs -l batch.kubernetes.io/job-name=values-file")
        advanced_output = machine.succeed("kubectl -n test logs -l batch.kubernetes.io/job-name=advanced")
        # strip the output to remove trailing whitespaces
        t.assertEqual(hello_output.rstrip(), "Hello, world!", "unexpected output of hello job")
        t.assertEqual(values_file_output.rstrip(), "Hello, file!", "unexpected output of values file job")
        t.assertEqual(advanced_output.rstrip(), "advanced hello", "unexpected output of advanced job")
        # wait for bundled traefik deployment
        machine.wait_until_succeeds("kubectl -n kube-system rollout status deployment traefik", timeout=180)
      '';
  }
)

nixos/tests/k3s/auto-deploy.nix

deleted100644 → 0
+0 −124
Original line number Diff line number Diff line
# Tests whether container images are imported and auto deploying manifests work
import ../make-test-python.nix (
  {
    pkgs,
    lib,
    k3s,
    ...
  }:
  let
    pauseImageEnv = pkgs.buildEnv {
      name = "k3s-pause-image-env";
      paths = with pkgs; [
        tini
        (lib.hiPrio coreutils)
        busybox
      ];
    };
    pauseImage = pkgs.dockerTools.buildImage {
      name = "test.local/pause";
      tag = "local";
      copyToRoot = pauseImageEnv;
      config.Entrypoint = [
        "/bin/tini"
        "--"
        "/bin/sleep"
        "inf"
      ];
    };
    helloImage = pkgs.dockerTools.buildImage {
      name = "test.local/hello";
      tag = "local";
      copyToRoot = pkgs.hello;
      config.Entrypoint = [ "${pkgs.hello}/bin/hello" ];
    };
  in
  {
    name = "${k3s.name}-auto-deploy";

    nodes.machine =
      { pkgs, ... }:
      {
        environment.systemPackages = [ k3s ];

        # k3s uses enough resources the default vm fails.
        virtualisation.memorySize = 1536;
        virtualisation.diskSize = 4096;

        services.k3s.enable = true;
        services.k3s.role = "server";
        services.k3s.package = k3s;
        # Slightly reduce resource usage
        services.k3s.extraFlags = [
          "--disable coredns"
          "--disable local-storage"
          "--disable metrics-server"
          "--disable servicelb"
          "--disable traefik"
          "--pause-image test.local/pause:local"
        ];
        services.k3s.images = [
          pauseImage
          helloImage
        ];
        services.k3s.manifests = {
          absent = {
            enable = false;
            content = {
              apiVersion = "v1";
              kind = "Namespace";
              metadata.name = "absent";
            };
          };

          present = {
            target = "foo-namespace.yaml";
            content = {
              apiVersion = "v1";
              kind = "Namespace";
              metadata.name = "foo";
            };
          };

          hello.content = {
            apiVersion = "batch/v1";
            kind = "Job";
            metadata.name = "hello";
            spec = {
              template.spec = {
                containers = [
                  {
                    name = "hello";
                    image = "test.local/hello:local";
                  }
                ];
                restartPolicy = "OnFailure";
              };
            };
          };
        };
      };

    testScript = # python
      ''
        start_all()

        machine.wait_for_unit("k3s")
        # check existence of the manifest files
        machine.fail("ls /var/lib/rancher/k3s/server/manifests/absent.yaml")
        machine.succeed("ls /var/lib/rancher/k3s/server/manifests/foo-namespace.yaml")
        machine.succeed("ls /var/lib/rancher/k3s/server/manifests/hello.yaml")

        # check if container images got imported
        machine.wait_until_succeeds("crictl img | grep 'test\.local/pause'")
        machine.wait_until_succeeds("crictl img | grep 'test\.local/hello'")

        # check if resources of manifests got created
        machine.wait_until_succeeds("kubectl get ns foo")
        machine.wait_until_succeeds("kubectl wait --for=condition=complete job/hello")
        machine.fail("kubectl get ns absent")
      '';

    meta.maintainers = lib.teams.k3s.members;
  }
)
Loading