Unverified Commit 83aa6721 authored by Adam C. Stephens's avatar Adam C. Stephens
Browse files

nixos/tests/incus: use child class

parent 55890046
Loading
Loading
Loading
Loading
+8 −12
Original line number Diff line number Diff line
@@ -742,18 +742,14 @@ in
  immich-vectorchord-migration = runTest ./web-apps/immich-vectorchord-migration.nix;
  immich-vectorchord-reindex = runTest ./web-apps/immich-vectorchord-reindex.nix;
  incron = runTest ./incron.nix;
  incus = recurseIntoAttrs (
    import ./incus {
  incus = import ./incus {
    inherit runTest;
    lts = false;
    }
  );
  incus-lts = recurseIntoAttrs (
    import ./incus {
  };
  incus-lts = import ./incus {
    inherit runTest;
    lts = true;
    }
  );
  };
  influxdb = runTest ./influxdb.nix;
  influxdb2 = runTest ./influxdb2.nix;
  initrd-luks-empty-passphrase = runTest ./initrd-luks-empty-passphrase.nix;
+8 −9
Original line number Diff line number Diff line
{
  runTest,
  lts ? true,
  ...
}:
let
  incusTest =
  incusRunTest =
    config:
    runTest {
      imports = [
@@ -19,18 +18,18 @@ let
    };
in
{
  all = incusTest { all = true; };
  all = incusRunTest { all = true; };

  appArmor = incusTest {
  appArmor = incusRunTest {
    all = true;
    appArmor = true;
  };

  container = incusTest { instance.container = true; };
  container = incusRunTest { instance.container = true; };

  lvm = incusTest { storage.lvm = true; };
  lvm = incusRunTest { storage.lvm = true; };

  openvswitch = incusTest { network.ovs = true; };
  openvswitch = incusRunTest { network.ovs = true; };

  ui = runTest {
    imports = [ ./ui.nix ];
@@ -38,7 +37,7 @@ in
    _module.args = { inherit lts; };
  };

  virtual-machine = incusTest { instance.virtual-machine = true; };
  virtual-machine = incusRunTest { instance.virtual-machine = true; };

  zfs = incusTest { storage.zfs = true; };
  zfs = incusRunTest { storage.zfs = true; };
}
+215 −271
Original line number Diff line number Diff line
@@ -145,68 +145,12 @@ in
    };
  };

  testScript = # python
  testScript =
    lib.readFile ./incus_machine.py
    +
      # python
      ''
    import json

    def wait_for_instance(name: str, project: str = "default"):
        machine.wait_until_succeeds(f"incus exec {name} --disable-stdin --force-interactive --project {project} -- /run/current-system/sw/bin/systemctl is-system-running")


    def wait_incus_exec_success(name: str, command: str, timeout: int = 900, project: str = "default"):
        def check_command(_) -> bool:
            status, _ = machine.execute(f"incus exec {name} --disable-stdin --force-interactive --project {project} -- {command}")
            return status == 0

        with machine.nested(f"Waiting for successful exec: {command}"):
          retry(check_command, timeout)


    def set_config(name: str, config: str, restart: bool = False, unset: bool = False):
        if restart:
            machine.succeed(f"incus stop {name}")

        if unset:
          machine.succeed(f"incus config unset {name} {config}")
        else:
          machine.succeed(f"incus config set {name} {config}")

        if restart:
            machine.succeed(f"incus start {name}")
            wait_for_instance(name)
        else:
            # give a moment to settle
            machine.sleep(1)


    def cleanup():
        # avoid conflict between preseed and cleanup operations
        machine.execute("systemctl kill incus-preseed.service")

        instances = json.loads(machine.succeed("incus list --format json --all-projects"))
        with subtest("Stopping all running instances"):
            for instance in [a for a in instances if a['status'] == 'Running']:
                machine.execute(f"incus stop --force {instance['name']} --project {instance['project']}")
                machine.execute(f"incus delete --force {instance['name']} --project {instance['project']}")


    def check_sysctl(name: str):
        with subtest("systemd sysctl settings are applied"):
            machine.succeed(f"incus exec {name} -- systemctl status systemd-sysctl")
            sysctl = machine.succeed(f"incus exec {name} -- sysctl net.ipv4.ip_forward").strip().split(" ")[-1]
            assert "1" == sysctl, f"systemd-sysctl configuration not correctly applied, {sysctl} != 1"


    with subtest("Wait for startup"):
        machine.wait_for_unit("incus.service")
        machine.wait_for_unit("incus-preseed.service")


    with subtest("Verify preseed resources created"):
        machine.succeed("incus profile show default")
        machine.succeed("incus network info incusbr0")
        machine.succeed("incus storage show default")

        server = IncusHost(machine)
      ''
    + lib.optionalString cfg.appArmor ''
      with subtest("Verify AppArmor service is started without issue"):
@@ -233,7 +177,7 @@ in

          with subtest("container can be launched and managed"):
              machine.succeed(f"incus launch {alias} container-{variant}1")
            wait_for_instance(f"container-{variant}1")
              machine.wait_for_instance(f"container-{variant}1")


          with subtest("container mounts lxcfs overlays"):
@@ -242,23 +186,23 @@ in


          with subtest("container CPU limits can be managed"):
            set_config(f"container-{variant}1", "limits.cpu 1", restart=True)
            wait_incus_exec_success(f"container-{variant}1", "nproc | grep '^1$'", timeout=90)
              machine.set_instance_config(f"container-{variant}1", "limits.cpu 1", restart=True)
              machine.wait_instance_exec_success(f"container-{variant}1", "nproc | grep '^1$'", timeout=90)


          with subtest("container CPU limits can be hotplug changed"):
            set_config(f"container-{variant}1", "limits.cpu 2")
            wait_incus_exec_success(f"container-{variant}1", "nproc | grep '^2$'", timeout=90)
              machine.set_instance_config(f"container-{variant}1", "limits.cpu 2")
              machine.wait_instance_exec_success(f"container-{variant}1", "nproc | grep '^2$'", timeout=90)


          with subtest("container memory limits can be managed"):
            set_config(f"container-{variant}1", "limits.memory 128MB", restart=True)
            wait_incus_exec_success(f"container-{variant}1", "grep 'MemTotal:[[:space:]]*125000 kB' /proc/meminfo", timeout=90)
              machine.set_instance_config(f"container-{variant}1", "limits.memory 128MB", restart=True)
              machine.wait_instance_exec_success(f"container-{variant}1", "grep 'MemTotal:[[:space:]]*125000 kB' /proc/meminfo", timeout=90)


          with subtest("container memory limits can be hotplug changed"):
            set_config(f"container-{variant}1", "limits.memory 256MB")
            wait_incus_exec_success(f"container-{variant}1", "grep 'MemTotal:[[:space:]]*250000 kB' /proc/meminfo", timeout=90)
              machine.set_instance_config(f"container-{variant}1", "limits.memory 256MB")
              machine.wait_instance_exec_success(f"container-{variant}1", "grep 'MemTotal:[[:space:]]*250000 kB' /proc/meminfo", timeout=90)


          with subtest("container software tpm can be configured"):
@@ -274,25 +218,25 @@ in
                  # default container is plain
                  machine.succeed(f"incus exec container-{variant}1 test -- -e /run/systemd/system/service.d/zzz-lxc-service.conf")

                check_sysctl(f"container-{variant}1")
                  machine.check_instance_sysctl(f"container-{variant}1")

              with subtest("lxc-container generator configures nested container"):
                set_config(f"container-{variant}1", "security.nesting=true", restart=True)
                  machine.set_instance_config(f"container-{variant}1", "security.nesting=true", restart=True)

                  machine.fail(f"incus exec container-{variant}1 test -- -e /run/systemd/system/service.d/zzz-lxc-service.conf")
                  target = machine.succeed(f"incus exec container-{variant}1 readlink -- -f /run/systemd/system/systemd-binfmt.service").strip()
                  assert target == "/dev/null", "lxc generator did not correctly mask /run/systemd/system/systemd-binfmt.service"

                check_sysctl(f"container-{variant}1")
                  machine.check_instance_sysctl(f"container-{variant}1")

              with subtest("lxc-container generator configures privileged container"):
                  # Create a new instance for a clean state
                  machine.succeed(f"incus launch {alias} container-{variant}2")
                wait_for_instance(f"container-{variant}2")
                  machine.wait_for_instance(f"container-{variant}2")

                  machine.succeed(f"incus exec container-{variant}2 test -- -e /run/systemd/system/service.d/zzz-lxc-service.conf")

                check_sysctl(f"container-{variant}2")
                  machine.check_instance_sysctl(f"container-{variant}2")

          with subtest("container supports per-instance lxcfs"):
              machine.succeed(f"incus stop container-{variant}1")
@@ -301,13 +245,13 @@ in
              machine.succeed("incus config set instances.lxcfs.per_instance=true")

              machine.succeed(f"incus start container-{variant}1")
            wait_for_instance(f"container-{variant}1")
              machine.wait_for_instance(f"container-{variant}1")
              machine.succeed(f"pgrep -a lxcfs | grep 'incus/devices/container-{variant}1/lxcfs'")


          with subtest("container can successfully restart"):
              machine.succeed(f"incus restart container-{variant}1")
            wait_for_instance(f"container-{variant}1")
              machine.wait_for_instance(f"container-{variant}1")


          with subtest("container remains running when softDaemonRestart is enabled and service is stopped"):
@@ -325,7 +269,7 @@ in
                  machine.succeed("systemctl start incus-startup.service")


        cleanup()
          machine.cleanup()
        ''
      ) "" initVariants
    )
@@ -354,7 +298,7 @@ in

          with subtest("virtual-machine can be launched and become available"):
              machine.succeed(f"incus start vm-{variant}1")
            wait_for_instance(f"vm-{variant}1")
              machine.wait_for_instance(f"vm-{variant}1")


          with subtest("virtual-machine incus-agent is started"):
@@ -366,18 +310,18 @@ in


          with subtest("virtual-machine CPU limits can be managed"):
            set_config(f"vm-{variant}1", "limits.cpu 1", restart=True)
            wait_incus_exec_success(f"vm-{variant}1", "nproc | grep '^1$'", timeout=90)
              machine.set_instance_config(f"vm-{variant}1", "limits.cpu 1", restart=True)
              machine.wait_instance_exec_success(f"vm-{variant}1", "nproc | grep '^1$'", timeout=90)


          with subtest("virtual-machine CPU limits can be hotplug changed"):
            set_config(f"vm-{variant}1", "limits.cpu 2")
            wait_incus_exec_success(f"vm-{variant}1", "nproc | grep '^2$'", timeout=90)
              machine.set_instance_config(f"vm-{variant}1", "limits.cpu 2")
              machine.wait_instance_exec_success(f"vm-{variant}1", "nproc | grep '^2$'", timeout=90)


          with subtest("virtual-machine can successfully restart"):
              machine.succeed(f"incus restart vm-{variant}1")
            wait_for_instance(f"vm-{variant}1")
              machine.wait_for_instance(f"vm-{variant}1")


          with subtest("virtual-machine remains running when softDaemonRestart is enabled and service is stopped"):
@@ -396,7 +340,7 @@ in
                  machine.succeed("systemctl start incus-startup.service")


        cleanup()
          machine.cleanup()
        ''
      ) "" initVariants)
      +
@@ -407,7 +351,7 @@ in
              machine.succeed("incus start csm")


        cleanup()
          machine.cleanup()
        ''
    )
    +
@@ -425,9 +369,9 @@ in
          with subtest("incus-user allows users to launch instances"):
              machine.succeed("su - testuser bash -c 'incus image import ${(images "systemd").container.metadata} ${(images "systemd").container.rootfs} --alias nixos'")
              machine.succeed("su - testuser bash -c 'incus launch nixos instance2'")
            wait_for_instance("instance2", "user-1000")
              machine.wait_for_instance("instance2", "user-1000")

        cleanup()
          machine.cleanup()
        ''
    +
      lib.optionalString cfg.network.ovs # python
+92 −0
Original line number Diff line number Diff line
import json


class IncusHost(Machine):
    def __init__(self, base):
        with subtest("Wait for startup"):
            base.wait_for_unit("incus.service")
            base.wait_for_unit("incus-preseed.service")

        with subtest("Verify preseed resources created"):
            base.succeed("incus profile show default")
            base.succeed("incus network info incusbr0")
            base.succeed("incus storage show default")

        self._parent = base

    # delegate attribute access to the parent
    def __getattr__(self, name):
        return getattr(self._parent, name)

    def instance_exec(self, name: str, command: str, project: str = "default"):
        return super().execute(
            f"incus exec {name} --disable-stdin --force-interactive --project {project} -- {command}"
        )

    def instance_succeed(self, name: str, command: str, project: str = "default"):
        return super().succeed(
            f"incus exec {name} --disable-stdin --force-interactive --project {project} -- {command}"
        )

    def wait_for_instance(self, name: str, project: str = "default"):
        self.wait_instance_exec_success(
            name,
            "/run/current-system/sw/bin/systemctl is-system-running",
            project=project,
        )

    def wait_instance_exec_success(
        self, name: str, command: str, timeout: int = 900, project: str = "default"
    ):
        def check_command(_) -> bool:
            status, _ = self.instance_exec(name, command, project)
            return status == 0

        with super().nested(
            f"Waiting for successful instance exec, instance={name}, project={project}, command={command}"
        ):
            retry(check_command, timeout)

    def set_instance_config(
        self, name: str, config: str, restart: bool = False, unset: bool = False
    ):
        if restart:
            super().succeed(f"incus stop {name}")

        if unset:
            super().succeed(f"incus config unset {name} {config}")
        else:
            super().succeed(f"incus config set {name} {config}")

        if restart:
            super().succeed(f"incus start {name}")
            self.wait_for_instance(name)
        else:
            # give a moment to settle
            super().sleep(1)

    def cleanup(self):
        # avoid conflict between preseed and cleanup operations
        super().execute("systemctl kill incus-preseed.service")

        instances = json.loads(
            super().succeed("incus list --format json --all-projects")
        )
        for instance in [a for a in instances if a["status"] == "Running"]:
            super().execute(
                f"incus stop --force {instance['name']} --project {instance['project']}"
            )
            super().execute(
                f"incus delete --force {instance['name']} --project {instance['project']}"
            )

    def check_instance_sysctl(self, name: str, project: str = "default"):
        self.instance_succeed(name, "systemctl status systemd-sysctl", project)
        sysctl = (
            self.instance_succeed(name, "sysctl net.ipv4.ip_forward", project)
            .strip()
            .split(" ")[-1]
        )
        assert (
            "1" == sysctl
        ), f"systemd-sysctl configuration not correctly applied, {sysctl} != 1"