nixos/windmill: Add module test (#460410)

  whoami = runTest ./whoami.nix;

  whoogle-search = runTest ./whoogle-search.nix;

  wiki-js = runTest ./wiki-js.nix;

  windmill = import ./windmill {

    inherit pkgs runTest;

    inherit (pkgs) lib;

  };

  wine = handleTest ./wine.nix { };

  wireguard = import ./wireguard {

    inherit pkgs runTest;

{ lib, pkgs, ... }:

{

  name = "windmill-api";

  nodes = {

    windmill =

      { pkgs, lib, ... }:

      {

        virtualisation.memorySize = 2048; # 2GB

        networking.firewall.allowedTCPPorts = [ 8001 ];

        services.windmill = {

          enable = true;

          serverPort = 8001;

          database.createLocally = true;

        };

        systemd.services = {

          windmill-worker = {

            # "dotnet restore" (csharp executor) attempts to download the package index from api.nuget.org by default.

            # Using the following per-job nuget.config file will erase all package source references to prevent downloading.

            environment.FORCE_NUGET_CONFIG = ''

              <?xml version="1.0" encoding="utf-8"?>

              <configuration>

                <packageSources>

                    <clear />

                </packageSources>

              </configuration>

            '';

          };

        };

        environment.systemPackages = [

          (pkgs.writers.writePython3Bin "integration-test" {

            flakeIgnore = [

              "E265" # block comment should start with '# '

              "E501" # line too long (NN > 79 characters)

            ];

          } (builtins.readFile ./api-integration.py))

        ];

      };

  };

  testScript = ''

    import time

    # windmill.forward_port(8001, 8001) # DEBUG interactive

    with subtest("Server smoketest"):

      windmill.wait_for_unit("windmill.target")

      # ERROR; Do not early timeout because windmill starts running migration scripts.

      # There is no communication to systemd that signals migrations have finished.

      windmill.wait_for_open_port(8001)

      # NOTE; Wait a couple of seconds for all windmill components to finalise their database migration flow. This prevents race conditions on schema constraints.

      time.sleep(10)  # seconds

      windmill.succeed("curl --silent --fail http://windmill:8001")

      t.assertIn("v${pkgs.windmill.version}", machine.succeed("curl --silent --fail http://windmill:8001/api/version"), "Mismatched version response")

    with subtest("Validation"):

      windmill.succeed("integration-test --language python3 --script ${./python3.script} --input ${./python3.input}")

      windmill.succeed("integration-test --language go --script ${./go.script} --input ${./go.input}")

      windmill.succeed("integration-test --language bun --script ${./bun.script} --input ${./bun.input}")

      windmill.succeed("integration-test --language deno --script ${./deno.script} --input ${./deno.input}")

      windmill.succeed("integration-test --language php --script ${./php.script} --input ${./php.input}")

      windmill.succeed("integration-test --language bash --script ${./bash.script} --input ${./bash.input}")

      windmill.succeed("integration-test --language powershell --script ${./powershell.script} --input ${./powershell.input}")

      # ERROR; "dotnet restore" requires write-access to <some-path(:home?)>/.dotnet

      # -- System.IO.IOException: Read-only file system : '/.dotnet'

      #

      # ERROR; "dotnet publish" requires internet connectivity to fetch "compilation workload" dependencies

      # -- error NU1100: Unable to resolve 'Microsoft.NET.ILLink.Tasks (>= 9.0.10)' for 'net9.0'.

      #

      # DISABLED windmill.succeed("integration-test --language csharp --script ${./csharp.script} --input ${./csharp.input}")

  '';

}

#!/usr/bin/env nix

#!nix shell nixpkgs#python3 --command python

from argparse import ArgumentParser

import pathlib

from urllib.request import build_opener, HTTPCookieProcessor, Request

from http.cookiejar import CookieJar

import json

import time

parser = ArgumentParser()

parser.add_argument("-l", "--language", dest="language", type=str,

                    help="Name of the scripting language", metavar="LANG", required=True)

parser.add_argument("-s", "--script", dest="content_file", type=pathlib.Path,

                    help="read script contents from FILE", metavar="FILE", required=True)

parser.add_argument("-i", "--input", dest="input_file", type=pathlib.Path,

                    help="read script arguments from FILE", metavar="FILE", required=True)

args = parser.parse_args()

cookiejar = CookieJar()

cookieprocessor = HTTPCookieProcessor(cookiejar)

http_client = build_opener(cookieprocessor)

admin_token = None

id_admin_workspace = "admins"

login_form = {"email": "admin@windmill.dev", "password": "changeme"}

login_req = Request(

    "http://localhost:8001/api/auth/login",

    method="POST",

    headers={'Content-Type': 'application/json'},

    data=json.dumps(login_form).encode('utf-8')

)

with http_client.open(login_req) as response:

    assert 200 == response.status, f"Failure {response.status}: Superuser login"

    assert any(cookie.name == "token" for cookie in cookiejar)

    admin_token = next(cookie.value for cookie in cookiejar if cookie.name == "token")

    assert admin_token, "Failed to receive a session key from admin login"

if "python" in args.language:

    # Windmill package recipe requires a manually set default python version in the global instance settings

    python_version_form = {

        # NOTE; Update hardcoded python version below to match the windmill package

        "value": "3.12"

    }

    python_version_req = Request(

        "http://localhost:8001/api/settings/global/instance_python_version",

        method="GET",

        headers={'Authorization': f'Bearer {admin_token}'},

        data=json.dumps(python_version_form).encode('utf-8')

    )

    with http_client.open(python_version_req) as response:

        assert 200 == response.status, f"Failure {response.status}: Update global instance python version."

workspace_req = Request(

    "http://localhost:8001/api/workspaces/list_as_superadmin",

    method="GET",

    headers={'Authorization': f'Bearer {admin_token}'}

)

with http_client.open(workspace_req) as response:

    assert 200 == response.status, f"Failure {response.status}: List workspaces"

    workspace_list = json.loads(response.read().decode('utf-8'))

    assert any(workspace['id'] == id_admin_workspace for workspace in workspace_list)

script_hash = None

script_form = {

    "path": f"u/admin/{args.language}_test",

    "summary": f"Test {args.language}",

    "description": "",

    "language": args.language,

    "content": args.content_file.read_text()

}

script_request = Request(

    f"http://localhost:8001/api/w/{id_admin_workspace}/scripts/create",

    method="POST",

    headers={

        'Content-Type': 'application/json',

        'Authorization': f'Bearer {admin_token}',

    },

    data=json.dumps(script_form).encode('utf-8')

)

with http_client.open(script_request) as response:

    assert 201 == response.status, f"Failure {response.status}: Create {args.language} script"

    script_hash = response.read().decode('utf-8')

    assert script_hash, "Failed to receive an identifier from script creation."

# NOTE; Some languages require dependencies and the depenceny collection tasks take some time to complete

if "bash" not in args.language:

    for i in [1, 2, 3, 4, 5, 6]:

        time.sleep(10)  # seconds

        script_request = Request(

            f"http://localhost:8001/api/w/{id_admin_workspace}/scripts/deployment_status/h/{script_hash}",

            method="GET",

        )

        with http_client.open(script_request) as response:

            try:

                assert 200 == response.status, f"Failure {response.status}: Retrieve {args.language} deployment status"

                script_metadata = json.loads(response.read().decode('utf-8'))

                #

                exists_lock_error = bool(script_metadata["lock_error_logs"])

                assert not exists_lock_error, "Script deployment did not succeed"

                is_deployment_success = script_metadata["lock"] is not None

                if is_deployment_success:

                    break

            except AssertionError:

                # Re-raise error on final attempt

                if i == 6:

                    raise

job_id = None

request = Request(

    f"http://localhost:8001/api/w/{id_admin_workspace}/jobs/run/h/{script_hash}",

    method="POST",

    headers={

        'Content-Type': 'application/json',

        'Authorization': f'Bearer {admin_token}',

    },

    data=args.input_file.read_bytes(),

)

with http_client.open(request) as response:

    assert 201 == response.status, f"Failure {response.status}: Run {args.language} script"

    job_id = response.read().decode('utf-8')

    assert job_id, "Failed to receive an identifier from job creation/scheduling."

started_jobs = set([job_id])

# NOTE; Some languages require script compilation and take longer to run until completion

timeout = 60  # seconds

timeout_end = time.time() + timeout

while any(started_jobs) and time.time() < timeout_end:

    time.sleep(10)  # seconds

    retrieve_jobs_req = Request(

        f"http://localhost:8001/api/w/{id_admin_workspace}/jobs/completed/list",

        method="GET",

        headers={

            'Authorization': f'Bearer {admin_token}',

        },

    )

    with http_client.open(retrieve_jobs_req) as response:

        assert 200 == response.status, f"Failure {response.status}: Retrieve jobs"

        job_results = json.loads(response.read().decode('utf-8'))

        for id in set(started_jobs):  # Must create copy of set being iterated over

            if any(job['id'] == id for job in job_results if bool(job['success'])):

                started_jobs.remove(id)

if any(started_jobs):

    # There are started jobs that have not completed before timeout

    exit(1)

{"dflt": "default value","msg": ""}

# shellcheck shell=bash

# arguments of the form X="$I" are parsed as parameters X of type string

msg="$1"

dflt="${2:-default value}"

# WARN; Cannot download dependency modules from internet during sandbox testing!

# the last line of the stdout is the return value

# unless you write json to './result.json' or a string to './result.out'

echo "Hello $msg"