Merge staging-next into staging

    name = "Hubert Jasudowicz";

  };

  chkno = {

    email = "chuck@intelligence.org";

    email = "scottworley@scottworley.com";

    github = "chkno";

    githubId = 1118859;

    name = "Scott Worley";

- The EC2 image module no longer fetches instance metadata in stage-1. This results in a significantly smaller initramfs, since network drivers no longer need to be included, and faster boots, since metadata fetching can happen in parallel with startup of other services.

  This breaks services which rely on metadata being present by the time stage-2 is entered. Anything which reads EC2 metadata from `/etc/ec2-metadata` should now have an `after` dependency on `fetch-ec2-metadata.service`

- The mailman service now defaults to using a randomly generated REST API password instead of a hardcoded one.

- `minio` removed support for its legacy filesystem backend in [RELEASE.2022-10-29T06-21-33Z](https://github.com/minio/minio/releases/tag/RELEASE.2022-10-29T06-21-33Z). This means if your storage was created with the old format, minio will no longer start. Unfortunately minio doesn't provide a an automatic migration, they only provide [instructions how to manually convert the node](https://min.io/docs/minio/windows/operations/install-deploy-manage/migrate-fs-gateway.html). To facilitate this migration we keep around the last version that still supports the old filesystem backend as `minio_legacy_fs`. Use it via `services.minio.package = minio_legacy_fs;` to export your data before switching to the new version. See the corresponding [issue](https://github.com/NixOS/nixpkgs/issues/199318) for more details.

- `services.sourcehut.dispatch` and the corresponding package (`sourcehut.dispatchsrht`) have been removed due to [upstream deprecation](https://sourcehut.org/blog/2022-08-01-dispatch-deprecation-plans/).

    transport_file_type: hash

  '';

  mailmanCfg = lib.generators.toINI {}

    (recursiveUpdate cfg.settings

      ((optionalAttrs (cfg.restApiPassFile != null) {

  mailmanCfg = lib.generators.toINI {} (recursiveUpdate cfg.settings {

    webservice.admin_pass = "#NIXOS_MAILMAN_REST_API_PASS_SECRET#";

      })));

  });

  mailmanCfgFile = pkgs.writeText "mailman-raw.cfg" mailmanCfg;

    environment.etc."mailman3/settings.py".text = ''

      import os

      from configparser import ConfigParser

      # Required by mailman_web.settings, but will be overridden when

      # settings_local.json is loaded.

      with open('/var/lib/mailman-web/settings_local.json') as f:

          globals().update(json.load(f))

      ${optionalString (cfg.restApiPassFile != null) ''

        with open('${cfg.restApiPassFile}') as f:

            MAILMAN_REST_API_PASS = f.read().rstrip('\n')

      ''}

      with open('/etc/mailman.cfg') as f:

          config = ConfigParser()

          config.read_file(f)

          MAILMAN_REST_API_PASS = config['webservice']['admin_pass']

      ${optionalString (cfg.ldap.enable) ''

        import ldap

        path = with pkgs; [ jq ];

        after = optional withPostgresql "postgresql.service";

        requires = optional withPostgresql "postgresql.service";

        serviceConfig.RemainAfterExit = true;

        serviceConfig.Type = "oneshot";

        script = ''

          install -m0750 -o mailman -g mailman ${mailmanCfgFile} /etc/mailman.cfg

          ${optionalString (cfg.restApiPassFile != null) ''

          ${if cfg.restApiPassFile == null then ''

            sed -i "s/#NIXOS_MAILMAN_REST_API_PASS_SECRET#/$(tr -dc A-Za-z0-9 < /dev/urandom | head -c 64)/g" \

              /etc/mailman.cfg

          '' else ''

            ${pkgs.replace-secret}/bin/replace-secret \

              '#NIXOS_MAILMAN_REST_API_PASS_SECRET#' \

              ${cfg.restApiPassFile} \

  magnetico = handleTest ./magnetico.nix {};

  mailcatcher = handleTest ./mailcatcher.nix {};

  mailhog = handleTest ./mailhog.nix {};

  mailman = handleTest ./mailman.nix {};

  man = handleTest ./man.nix {};

  mariadb-galera = handleTest ./mysql/mariadb-galera.nix {};

  mastodon = discoverTests (import ./web-apps/mastodon { inherit handleTestOn; });

import ./make-test-python.nix {

  name = "mailman";

  nodes.machine = { pkgs, ... }: {

    environment.systemPackages = with pkgs; [ mailutils ];

    services.mailman.enable = true;

    services.mailman.serve.enable = true;

    services.mailman.siteOwner = "postmaster@example.com";

    services.mailman.webHosts = [ "example.com" ];

    services.postfix.enable = true;

    services.postfix.destination = [ "example.com" "example.net" ];

    services.postfix.relayDomains = [ "hash:/var/lib/mailman/data/postfix_domains" ];

    services.postfix.config.local_recipient_maps = [ "hash:/var/lib/mailman/data/postfix_lmtp" "proxy:unix:passwd.byname" ];

    services.postfix.config.transport_maps = [ "hash:/var/lib/mailman/data/postfix_lmtp" ];

    users.users.user = { isNormalUser = true; };

    virtualisation.memorySize = 2048;

    specialisation.restApiPassFileSystem.configuration = {

      services.mailman.restApiPassFile = "/var/lib/mailman/pass";

    };

  };

  testScript = { nodes, ... }: let

    restApiPassFileSystem = "${nodes.machine.system.build.toplevel}/specialisation/restApiPassFileSystem";

  in ''

    def check_mail(_) -> bool:

        status, _ = machine.execute("grep -q hello /var/spool/mail/user/new/*")

        return status == 0

    def try_api(_) -> bool:

        status, _ = machine.execute("curl -s http://localhost:8001/")

        return status == 0

    def wait_for_api():

        with machine.nested("waiting for Mailman REST API to be available"):

            retry(try_api)

    machine.wait_for_unit("mailman.service")

    wait_for_api()

    with subtest("subscription and delivery"):

        creds = machine.succeed("su -s /bin/sh -c 'mailman info' mailman | grep '^REST credentials: ' | sed 's/^REST credentials: //'").strip()

        machine.succeed(f"curl --fail-with-body -sLSu {creds} -d mail_host=example.com http://localhost:8001/3.1/domains")

        machine.succeed(f"curl --fail-with-body -sLSu {creds} -d fqdn_listname=list@example.com http://localhost:8001/3.1/lists")

        machine.succeed(f"curl --fail-with-body -sLSu {creds} -d list_id=list.example.com -d subscriber=root@example.com -d pre_confirmed=True -d pre_verified=True -d send_welcome_message=False http://localhost:8001/3.1/members")

        machine.succeed(f"curl --fail-with-body -sLSu {creds} -d list_id=list.example.com -d subscriber=user@example.net -d pre_confirmed=True -d pre_verified=True -d send_welcome_message=False http://localhost:8001/3.1/members")

        machine.succeed("mail -a 'From: root@example.com' -s hello list@example.com < /dev/null")

        with machine.nested("waiting for mail from list"):

            retry(check_mail)

    with subtest("Postorius"):

        machine.succeed("curl --fail-with-body -sILS http://localhost/")

    with subtest("restApiPassFile"):

        machine.succeed("echo secretpassword > /var/lib/mailman/pass")

        machine.succeed("${restApiPassFileSystem}/bin/switch-to-configuration test >&2")

        machine.succeed("grep secretpassword /etc/mailman.cfg")

        machine.succeed("su -s /bin/sh -c 'mailman info' mailman | grep secretpassword")

        wait_for_api()

        machine.succeed("curl --fail-with-body -sLSu restadmin:secretpassword http://localhost:8001/3.1/domains")

        machine.succeed("curl --fail-with-body -sILS http://localhost/")

  '';

}