Merge: postgresql: improve passthru.tests (#352966)

/pkgs/servers/sql/postgresql @NixOS/postgres

/nixos/modules/services/databases/postgresql.md @NixOS/postgres

/nixos/modules/services/databases/postgresql.nix @NixOS/postgres

/nixos/tests/postgresql.nix @NixOS/postgres

/nixos/tests/postgresql @NixOS/postgres

# Hardened profile & related modules

/nixos/modules/profiles/hardened.nix                       @joachifm

  peering-manager = handleTest ./web-apps/peering-manager.nix {};

  peertube = handleTestOn ["x86_64-linux"] ./web-apps/peertube.nix {};

  peroxide = handleTest ./peroxide.nix {};

  pg_anonymizer = handleTest ./pg_anonymizer.nix {};

  pgadmin4 = handleTest ./pgadmin4.nix {};

  pgbouncer = handleTest ./pgbouncer.nix {};

  pghero = runTest ./pghero.nix;

  pgjwt = handleTest ./pgjwt.nix {};

  pgmanage = handleTest ./pgmanage.nix {};

  pgvecto-rs = handleTest ./pgvecto-rs.nix {};

  phosh = handleTest ./phosh.nix {};

  photonvision = handleTest ./photonvision.nix {};

  photoprism = handleTest ./photoprism.nix {};

  postfix = handleTest ./postfix.nix {};

  postfix-raise-smtpd-tls-security-level = handleTest ./postfix-raise-smtpd-tls-security-level.nix {};

  postfixadmin = handleTest ./postfixadmin.nix {};

  postgis = handleTest ./postgis.nix {};

  apache_datasketches = handleTest ./apache_datasketches.nix {};

  postgresql = handleTest ./postgresql.nix {};

  postgresql-jit = handleTest ./postgresql-jit.nix {};

  postgresql-wal-receiver = handleTest ./postgresql-wal-receiver.nix {};

  postgresql-tls-client-cert = handleTest ./postgresql-tls-client-cert.nix {};

  postgresql-wal2json = handleTest ./postgresql-wal2json.nix {};

  postgresql = handleTest ./postgresql {};

  powerdns = handleTest ./powerdns.nix {};

  powerdns-admin = handleTest ./powerdns-admin.nix {};

  power-profiles-daemon = handleTest ./power-profiles-daemon.nix {};

  tiddlywiki = handleTest ./tiddlywiki.nix {};

  tigervnc = handleTest ./tigervnc.nix {};

  tika = runTest ./tika.nix;

  timescaledb = handleTest ./timescaledb.nix {};

  timezone = handleTest ./timezone.nix {};

  timidity = handleTestOn ["aarch64-linux" "x86_64-linux"] ./timidity {};

  tinc = handleTest ./tinc {};

  trezord = handleTest ./trezord.nix {};

  trickster = handleTest ./trickster.nix {};

  trilium-server = handleTestOn ["x86_64-linux"] ./trilium-server.nix {};

  tsja = handleTest ./tsja.nix {};

  tsm-client-gui = handleTest ./tsm-client-gui.nix {};

  ttyd = handleTest ./web-servers/ttyd.nix {};

  txredisapi = handleTest ./txredisapi.nix {};

import ./make-test-python.nix ({ pkgs, ...} : {

  name = "postgis";

  meta = with pkgs.lib.maintainers; {

    maintainers = [ lsix ]; # TODO: Who's the maintener now?

  };

  nodes = {

    master =

      { pkgs, ... }:

      {

        services.postgresql = let mypg = pkgs.postgresql_15; in {

            enable = true;

            package = mypg;

            extraPlugins = with mypg.pkgs; [

              apache_datasketches

            ];

        };

      };

  };

  testScript = ''

    start_all()

    master.wait_for_unit("postgresql")

    master.sleep(10)  # Hopefully this is long enough!!

    master.succeed("sudo -u postgres psql -c 'CREATE EXTENSION datasketches;'")

    master.succeed("sudo -u postgres psql -c 'SELECT hll_sketch_to_string(hll_sketch_build(1));'")

  '';

})

import ./make-test-python.nix ({ pkgs, lib, ... }: {

  name = "pg_anonymizer";

  meta.maintainers = lib.teams.flyingcircus.members;

  nodes.machine = { pkgs, ... }: {

    environment.systemPackages = [ pkgs.pg-dump-anon ];

    services.postgresql = {

      enable = true;

      extraPlugins = ps: [ ps.anonymizer ];

      settings.shared_preload_libraries = [ "anon" ];

    };

  };

  testScript = ''

    start_all()

    machine.wait_for_unit("multi-user.target")

    machine.wait_for_unit("postgresql.service")

    with subtest("Setup"):

        machine.succeed("sudo -u postgres psql --command 'create database demo'")

        machine.succeed(

            "sudo -u postgres psql -d demo -f ${pkgs.writeText "init.sql" ''

              create extension anon cascade;

              select anon.init();

              create table player(id serial, name text, points int);

              insert into player(id,name,points) values (1,'Foo', 23);

              insert into player(id,name,points) values (2,'Bar',42);

              security label for anon on column player.name is 'MASKED WITH FUNCTION anon.fake_last_name();';

              security label for anon on column player.points is 'MASKED WITH VALUE NULL';

            ''}"

        )

    def get_player_table_contents():

        return [

            x.split(',') for x in machine.succeed("sudo -u postgres psql -d demo --csv --command 'select * from player'").splitlines()[1:]

        ]

    def check_anonymized_row(row, id, original_name):

        assert row[0] == id, f"Expected first row to have ID {id}, but got {row[0]}"

        assert row[1] != original_name, f"Expected first row to have a name other than {original_name}"

        assert not bool(row[2]), "Expected points to be NULL in first row"

    def find_xsv_in_dump(dump, sep=','):

        """

        Expecting to find a CSV (for pg_dump_anon) or TSV (for pg_dump) structure, looking like

            COPY public.player ...

            1,Shields,

            2,Salazar,

            \.

        in the given dump (the commas are tabs in case of pg_dump).

              Extract the CSV lines and split by `sep`.

        """

        try:

            from itertools import dropwhile, takewhile

            return [x.split(sep) for x in list(takewhile(

                lambda x: x != "\\.",

                dropwhile(

                    lambda x: not x.startswith("COPY public.player"),

                    dump.splitlines()

                )

            ))[1:]]

        except:

            print(f"Dump to process: {dump}")

            raise

    def check_original_data(output):

        assert output[0] == ['1','Foo','23'], f"Expected first row from player table to be 1,Foo,23; got {output[0]}"

        assert output[1] == ['2','Bar','42'], f"Expected first row from player table to be 2,Bar,42; got {output[1]}"

    def check_anonymized_rows(output):

        check_anonymized_row(output[0], '1', 'Foo')

        check_anonymized_row(output[1], '2', 'Bar')

    with subtest("Check initial state"):

        check_original_data(get_player_table_contents())

    with subtest("Anonymous dumps"):

        check_original_data(find_xsv_in_dump(

            machine.succeed("sudo -u postgres pg_dump demo"),

            sep='\t'

        ))

        check_anonymized_rows(find_xsv_in_dump(

            machine.succeed("sudo -u postgres pg_dump_anon -U postgres -h /run/postgresql -d demo"),

            sep=','

        ))

    with subtest("Anonymize"):

        machine.succeed("sudo -u postgres psql -d demo --command 'select anon.anonymize_database();'")

        check_anonymized_rows(get_player_table_contents())

  '';

})

import ./make-test-python.nix ({ pkgs, lib, ...}:

with pkgs; {

  name = "pgjwt";

  meta = with lib.maintainers; {

    maintainers = [ spinus willibutz ];

  };

  nodes = {

    master = { ... }:

    {

      services.postgresql = {

        enable = true;

        extraPlugins = ps: with ps; [ pgjwt pgtap ];

      };

    };

  };

  testScript = { nodes, ... }:

  let

    sqlSU = "${nodes.master.services.postgresql.superUser}";

    pgProve = "${pkgs.perlPackages.TAPParserSourceHandlerpgTAP}";

    inherit (nodes.master.services.postgresql.package.pkgs) pgjwt;

  in

  ''

    start_all()

    master.wait_for_unit("postgresql")

    master.succeed(

        "${pkgs.gnused}/bin/sed -e '12 i CREATE EXTENSION pgcrypto;\\nCREATE EXTENSION pgtap;\\nSET search_path TO tap,public;' ${pgjwt.src}/test.sql > /tmp/test.sql"

    )

    master.succeed(

        "${pkgs.sudo}/bin/sudo -u ${sqlSU} PGOPTIONS=--search_path=tap,public ${pgProve}/bin/pg_prove -d postgres -v -f /tmp/test.sql"

    )

  '';

})