Unverified Commit 0cdaede1 authored by Maximilian Bosch's avatar Maximilian Bosch
Browse files

pg-dump-anon: init at 1.3.1 

This is a Go program inside the sources of `postgresql_anonymizer` that
allows to perform database dumps, but with anonymized data. I figured
that it's a little awkward to have a client program to be part of the
extension package.

So I decided to create a second package called `pg-dump-anon`. Since
it's one repository, both share `version` & `src`.

Also extended the VM test to make sure we're getting properly anonymized
data when dumping with `pg_dump_anon`.
parent 8d0e5a34

nixos/tests/pg_anonymizer.nix

+47 −7
Original line number Diff line number Diff line
@@ -2,7 +2,8 @@ import ./make-test-python.nix ({ pkgs, lib, ... }: { 
  name = "pg_anonymizer";

  meta.maintainers = lib.teams.flyingcircus.members;



  nodes.machine = {

  nodes.machine = { pkgs, ... }: {

    environment.systemPackages = [ pkgs.pg-dump-anon ];

    services.postgresql = {

      enable = true;

      extraPlugins = ps: [ ps.anonymizer ];
@@ -39,16 +40,55 @@ import ./make-test-python.nix ({ pkgs, lib, ... }: { 
        assert row[1] != original_name, f"Expected first row to have a name other than {original_name}"

        assert not bool(row[2]), "Expected points to be NULL in first row"



    with subtest("Check initial state"):

        output = get_player_table_contents()

    def find_xsv_in_dump(dump, sep=','):

        """

        Expecting to find a CSV (for pg_dump_anon) or TSV (for pg_dump) structure, looking like



            COPY public.player ...

            1,Shields,

            2,Salazar,

            \.



        in the given dump (the commas are tabs in case of pg_dump).

              Extract the CSV lines and split by `sep`.

        """



        try:

            from itertools import dropwhile, takewhile

            return [x.split(sep) for x in list(takewhile(

                lambda x: x != "\\.",

                dropwhile(

                    lambda x: not x.startswith("COPY public.player"),

                    dump.splitlines()

                )

            ))[1:]]

        except:

            print(f"Dump to process: {dump}")

            raise



    def check_original_data(output):

        assert output[0] == ['1','Foo','23'], f"Expected first row from player table to be 1,Foo,23; got {output[0]}"

        assert output[1] == ['2','Bar','42'], f"Expected first row from player table to be 2,Bar,42; got {output[1]}"



    with subtest("Anonymize"):

        machine.succeed("sudo -u postgres psql -d demo --command 'select anon.anonymize_database();'")

        output = get_player_table_contents()



    def check_anonymized_rows(output):

        check_anonymized_row(output[0], '1', 'Foo')

        check_anonymized_row(output[1], '2', 'Bar')



    with subtest("Check initial state"):

        check_original_data(get_player_table_contents())



    with subtest("Anonymous dumps"):

        check_original_data(find_xsv_in_dump(

            machine.succeed("sudo -u postgres pg_dump demo"),

            sep='\t'

        ))

        check_anonymized_rows(find_xsv_in_dump(

            machine.succeed("sudo -u postgres pg_dump_anon -U postgres -h /run/postgresql -d demo"),

            sep=','

        ))



    with subtest("Anonymize"):

        machine.succeed("sudo -u postgres psql -d demo --command 'select anon.anonymize_database();'")

        check_anonymized_rows(get_player_table_contents())

  '';

})

pkgs/by-name/pg/pg-dump-anon/package.nix

0 → 100644
+32 −0
Original line number Diff line number Diff line 
{ lib, fetchFromGitLab, buildGoModule, nixosTests, postgresql, makeWrapper }:



buildGoModule rec {

  pname = "pg-dump-anon";

  version = "1.3.1";

  src = fetchFromGitLab {

    owner = "dalibo";

    repo = "postgresql_anonymizer";

    rev = version;

    hash = "sha256-Z5Oz/cIYDxFUZwQijRk4xAOUdOK0LWR+px8WOcs+Rs0=";

  };



  sourceRoot = "${src.name}/pg_dump_anon";



  vendorHash = "sha256-CwU1zoIayxvfnGL9kPdummPJiV+ECfSz4+q6gZGb8pw=";



  passthru.tests = { inherit (nixosTests) pg_anonymizer; };



  nativeBuildInputs = [ makeWrapper ];

  postInstall = ''

    wrapProgram $out/bin/pg_dump_anon \

      --prefix PATH : ${lib.makeBinPath [ postgresql ]}

  '';



  meta = with lib; {

    description = "Export databases with data being anonymized with the anonymizer extension";

    homepage = "https://postgresql-anonymizer.readthedocs.io/en/stable/";

    maintainers = teams.flyingcircus.members;

    license = licenses.postgresql;

    mainProgram = "pg_dump_anon";

  };

}

pkgs/servers/sql/postgresql/ext/anonymizer.nix

+11 −13
Original line number Diff line number Diff line 
{ lib, stdenv, fetchFromGitLab, postgresql, nixosTests, ... }:

{ lib, stdenv, pg-dump-anon, postgresql, runtimeShell }:



stdenv.mkDerivation (finalAttrs: {

  pname = "postgresql_anonymizer";

  version = "1.3.1";



  src = fetchFromGitLab {

    owner = "dalibo";

    repo = "postgresql_anonymizer";

    rev = finalAttrs.version;

    hash = "sha256-Z5Oz/cIYDxFUZwQijRk4xAOUdOK0LWR+px8WOcs+Rs0=";

  };

  inherit (pg-dump-anon) version src passthru;



  buildInputs = [ postgresql ];

  nativeBuildInputs = [ postgresql ] ++ lib.optional postgresql.jitSupport postgresql.llvm;
@@ -23,12 +17,16 @@ stdenv.mkDerivation (finalAttrs: { 
    "DESTDIR="

  ];



  passthru.tests = { inherit (nixosTests) pg_anonymizer; };

  postInstall = ''

    cat >$out/bin/pg_dump_anon.sh <<'EOF'

    #!${runtimeShell}

    echo "This script is deprecated by upstream. To use the new script,"

    echo "please install pkgs.pg-dump-anon."

    exit 1

    EOF

  '';



  meta = with lib; {

  meta = pg-dump-anon.meta // {

    description = "postgresql_anonymizer is an extension to mask or replace personally identifiable information (PII) or commercially sensitive data from a PostgreSQL database.";

    homepage = "https://postgresql-anonymizer.readthedocs.io/en/stable/";

    maintainers = teams.flyingcircus.members;

    license = licenses.postgresql;

  };

})