diff --git a/ci/OWNERS b/ci/OWNERS index 8e227b1525668..b8aa53105fc94 100644 --- a/ci/OWNERS +++ b/ci/OWNERS @@ -229,7 +229,7 @@ pkgs/development/python-modules/buildcatrust/ @ajs124 @lukegb @mweinelt /pkgs/servers/sql/postgresql @NixOS/postgres /nixos/modules/services/databases/postgresql.md @NixOS/postgres /nixos/modules/services/databases/postgresql.nix @NixOS/postgres -/nixos/tests/postgresql.nix @NixOS/postgres +/nixos/tests/postgresql @NixOS/postgres # Hardened profile & related modules /nixos/modules/profiles/hardened.nix @joachifm diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index 1288955272aa3..37e005f128a2e 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -775,13 +775,10 @@ in { peering-manager = handleTest ./web-apps/peering-manager.nix {}; peertube = handleTestOn ["x86_64-linux"] ./web-apps/peertube.nix {}; peroxide = handleTest ./peroxide.nix {}; - pg_anonymizer = handleTest ./pg_anonymizer.nix {}; pgadmin4 = handleTest ./pgadmin4.nix {}; pgbouncer = handleTest ./pgbouncer.nix {}; pghero = runTest ./pghero.nix; - pgjwt = handleTest ./pgjwt.nix {}; pgmanage = handleTest ./pgmanage.nix {}; - pgvecto-rs = handleTest ./pgvecto-rs.nix {}; phosh = handleTest ./phosh.nix {}; photonvision = handleTest ./photonvision.nix {}; photoprism = handleTest ./photoprism.nix {}; @@ -814,13 +811,7 @@ in { postfix = handleTest ./postfix.nix {}; postfix-raise-smtpd-tls-security-level = handleTest ./postfix-raise-smtpd-tls-security-level.nix {}; postfixadmin = handleTest ./postfixadmin.nix {}; - postgis = handleTest ./postgis.nix {}; - apache_datasketches = handleTest ./apache_datasketches.nix {}; - postgresql = handleTest ./postgresql.nix {}; - postgresql-jit = handleTest ./postgresql-jit.nix {}; - postgresql-wal-receiver = handleTest ./postgresql-wal-receiver.nix {}; - postgresql-tls-client-cert = handleTest ./postgresql-tls-client-cert.nix {}; - postgresql-wal2json = handleTest ./postgresql-wal2json.nix {}; + postgresql = handleTest ./postgresql {}; powerdns = handleTest ./powerdns.nix {}; powerdns-admin = handleTest ./powerdns-admin.nix {}; power-profiles-daemon = handleTest ./power-profiles-daemon.nix {}; @@ -1047,7 +1038,6 @@ in { tiddlywiki = handleTest ./tiddlywiki.nix {}; tigervnc = handleTest ./tigervnc.nix {}; tika = runTest ./tika.nix; - timescaledb = handleTest ./timescaledb.nix {}; timezone = handleTest ./timezone.nix {}; timidity = handleTestOn ["aarch64-linux" "x86_64-linux"] ./timidity {}; tinc = handleTest ./tinc {}; @@ -1067,7 +1057,6 @@ in { trezord = handleTest ./trezord.nix {}; trickster = handleTest ./trickster.nix {}; trilium-server = handleTestOn ["x86_64-linux"] ./trilium-server.nix {}; - tsja = handleTest ./tsja.nix {}; tsm-client-gui = handleTest ./tsm-client-gui.nix {}; ttyd = handleTest ./web-servers/ttyd.nix {}; txredisapi = handleTest ./txredisapi.nix {}; diff --git a/nixos/tests/apache_datasketches.nix b/nixos/tests/apache_datasketches.nix deleted file mode 100644 index 2bf099ac7991b..0000000000000 --- a/nixos/tests/apache_datasketches.nix +++ /dev/null @@ -1,29 +0,0 @@ -import ./make-test-python.nix ({ pkgs, ...} : { - name = "postgis"; - meta = with pkgs.lib.maintainers; { - maintainers = [ lsix ]; # TODO: Who's the maintener now? - }; - - nodes = { - master = - { pkgs, ... }: - - { - services.postgresql = let mypg = pkgs.postgresql_15; in { - enable = true; - package = mypg; - extraPlugins = with mypg.pkgs; [ - apache_datasketches - ]; - }; - }; - }; - - testScript = '' - start_all() - master.wait_for_unit("postgresql") - master.sleep(10) # Hopefully this is long enough!! - master.succeed("sudo -u postgres psql -c 'CREATE EXTENSION datasketches;'") - master.succeed("sudo -u postgres psql -c 'SELECT hll_sketch_to_string(hll_sketch_build(1));'") - ''; -}) diff --git a/nixos/tests/pg_anonymizer.nix b/nixos/tests/pg_anonymizer.nix deleted file mode 100644 index b26e4dca05809..0000000000000 --- a/nixos/tests/pg_anonymizer.nix +++ /dev/null @@ -1,94 +0,0 @@ -import ./make-test-python.nix ({ pkgs, lib, ... }: { - name = "pg_anonymizer"; - meta.maintainers = lib.teams.flyingcircus.members; - - nodes.machine = { pkgs, ... }: { - environment.systemPackages = [ pkgs.pg-dump-anon ]; - services.postgresql = { - enable = true; - extraPlugins = ps: [ ps.anonymizer ]; - settings.shared_preload_libraries = [ "anon" ]; - }; - }; - - testScript = '' - start_all() - machine.wait_for_unit("multi-user.target") - machine.wait_for_unit("postgresql.service") - - with subtest("Setup"): - machine.succeed("sudo -u postgres psql --command 'create database demo'") - machine.succeed( - "sudo -u postgres psql -d demo -f ${pkgs.writeText "init.sql" '' - create extension anon cascade; - select anon.init(); - create table player(id serial, name text, points int); - insert into player(id,name,points) values (1,'Foo', 23); - insert into player(id,name,points) values (2,'Bar',42); - security label for anon on column player.name is 'MASKED WITH FUNCTION anon.fake_last_name();'; - security label for anon on column player.points is 'MASKED WITH VALUE NULL'; - ''}" - ) - - def get_player_table_contents(): - return [ - x.split(',') for x in machine.succeed("sudo -u postgres psql -d demo --csv --command 'select * from player'").splitlines()[1:] - ] - - def check_anonymized_row(row, id, original_name): - assert row[0] == id, f"Expected first row to have ID {id}, but got {row[0]}" - assert row[1] != original_name, f"Expected first row to have a name other than {original_name}" - assert not bool(row[2]), "Expected points to be NULL in first row" - - def find_xsv_in_dump(dump, sep=','): - """ - Expecting to find a CSV (for pg_dump_anon) or TSV (for pg_dump) structure, looking like - - COPY public.player ... - 1,Shields, - 2,Salazar, - \. - - in the given dump (the commas are tabs in case of pg_dump). - Extract the CSV lines and split by `sep`. - """ - - try: - from itertools import dropwhile, takewhile - return [x.split(sep) for x in list(takewhile( - lambda x: x != "\\.", - dropwhile( - lambda x: not x.startswith("COPY public.player"), - dump.splitlines() - ) - ))[1:]] - except: - print(f"Dump to process: {dump}") - raise - - def check_original_data(output): - assert output[0] == ['1','Foo','23'], f"Expected first row from player table to be 1,Foo,23; got {output[0]}" - assert output[1] == ['2','Bar','42'], f"Expected first row from player table to be 2,Bar,42; got {output[1]}" - - def check_anonymized_rows(output): - check_anonymized_row(output[0], '1', 'Foo') - check_anonymized_row(output[1], '2', 'Bar') - - with subtest("Check initial state"): - check_original_data(get_player_table_contents()) - - with subtest("Anonymous dumps"): - check_original_data(find_xsv_in_dump( - machine.succeed("sudo -u postgres pg_dump demo"), - sep='\t' - )) - check_anonymized_rows(find_xsv_in_dump( - machine.succeed("sudo -u postgres pg_dump_anon -U postgres -h /run/postgresql -d demo"), - sep=',' - )) - - with subtest("Anonymize"): - machine.succeed("sudo -u postgres psql -d demo --command 'select anon.anonymize_database();'") - check_anonymized_rows(get_player_table_contents()) - ''; -}) diff --git a/nixos/tests/pgjwt.nix b/nixos/tests/pgjwt.nix deleted file mode 100644 index 3ab905cea9ac9..0000000000000 --- a/nixos/tests/pgjwt.nix +++ /dev/null @@ -1,35 +0,0 @@ -import ./make-test-python.nix ({ pkgs, lib, ...}: - -with pkgs; { - name = "pgjwt"; - meta = with lib.maintainers; { - maintainers = [ spinus willibutz ]; - }; - - nodes = { - master = { ... }: - { - services.postgresql = { - enable = true; - extraPlugins = ps: with ps; [ pgjwt pgtap ]; - }; - }; - }; - - testScript = { nodes, ... }: - let - sqlSU = "${nodes.master.services.postgresql.superUser}"; - pgProve = "${pkgs.perlPackages.TAPParserSourceHandlerpgTAP}"; - inherit (nodes.master.services.postgresql.package.pkgs) pgjwt; - in - '' - start_all() - master.wait_for_unit("postgresql") - master.succeed( - "${pkgs.gnused}/bin/sed -e '12 i CREATE EXTENSION pgcrypto;\\nCREATE EXTENSION pgtap;\\nSET search_path TO tap,public;' ${pgjwt.src}/test.sql > /tmp/test.sql" - ) - master.succeed( - "${pkgs.sudo}/bin/sudo -u ${sqlSU} PGOPTIONS=--search_path=tap,public ${pgProve}/bin/pg_prove -d postgres -v -f /tmp/test.sql" - ) - ''; -}) diff --git a/nixos/tests/pgvecto-rs.nix b/nixos/tests/pgvecto-rs.nix deleted file mode 100644 index 8d9d6c0b88f51..0000000000000 --- a/nixos/tests/pgvecto-rs.nix +++ /dev/null @@ -1,76 +0,0 @@ -# mostly copied from ./timescaledb.nix which was copied from ./postgresql.nix -# as it seemed unapproriate to test additional extensions for postgresql there. - -{ system ? builtins.currentSystem -, config ? { } -, pkgs ? import ../.. { inherit system config; } -}: - -with import ../lib/testing-python.nix { inherit system pkgs; }; -with pkgs.lib; - -let - postgresql-versions = import ../../pkgs/servers/sql/postgresql pkgs; - # Test cases from https://docs.pgvecto.rs/use-cases/hybrid-search.html - test-sql = pkgs.writeText "postgresql-test" '' - CREATE EXTENSION vectors; - - CREATE TABLE items ( - id bigserial PRIMARY KEY, - content text NOT NULL, - embedding vectors.vector(3) NOT NULL -- 3 dimensions - ); - - INSERT INTO items (content, embedding) VALUES - ('a fat cat sat on a mat and ate a fat rat', '[1, 2, 3]'), - ('a fat dog sat on a mat and ate a fat rat', '[4, 5, 6]'), - ('a thin cat sat on a mat and ate a thin rat', '[7, 8, 9]'), - ('a thin dog sat on a mat and ate a thin rat', '[10, 11, 12]'); - ''; - make-postgresql-test = postgresql-name: postgresql-package: makeTest { - name = postgresql-name; - meta = with pkgs.lib.maintainers; { - maintainers = [ diogotcorreia ]; - }; - - nodes.machine = { ... }: - { - services.postgresql = { - enable = true; - package = postgresql-package; - extraPlugins = ps: with ps; [ - pgvecto-rs - ]; - settings.shared_preload_libraries = "vectors"; - }; - }; - - testScript = '' - def check_count(statement, lines): - return 'test $(sudo -u postgres psql postgres -tAc "{}"|wc -l) -eq {}'.format( - statement, lines - ) - - - machine.start() - machine.wait_for_unit("postgresql") - - with subtest("Postgresql with extension vectors is available just after unit start"): - machine.succeed(check_count("SELECT * FROM pg_available_extensions WHERE name = 'vectors' AND default_version = '${postgresql-package.pkgs.pgvecto-rs.version}';", 1)) - - machine.succeed("sudo -u postgres psql -f ${test-sql}") - - machine.succeed(check_count("SELECT content, embedding FROM items WHERE to_tsvector('english', content) @@ 'cat & rat'::tsquery;", 2)) - - machine.shutdown() - ''; - - }; - applicablePostgresqlVersions = filterAttrs (_: value: versionAtLeast value.version "14") postgresql-versions; -in -mapAttrs' - (name: package: { - inherit name; - value = make-postgresql-test name package; - }) - applicablePostgresqlVersions diff --git a/nixos/tests/postgis.nix b/nixos/tests/postgis.nix deleted file mode 100644 index 5b63e2ba73eb1..0000000000000 --- a/nixos/tests/postgis.nix +++ /dev/null @@ -1,38 +0,0 @@ -import ./make-test-python.nix ({ pkgs, ...} : { - name = "postgis"; - meta = with pkgs.lib.maintainers; { - maintainers = [ lsix ]; - }; - - nodes = { - master = - { pkgs, ... }: - - { - services.postgresql = { - enable = true; - package = pkgs.postgresql; - extraPlugins = ps: with ps; [ - postgis - ]; - }; - }; - }; - - testScript = '' - start_all() - master.wait_for_unit("postgresql") - master.sleep(10) # Hopefully this is long enough!! - master.succeed("sudo -u postgres psql -c 'CREATE EXTENSION postgis;'") - master.succeed("sudo -u postgres psql -c 'CREATE EXTENSION postgis_raster;'") - master.succeed("sudo -u postgres psql -c 'CREATE EXTENSION postgis_topology;'") - master.succeed("sudo -u postgres psql -c 'select postgis_version();'") - master.succeed("[ \"$(sudo -u postgres psql --no-psqlrc --tuples-only -c 'select postgis_version();')\" = \" ${ - pkgs.lib.versions.major pkgs.postgis.version - }.${ - pkgs.lib.versions.minor pkgs.postgis.version - } USE_GEOS=1 USE_PROJ=1 USE_STATS=1\" ]") - # st_makepoint goes through c code - master.succeed("sudo -u postgres psql --no-psqlrc --tuples-only -c 'select st_makepoint(1, 1)'") - ''; -}) diff --git a/nixos/tests/postgresql-jit.nix b/nixos/tests/postgresql-jit.nix deleted file mode 100644 index f4b1d07a7faf8..0000000000000 --- a/nixos/tests/postgresql-jit.nix +++ /dev/null @@ -1,55 +0,0 @@ -{ system ? builtins.currentSystem -, config ? {} -, pkgs ? import ../.. { inherit system config; } -, package ? null -}: - -with import ../lib/testing-python.nix { inherit system pkgs; }; - -let - inherit (pkgs) lib; - packages = builtins.attrNames (import ../../pkgs/servers/sql/postgresql pkgs); - - mkJitTestFromName = name: - mkJitTest pkgs.${name}; - - mkJitTest = package: makeTest { - name = package.name; - meta.maintainers = with lib.maintainers; [ ma27 ]; - nodes.machine = { pkgs, lib, ... }: { - services.postgresql = { - inherit package; - enable = true; - enableJIT = true; - initialScript = pkgs.writeText "init.sql" '' - create table demo (id int); - insert into demo (id) select generate_series(1, 5); - ''; - }; - }; - testScript = '' - machine.start() - machine.wait_for_unit("postgresql.service") - - with subtest("JIT is enabled"): - machine.succeed("sudo -u postgres psql <<<'show jit;' | grep 'on'") - - with subtest("Test JIT works fine"): - output = machine.succeed( - "cat ${pkgs.writeText "test.sql" '' - set jit_above_cost = 1; - EXPLAIN ANALYZE SELECT CONCAT('jit result = ', SUM(id)) FROM demo; - SELECT CONCAT('jit result = ', SUM(id)) from demo; - ''} | sudo -u postgres psql" - ) - assert "JIT:" in output - assert "jit result = 15" in output - - machine.shutdown() - ''; - }; -in -if package == null then - lib.genAttrs packages mkJitTestFromName -else - mkJitTest package diff --git a/nixos/tests/postgresql-tls-client-cert.nix b/nixos/tests/postgresql-tls-client-cert.nix deleted file mode 100644 index c1678ed733beb..0000000000000 --- a/nixos/tests/postgresql-tls-client-cert.nix +++ /dev/null @@ -1,141 +0,0 @@ -{ system ? builtins.currentSystem -, config ? { } -, pkgs ? import ../.. { inherit system config; } -, package ? null -}: - -with import ../lib/testing-python.nix { inherit system pkgs; }; - -let - lib = pkgs.lib; - - # Makes a test for a PostgreSQL package, given by name and looked up from `pkgs`. - makeTestAttribute = name: - { - inherit name; - value = makePostgresqlTlsClientCertTest pkgs."${name}"; - }; - - makePostgresqlTlsClientCertTest = pkg: - let - runWithOpenSSL = file: cmd: pkgs.runCommand file - { - buildInputs = [ pkgs.openssl ]; - } - cmd; - caKey = runWithOpenSSL "ca.key" "openssl ecparam -name prime256v1 -genkey -noout -out $out"; - caCert = runWithOpenSSL - "ca.crt" - '' - openssl req -new -x509 -sha256 -key ${caKey} -out $out -subj "/CN=test.example" -days 36500 - ''; - serverKey = - runWithOpenSSL "server.key" "openssl ecparam -name prime256v1 -genkey -noout -out $out"; - serverKeyPath = "/var/lib/postgresql"; - serverCert = - runWithOpenSSL "server.crt" '' - openssl req -new -sha256 -key ${serverKey} -out server.csr -subj "/CN=db.test.example" - openssl x509 -req -in server.csr -CA ${caCert} -CAkey ${caKey} \ - -CAcreateserial -out $out -days 36500 -sha256 - ''; - clientKey = - runWithOpenSSL "client.key" "openssl ecparam -name prime256v1 -genkey -noout -out $out"; - clientCert = - runWithOpenSSL "client.crt" '' - openssl req -new -sha256 -key ${clientKey} -out client.csr -subj "/CN=test" - openssl x509 -req -in client.csr -CA ${caCert} -CAkey ${caKey} \ - -CAcreateserial -out $out -days 36500 -sha256 - ''; - clientKeyPath = "/root"; - - in - makeTest { - name = "postgresql-tls-client-cert-${pkg.name}"; - meta.maintainers = with lib.maintainers; [ erictapen ]; - - nodes.server = { ... }: { - system.activationScripts = { - keyPlacement.text = '' - mkdir -p '${serverKeyPath}' - cp '${serverKey}' '${serverKeyPath}/server.key' - chown postgres:postgres '${serverKeyPath}/server.key' - chmod 600 '${serverKeyPath}/server.key' - ''; - }; - services.postgresql = { - package = pkg; - enable = true; - enableTCPIP = true; - ensureUsers = [ - { - name = "test"; - ensureDBOwnership = true; - } - ]; - ensureDatabases = [ "test" ]; - settings = { - ssl = "on"; - ssl_ca_file = toString caCert; - ssl_cert_file = toString serverCert; - ssl_key_file = "${serverKeyPath}/server.key"; - }; - authentication = '' - hostssl test test ::/0 cert clientcert=verify-full - ''; - }; - networking = { - interfaces.eth1 = { - ipv6.addresses = [ - { address = "fc00::1"; prefixLength = 120; } - ]; - }; - firewall.allowedTCPPorts = [ 5432 ]; - }; - }; - - nodes.client = { ... }: { - system.activationScripts = { - keyPlacement.text = '' - mkdir -p '${clientKeyPath}' - cp '${clientKey}' '${clientKeyPath}/client.key' - chown root:root '${clientKeyPath}/client.key' - chmod 600 '${clientKeyPath}/client.key' - ''; - }; - environment = { - variables = { - PGHOST = "db.test.example"; - PGPORT = "5432"; - PGDATABASE = "test"; - PGUSER = "test"; - PGSSLMODE = "verify-full"; - PGSSLCERT = clientCert; - PGSSLKEY = "${clientKeyPath}/client.key"; - PGSSLROOTCERT = caCert; - }; - systemPackages = [ pkg ]; - }; - networking = { - interfaces.eth1 = { - ipv6.addresses = [ - { address = "fc00::2"; prefixLength = 120; } - ]; - }; - hosts = { "fc00::1" = [ "db.test.example" ]; }; - }; - }; - - testScript = '' - server.wait_for_unit("multi-user.target") - client.wait_for_unit("multi-user.target") - client.succeed("psql -c \"SELECT 1;\"") - ''; - }; - -in -if package == null then -# all-tests.nix: Maps the generic function over all attributes of PostgreSQL packages - builtins.listToAttrs (map makeTestAttribute (builtins.attrNames (import ../../pkgs/servers/sql/postgresql pkgs))) -else -# Called directly from .tests - makePostgresqlTlsClientCertTest package diff --git a/nixos/tests/postgresql-wal-receiver.nix b/nixos/tests/postgresql-wal-receiver.nix deleted file mode 100644 index a984f73c2be5b..0000000000000 --- a/nixos/tests/postgresql-wal-receiver.nix +++ /dev/null @@ -1,124 +0,0 @@ -{ system ? builtins.currentSystem, - config ? {}, - pkgs ? import ../.. { inherit system config; }, - package ? null -}: - -with import ../lib/testing-python.nix { inherit system pkgs; }; - -let - lib = pkgs.lib; - - # Makes a test for a PostgreSQL package, given by name and looked up from `pkgs`. - makeTestAttribute = name: - { - inherit name; - value = makePostgresqlWalReceiverTest pkgs."${name}"; - }; - - makePostgresqlWalReceiverTest = pkg: - let - postgresqlDataDir = "/var/lib/postgresql/${pkg.psqlSchema}"; - replicationUser = "wal_receiver_user"; - replicationSlot = "wal_receiver_slot"; - replicationConn = "postgresql://${replicationUser}@localhost"; - baseBackupDir = "/var/cache/wals/pg_basebackup"; - walBackupDir = "/var/cache/wals/pg_wal"; - - recoveryFile = pkgs.writeTextDir "recovery.signal" ""; - - in makeTest { - name = "postgresql-wal-receiver-${pkg.name}"; - meta.maintainers = with lib.maintainers; [ pacien ]; - - nodes.machine = { ... }: { - systemd.tmpfiles.rules = [ - "d /var/cache/wals 0750 postgres postgres - -" - ]; - - services.postgresql = { - package = pkg; - enable = true; - settings = { - max_replication_slots = 10; - max_wal_senders = 10; - recovery_end_command = "touch recovery.done"; - restore_command = "cp ${walBackupDir}/%f %p"; - wal_level = "archive"; # alias for replica on pg >= 9.6 - }; - authentication = '' - host replication ${replicationUser} all trust - ''; - initialScript = pkgs.writeText "init.sql" '' - create user ${replicationUser} replication; - select * from pg_create_physical_replication_slot('${replicationSlot}'); - ''; - }; - - services.postgresqlWalReceiver.receivers.main = { - postgresqlPackage = pkg; - connection = replicationConn; - slot = replicationSlot; - directory = walBackupDir; - }; - # This is only to speedup test, it isn't time racing. Service is set to autorestart always, - # default 60sec is fine for real system, but is too much for a test - systemd.services.postgresql-wal-receiver-main.serviceConfig.RestartSec = lib.mkForce 5; - systemd.services.postgresql.serviceConfig.ReadWritePaths = [ "/var/cache/wals" ]; - }; - - testScript = '' - # make an initial base backup - machine.wait_for_unit("postgresql") - machine.wait_for_unit("postgresql-wal-receiver-main") - # WAL receiver healthchecks PG every 5 seconds, so let's be sure they have connected each other - # required only for 9.4 - machine.sleep(5) - machine.succeed( - "${pkg}/bin/pg_basebackup --dbname=${replicationConn} --pgdata=${baseBackupDir}" - ) - - # create a dummy table with 100 records - machine.succeed( - "sudo -u postgres psql --command='create table dummy as select * from generate_series(1, 100) as val;'" - ) - - # stop postgres and destroy data - machine.systemctl("stop postgresql") - machine.systemctl("stop postgresql-wal-receiver-main") - machine.succeed("rm -r ${postgresqlDataDir}/{base,global,pg_*}") - - # restore the base backup - machine.succeed( - "cp -r ${baseBackupDir}/* ${postgresqlDataDir} && chown postgres:postgres -R ${postgresqlDataDir}" - ) - - # prepare WAL and recovery - machine.succeed("chmod a+rX -R ${walBackupDir}") - machine.execute( - "for part in ${walBackupDir}/*.partial; do mv $part ''${part%%.*}; done" - ) # make use of partial segments too - machine.succeed( - "cp ${recoveryFile}/* ${postgresqlDataDir}/ && chmod 666 ${postgresqlDataDir}/recovery*" - ) - - # replay WAL - machine.systemctl("start postgresql") - machine.wait_for_file("${postgresqlDataDir}/recovery.done") - machine.systemctl("restart postgresql") - machine.wait_for_unit("postgresql") - - # check that our records have been restored - machine.succeed( - "test $(sudo -u postgres psql --pset='pager=off' --tuples-only --command='select count(distinct val) from dummy;') -eq 100" - ) - ''; - }; - -in -if package == null then - # all-tests.nix: Maps the generic function over all attributes of PostgreSQL packages - builtins.listToAttrs (map makeTestAttribute (builtins.attrNames (import ../../pkgs/servers/sql/postgresql pkgs))) -else - # Called directly from .tests - makePostgresqlWalReceiverTest package diff --git a/nixos/tests/postgresql-wal2json.nix b/nixos/tests/postgresql-wal2json.nix deleted file mode 100644 index 043ad48cbc6ec..0000000000000 --- a/nixos/tests/postgresql-wal2json.nix +++ /dev/null @@ -1,60 +0,0 @@ -{ - system ? builtins.currentSystem, - config ? { }, - pkgs ? import ../.. { inherit system config; }, - postgresql ? null, -}: - -let - makeTest = import ./make-test-python.nix; - # Makes a test for a PostgreSQL package, given by name and looked up from `pkgs`. - makeTestAttribute = name: { - inherit name; - value = makePostgresqlWal2jsonTest pkgs."${name}"; - }; - - makePostgresqlWal2jsonTest = - postgresqlPackage: - makeTest { - name = "postgresql-wal2json-${postgresqlPackage.name}"; - meta.maintainers = with pkgs.lib.maintainers; [ euank ]; - - nodes.machine = { - services.postgresql = { - package = postgresqlPackage; - enable = true; - extraPlugins = with postgresqlPackage.pkgs; [ wal2json ]; - settings = { - wal_level = "logical"; - max_replication_slots = "10"; - max_wal_senders = "10"; - }; - }; - }; - - testScript = '' - machine.wait_for_unit("postgresql") - machine.succeed( - "sudo -u postgres psql -qAt -f ${./postgresql/wal2json/example2.sql} postgres > /tmp/example2.out" - ) - machine.succeed( - "diff ${./postgresql/wal2json/example2.out} /tmp/example2.out" - ) - machine.succeed( - "sudo -u postgres psql -qAt -f ${./postgresql/wal2json/example3.sql} postgres > /tmp/example3.out" - ) - machine.succeed( - "diff ${./postgresql/wal2json/example3.out} /tmp/example3.out" - ) - ''; - }; - -in -# By default, create one test per postgresql version -if postgresql == null then - builtins.listToAttrs ( - map makeTestAttribute (builtins.attrNames (import ../../pkgs/servers/sql/postgresql pkgs)) - ) -# but if postgresql is set, we're being made as a passthru test for a specific postgres + wal2json version, just run one -else - makePostgresqlWal2jsonTest postgresql diff --git a/nixos/tests/postgresql.nix b/nixos/tests/postgresql.nix deleted file mode 100644 index ce16e54edf662..0000000000000 --- a/nixos/tests/postgresql.nix +++ /dev/null @@ -1,226 +0,0 @@ -{ system ? builtins.currentSystem, - config ? {}, - pkgs ? import ../.. { inherit system config; } -}: - -with import ../lib/testing-python.nix { inherit system pkgs; }; -with pkgs.lib; - -let - postgresql-versions = import ../../pkgs/servers/sql/postgresql pkgs; - test-sql = pkgs.writeText "postgresql-test" '' - CREATE EXTENSION pgcrypto; -- just to check if lib loading works - CREATE TABLE sth ( - id int - ); - INSERT INTO sth (id) VALUES (1); - INSERT INTO sth (id) VALUES (1); - INSERT INTO sth (id) VALUES (1); - INSERT INTO sth (id) VALUES (1); - INSERT INTO sth (id) VALUES (1); - CREATE TABLE xmltest ( doc xml ); - INSERT INTO xmltest (doc) VALUES ('ok'); -- check if libxml2 enabled - ''; - make-postgresql-test = postgresql-name: postgresql-package: backup-all: makeTest { - name = postgresql-name; - meta = with pkgs.lib.maintainers; { - maintainers = [ zagy ]; - }; - - nodes.machine = {...}: - { - services.postgresql = { - enable = true; - package = postgresql-package; - }; - - services.postgresqlBackup = { - enable = true; - databases = optional (!backup-all) "postgres"; - }; - }; - - testScript = let - backupName = if backup-all then "all" else "postgres"; - backupService = if backup-all then "postgresqlBackup" else "postgresqlBackup-postgres"; - backupFileBase = "/var/backup/postgresql/${backupName}"; - in '' - def check_count(statement, lines): - return 'test $(sudo -u postgres psql postgres -tAc "{}"|wc -l) -eq {}'.format( - statement, lines - ) - - - machine.start() - machine.wait_for_unit("postgresql") - - with subtest("Postgresql is available just after unit start"): - machine.succeed( - "cat ${test-sql} | sudo -u postgres psql" - ) - - with subtest("Postgresql survives restart (bug #1735)"): - machine.shutdown() - import time - time.sleep(2) - machine.start() - machine.wait_for_unit("postgresql") - - machine.fail(check_count("SELECT * FROM sth;", 3)) - machine.succeed(check_count("SELECT * FROM sth;", 5)) - machine.fail(check_count("SELECT * FROM sth;", 4)) - machine.succeed(check_count("SELECT xpath('/test/text()', doc) FROM xmltest;", 1)) - - with subtest("Backup service works"): - machine.succeed( - "systemctl start ${backupService}.service", - "zcat ${backupFileBase}.sql.gz | grep 'ok'", - "ls -hal /var/backup/postgresql/ >/dev/console", - "stat -c '%a' ${backupFileBase}.sql.gz | grep 600", - ) - with subtest("Backup service removes prev files"): - machine.succeed( - # Create dummy prev files. - "touch ${backupFileBase}.prev.sql{,.gz,.zstd}", - "chown postgres:postgres ${backupFileBase}.prev.sql{,.gz,.zstd}", - - # Run backup. - "systemctl start ${backupService}.service", - "ls -hal /var/backup/postgresql/ >/dev/console", - - # Since nothing has changed in the database, the cur and prev files - # should match. - "zcat ${backupFileBase}.sql.gz | grep 'ok'", - "cmp ${backupFileBase}.sql.gz ${backupFileBase}.prev.sql.gz", - - # The prev files with unused suffix should be removed. - "[ ! -f '${backupFileBase}.prev.sql' ]", - "[ ! -f '${backupFileBase}.prev.sql.zstd' ]", - - # Both cur and prev file should only be accessible by the postgres user. - "stat -c '%a' ${backupFileBase}.sql.gz | grep 600", - "stat -c '%a' '${backupFileBase}.prev.sql.gz' | grep 600", - ) - with subtest("Backup service fails gracefully"): - # Sabotage the backup process - machine.succeed("rm /run/postgresql/.s.PGSQL.5432") - machine.fail( - "systemctl start ${backupService}.service", - ) - machine.succeed( - "ls -hal /var/backup/postgresql/ >/dev/console", - "zcat ${backupFileBase}.prev.sql.gz | grep 'ok'", - "stat ${backupFileBase}.in-progress.sql.gz", - ) - # In a previous version, the second run would overwrite prev.sql.gz, - # so we test a second run as well. - machine.fail( - "systemctl start ${backupService}.service", - ) - machine.succeed( - "stat ${backupFileBase}.in-progress.sql.gz", - "zcat ${backupFileBase}.prev.sql.gz | grep 'ok'", - ) - - - with subtest("Initdb works"): - machine.succeed("sudo -u postgres initdb -D /tmp/testpostgres2") - - machine.log(machine.execute("systemd-analyze security postgresql.service | grep -v ✓")[1]) - - machine.shutdown() - ''; - - }; - - mk-ensure-clauses-test = postgresql-name: postgresql-package: makeTest { - name = postgresql-name; - meta = with pkgs.lib.maintainers; { - maintainers = [ zagy ]; - }; - - nodes.machine = {...}: - { - services.postgresql = { - enable = true; - package = postgresql-package; - ensureUsers = [ - { - name = "all-clauses"; - ensureClauses = { - superuser = true; - createdb = true; - createrole = true; - "inherit" = true; - login = true; - replication = true; - bypassrls = true; - }; - } - { - name = "default-clauses"; - } - ]; - }; - }; - - testScript = let - getClausesQuery = user: pkgs.lib.concatStringsSep " " - [ - "SELECT row_to_json(row)" - "FROM (" - "SELECT" - "rolsuper," - "rolinherit," - "rolcreaterole," - "rolcreatedb," - "rolcanlogin," - "rolreplication," - "rolbypassrls" - "FROM pg_roles" - "WHERE rolname = '${user}'" - ") row;" - ]; - in '' - import json - machine.start() - machine.wait_for_unit("postgresql") - - with subtest("All user permissions are set according to the ensureClauses attr"): - clauses = json.loads( - machine.succeed( - "sudo -u postgres psql -tc \"${getClausesQuery "all-clauses"}\"" - ) - ) - print(clauses) - assert clauses['rolsuper'], 'expected user with clauses to have superuser clause' - assert clauses['rolinherit'], 'expected user with clauses to have inherit clause' - assert clauses['rolcreaterole'], 'expected user with clauses to have create role clause' - assert clauses['rolcreatedb'], 'expected user with clauses to have create db clause' - assert clauses['rolcanlogin'], 'expected user with clauses to have login clause' - assert clauses['rolreplication'], 'expected user with clauses to have replication clause' - assert clauses['rolbypassrls'], 'expected user with clauses to have bypassrls clause' - - with subtest("All user permissions default when ensureClauses is not provided"): - clauses = json.loads( - machine.succeed( - "sudo -u postgres psql -tc \"${getClausesQuery "default-clauses"}\"" - ) - ) - assert not clauses['rolsuper'], 'expected user with no clauses set to have default superuser clause' - assert clauses['rolinherit'], 'expected user with no clauses set to have default inherit clause' - assert not clauses['rolcreaterole'], 'expected user with no clauses set to have default create role clause' - assert not clauses['rolcreatedb'], 'expected user with no clauses set to have default create db clause' - assert clauses['rolcanlogin'], 'expected user with no clauses set to have default login clause' - assert not clauses['rolreplication'], 'expected user with no clauses set to have default replication clause' - assert not clauses['rolbypassrls'], 'expected user with no clauses set to have default bypassrls clause' - - machine.shutdown() - ''; - }; -in - concatMapAttrs (name: package: { - ${name} = make-postgresql-test name package false; - ${name + "-backup-all"} = make-postgresql-test "${name + "-backup-all"}" package true; - ${name + "-clauses"} = mk-ensure-clauses-test name package; - }) postgresql-versions diff --git a/nixos/tests/postgresql/anonymizer.nix b/nixos/tests/postgresql/anonymizer.nix new file mode 100644 index 0000000000000..3a5f69086eaac --- /dev/null +++ b/nixos/tests/postgresql/anonymizer.nix @@ -0,0 +1,116 @@ +{ + pkgs, + makeTest, +}: + +let + inherit (pkgs) lib; + + makeTestFor = + package: + makeTest { + name = "postgresql_anonymizer-${package.name}"; + meta.maintainers = lib.teams.flyingcircus.members; + + nodes.machine = + { pkgs, ... }: + { + environment.systemPackages = [ pkgs.pg-dump-anon ]; + services.postgresql = { + inherit package; + enable = true; + extraPlugins = ps: [ ps.anonymizer ]; + settings.shared_preload_libraries = [ "anon" ]; + }; + }; + + testScript = '' + start_all() + machine.wait_for_unit("multi-user.target") + machine.wait_for_unit("postgresql.service") + + with subtest("Setup"): + machine.succeed("sudo -u postgres psql --command 'create database demo'") + machine.succeed( + "sudo -u postgres psql -d demo -f ${pkgs.writeText "init.sql" '' + create extension anon cascade; + select anon.init(); + create table player(id serial, name text, points int); + insert into player(id,name,points) values (1,'Foo', 23); + insert into player(id,name,points) values (2,'Bar',42); + security label for anon on column player.name is 'MASKED WITH FUNCTION anon.fake_last_name();'; + security label for anon on column player.points is 'MASKED WITH VALUE NULL'; + ''}" + ) + + def get_player_table_contents(): + return [ + x.split(',') for x in machine.succeed("sudo -u postgres psql -d demo --csv --command 'select * from player'").splitlines()[1:] + ] + + def check_anonymized_row(row, id, original_name): + assert row[0] == id, f"Expected first row to have ID {id}, but got {row[0]}" + assert row[1] != original_name, f"Expected first row to have a name other than {original_name}" + assert not bool(row[2]), "Expected points to be NULL in first row" + + def find_xsv_in_dump(dump, sep=','): + """ + Expecting to find a CSV (for pg_dump_anon) or TSV (for pg_dump) structure, looking like + + COPY public.player ... + 1,Shields, + 2,Salazar, + \. + + in the given dump (the commas are tabs in case of pg_dump). + Extract the CSV lines and split by `sep`. + """ + + try: + from itertools import dropwhile, takewhile + return [x.split(sep) for x in list(takewhile( + lambda x: x != "\\.", + dropwhile( + lambda x: not x.startswith("COPY public.player"), + dump.splitlines() + ) + ))[1:]] + except: + print(f"Dump to process: {dump}") + raise + + def check_original_data(output): + assert output[0] == ['1','Foo','23'], f"Expected first row from player table to be 1,Foo,23; got {output[0]}" + assert output[1] == ['2','Bar','42'], f"Expected first row from player table to be 2,Bar,42; got {output[1]}" + + def check_anonymized_rows(output): + check_anonymized_row(output[0], '1', 'Foo') + check_anonymized_row(output[1], '2', 'Bar') + + with subtest("Check initial state"): + check_original_data(get_player_table_contents()) + + with subtest("Anonymous dumps"): + check_original_data(find_xsv_in_dump( + machine.succeed("sudo -u postgres pg_dump demo"), + sep='\t' + )) + check_anonymized_rows(find_xsv_in_dump( + machine.succeed("sudo -u postgres pg_dump_anon -U postgres -h /run/postgresql -d demo"), + sep=',' + )) + + with subtest("Anonymize"): + machine.succeed("sudo -u postgres psql -d demo --command 'select anon.anonymize_database();'") + check_anonymized_rows(get_player_table_contents()) + ''; + }; +in +lib.recurseIntoAttrs ( + lib.concatMapAttrs (n: p: { ${n} = makeTestFor p; }) ( + lib.filterAttrs (_: p: !p.pkgs.anonymizer.meta.broken) pkgs.postgresqlVersions + ) + // { + passthru.override = p: makeTestFor p; + } +) diff --git a/nixos/tests/postgresql/default.nix b/nixos/tests/postgresql/default.nix new file mode 100644 index 0000000000000..4fe7e7a37e7ef --- /dev/null +++ b/nixos/tests/postgresql/default.nix @@ -0,0 +1,26 @@ +{ + system ? builtins.currentSystem, + config ? { }, + pkgs ? import ../../.. { inherit system config; }, +}: + +with import ../../lib/testing-python.nix { inherit system pkgs; }; + +let + importWithArgs = path: import path { inherit pkgs makeTest; }; +in +{ + # postgresql + postgresql = importWithArgs ./postgresql.nix; + postgresql-jit = importWithArgs ./postgresql-jit.nix; + postgresql-wal-receiver = importWithArgs ./postgresql-wal-receiver.nix; + postgresql-tls-client-cert = importWithArgs ./postgresql-tls-client-cert.nix; + + # extensions + anonymizer = importWithArgs ./anonymizer.nix; + pgjwt = importWithArgs ./pgjwt.nix; + pgvecto-rs = importWithArgs ./pgvecto-rs.nix; + timescaledb = importWithArgs ./timescaledb.nix; + tsja = importWithArgs ./tsja.nix; + wal2json = importWithArgs ./wal2json.nix; +} diff --git a/nixos/tests/postgresql/pgjwt.nix b/nixos/tests/postgresql/pgjwt.nix new file mode 100644 index 0000000000000..81e5dac41adae --- /dev/null +++ b/nixos/tests/postgresql/pgjwt.nix @@ -0,0 +1,57 @@ +{ + pkgs, + makeTest, +}: + +let + inherit (pkgs) lib; + + makeTestFor = + package: + makeTest { + name = "pgjwt-${package.name}"; + meta = with lib.maintainers; { + maintainers = [ + spinus + willibutz + ]; + }; + + nodes.master = + { ... }: + { + services.postgresql = { + inherit package; + enable = true; + extraPlugins = + ps: with ps; [ + pgjwt + pgtap + ]; + }; + }; + + testScript = + { nodes, ... }: + let + sqlSU = "${nodes.master.services.postgresql.superUser}"; + pgProve = "${pkgs.perlPackages.TAPParserSourceHandlerpgTAP}"; + inherit (nodes.master.services.postgresql.package.pkgs) pgjwt; + in + '' + start_all() + master.wait_for_unit("postgresql") + master.succeed( + "${pkgs.sudo}/bin/sudo -u ${sqlSU} ${pgProve}/bin/pg_prove -d postgres -v -f ${pgjwt.src}/test.sql" + ) + ''; + }; +in +lib.recurseIntoAttrs ( + lib.concatMapAttrs (n: p: { ${n} = makeTestFor p; }) ( + lib.filterAttrs (_: p: !p.pkgs.pgjwt.meta.broken) pkgs.postgresqlVersions + ) + // { + passthru.override = p: makeTestFor p; + } +) diff --git a/nixos/tests/postgresql/pgvecto-rs.nix b/nixos/tests/postgresql/pgvecto-rs.nix new file mode 100644 index 0000000000000..9d8389eecf99a --- /dev/null +++ b/nixos/tests/postgresql/pgvecto-rs.nix @@ -0,0 +1,81 @@ +{ + pkgs, + makeTest, +}: + +let + inherit (pkgs) lib; + + # Test cases from https://docs.pgvecto.rs/use-cases/hybrid-search.html + test-sql = pkgs.writeText "postgresql-test" '' + CREATE EXTENSION vectors; + + CREATE TABLE items ( + id bigserial PRIMARY KEY, + content text NOT NULL, + embedding vectors.vector(3) NOT NULL -- 3 dimensions + ); + + INSERT INTO items (content, embedding) VALUES + ('a fat cat sat on a mat and ate a fat rat', '[1, 2, 3]'), + ('a fat dog sat on a mat and ate a fat rat', '[4, 5, 6]'), + ('a thin cat sat on a mat and ate a thin rat', '[7, 8, 9]'), + ('a thin dog sat on a mat and ate a thin rat', '[10, 11, 12]'); + ''; + + makeTestFor = + postgresqlPackage: + makeTest { + name = "pgvecto-rs-${postgresqlPackage.name}"; + meta = with lib.maintainers; { + maintainers = [ diogotcorreia ]; + }; + + nodes.machine = + { ... }: + { + services.postgresql = { + enable = true; + package = postgresqlPackage; + extraPlugins = + ps: with ps; [ + pgvecto-rs + ]; + settings.shared_preload_libraries = "vectors"; + }; + }; + + testScript = + { nodes, ... }: + let + inherit (nodes.machine.services.postgresql.package.pkgs) pgvecto-rs; + in + '' + def check_count(statement, lines): + return 'test $(sudo -u postgres psql postgres -tAc "{}"|wc -l) -eq {}'.format( + statement, lines + ) + + + machine.start() + machine.wait_for_unit("postgresql") + + with subtest("Postgresql with extension vectors is available just after unit start"): + machine.succeed(check_count("SELECT * FROM pg_available_extensions WHERE name = 'vectors' AND default_version = '${pgvecto-rs.version}';", 1)) + + machine.succeed("sudo -u postgres psql -f ${test-sql}") + + machine.succeed(check_count("SELECT content, embedding FROM items WHERE to_tsvector('english', content) @@ 'cat & rat'::tsquery;", 2)) + + machine.shutdown() + ''; + }; +in +lib.recurseIntoAttrs ( + lib.concatMapAttrs (n: p: { ${n} = makeTestFor p; }) ( + lib.filterAttrs (_: p: !p.pkgs.pgvecto-rs.meta.broken) pkgs.postgresqlVersions + ) + // { + passthru.override = p: makeTestFor p; + } +) diff --git a/nixos/tests/postgresql/postgresql-jit.nix b/nixos/tests/postgresql/postgresql-jit.nix new file mode 100644 index 0000000000000..5d0406062eae3 --- /dev/null +++ b/nixos/tests/postgresql/postgresql-jit.nix @@ -0,0 +1,58 @@ +{ + pkgs, + makeTest, +}: + +let + inherit (pkgs) lib; + + makeTestFor = + package: + makeTest { + name = "postgresql-jit-${package.name}"; + meta.maintainers = with lib.maintainers; [ ma27 ]; + + nodes.machine = + { pkgs, ... }: + { + services.postgresql = { + inherit package; + enable = true; + enableJIT = true; + initialScript = pkgs.writeText "init.sql" '' + create table demo (id int); + insert into demo (id) select generate_series(1, 5); + ''; + }; + }; + + testScript = '' + machine.start() + machine.wait_for_unit("postgresql.service") + + with subtest("JIT is enabled"): + machine.succeed("sudo -u postgres psql <<<'show jit;' | grep 'on'") + + with subtest("Test JIT works fine"): + output = machine.succeed( + "cat ${pkgs.writeText "test.sql" '' + set jit_above_cost = 1; + EXPLAIN ANALYZE SELECT CONCAT('jit result = ', SUM(id)) FROM demo; + SELECT CONCAT('jit result = ', SUM(id)) from demo; + ''} | sudo -u postgres psql" + ) + assert "JIT:" in output + assert "jit result = 15" in output + + machine.shutdown() + ''; + }; +in +lib.recurseIntoAttrs ( + lib.concatMapAttrs (n: p: { ${n} = makeTestFor p; }) ( + lib.filterAttrs (n: _: lib.hasSuffix "_jit" n) pkgs.postgresqlVersions + ) + // { + passthru.override = p: makeTestFor p; + } +) diff --git a/nixos/tests/postgresql/postgresql-tls-client-cert.nix b/nixos/tests/postgresql/postgresql-tls-client-cert.nix new file mode 100644 index 0000000000000..d7cddb625256c --- /dev/null +++ b/nixos/tests/postgresql/postgresql-tls-client-cert.nix @@ -0,0 +1,135 @@ +{ + pkgs, + makeTest, +}: + +let + inherit (pkgs) lib; + + runWithOpenSSL = + file: cmd: + pkgs.runCommand file { + buildInputs = [ pkgs.openssl ]; + } cmd; + caKey = runWithOpenSSL "ca.key" "openssl ecparam -name prime256v1 -genkey -noout -out $out"; + caCert = runWithOpenSSL "ca.crt" '' + openssl req -new -x509 -sha256 -key ${caKey} -out $out -subj "/CN=test.example" -days 36500 + ''; + serverKey = runWithOpenSSL "server.key" "openssl ecparam -name prime256v1 -genkey -noout -out $out"; + serverKeyPath = "/var/lib/postgresql"; + serverCert = runWithOpenSSL "server.crt" '' + openssl req -new -sha256 -key ${serverKey} -out server.csr -subj "/CN=db.test.example" + openssl x509 -req -in server.csr -CA ${caCert} -CAkey ${caKey} \ + -CAcreateserial -out $out -days 36500 -sha256 + ''; + clientKey = runWithOpenSSL "client.key" "openssl ecparam -name prime256v1 -genkey -noout -out $out"; + clientCert = runWithOpenSSL "client.crt" '' + openssl req -new -sha256 -key ${clientKey} -out client.csr -subj "/CN=test" + openssl x509 -req -in client.csr -CA ${caCert} -CAkey ${caKey} \ + -CAcreateserial -out $out -days 36500 -sha256 + ''; + clientKeyPath = "/root"; + + makeTestFor = + package: + makeTest { + name = "postgresql-tls-client-cert-${package.name}"; + meta.maintainers = with lib.maintainers; [ erictapen ]; + + nodes.server = + { ... }: + { + system.activationScripts = { + keyPlacement.text = '' + mkdir -p '${serverKeyPath}' + cp '${serverKey}' '${serverKeyPath}/server.key' + chown postgres:postgres '${serverKeyPath}/server.key' + chmod 600 '${serverKeyPath}/server.key' + ''; + }; + services.postgresql = { + inherit package; + enable = true; + enableTCPIP = true; + ensureUsers = [ + { + name = "test"; + ensureDBOwnership = true; + } + ]; + ensureDatabases = [ "test" ]; + settings = { + ssl = "on"; + ssl_ca_file = toString caCert; + ssl_cert_file = toString serverCert; + ssl_key_file = "${serverKeyPath}/server.key"; + }; + authentication = '' + hostssl test test ::/0 cert clientcert=verify-full + ''; + }; + networking = { + interfaces.eth1 = { + ipv6.addresses = [ + { + address = "fc00::1"; + prefixLength = 120; + } + ]; + }; + firewall.allowedTCPPorts = [ 5432 ]; + }; + }; + + nodes.client = + { ... }: + { + system.activationScripts = { + keyPlacement.text = '' + mkdir -p '${clientKeyPath}' + cp '${clientKey}' '${clientKeyPath}/client.key' + chown root:root '${clientKeyPath}/client.key' + chmod 600 '${clientKeyPath}/client.key' + ''; + }; + environment = { + variables = { + PGHOST = "db.test.example"; + PGPORT = "5432"; + PGDATABASE = "test"; + PGUSER = "test"; + PGSSLMODE = "verify-full"; + PGSSLCERT = clientCert; + PGSSLKEY = "${clientKeyPath}/client.key"; + PGSSLROOTCERT = caCert; + }; + systemPackages = [ package ]; + }; + networking = { + interfaces.eth1 = { + ipv6.addresses = [ + { + address = "fc00::2"; + prefixLength = 120; + } + ]; + }; + hosts = { + "fc00::1" = [ "db.test.example" ]; + }; + }; + }; + + testScript = '' + server.wait_for_unit("multi-user.target") + client.wait_for_unit("multi-user.target") + client.succeed("psql -c \"SELECT 1;\"") + ''; + }; +in +lib.recurseIntoAttrs ( + lib.concatMapAttrs (n: p: { ${n} = makeTestFor p; }) pkgs.postgresqlVersions + // { + passthru.override = p: makeTestFor p; + } +) diff --git a/nixos/tests/postgresql/postgresql-wal-receiver.nix b/nixos/tests/postgresql/postgresql-wal-receiver.nix new file mode 100644 index 0000000000000..5c1551c5f2fd2 --- /dev/null +++ b/nixos/tests/postgresql/postgresql-wal-receiver.nix @@ -0,0 +1,115 @@ +{ + pkgs, + makeTest, +}: + +let + inherit (pkgs) lib; + + makeTestFor = + package: + let + postgresqlDataDir = "/var/lib/postgresql/${package.psqlSchema}"; + replicationUser = "wal_receiver_user"; + replicationSlot = "wal_receiver_slot"; + replicationConn = "postgresql://${replicationUser}@localhost"; + baseBackupDir = "/var/cache/wals/pg_basebackup"; + walBackupDir = "/var/cache/wals/pg_wal"; + recoveryFile = pkgs.writeTextDir "recovery.signal" ""; + in + makeTest { + name = "postgresql-wal-receiver-${package.name}"; + meta.maintainers = with lib.maintainers; [ pacien ]; + + nodes.machine = + { ... }: + { + systemd.tmpfiles.rules = [ + "d /var/cache/wals 0750 postgres postgres - -" + ]; + + services.postgresql = { + inherit package; + enable = true; + settings = { + max_replication_slots = 10; + max_wal_senders = 10; + recovery_end_command = "touch recovery.done"; + restore_command = "cp ${walBackupDir}/%f %p"; + wal_level = "archive"; # alias for replica on pg >= 9.6 + }; + authentication = '' + host replication ${replicationUser} all trust + ''; + initialScript = pkgs.writeText "init.sql" '' + create user ${replicationUser} replication; + select * from pg_create_physical_replication_slot('${replicationSlot}'); + ''; + }; + + services.postgresqlWalReceiver.receivers.main = { + postgresqlPackage = package; + connection = replicationConn; + slot = replicationSlot; + directory = walBackupDir; + }; + # This is only to speedup test, it isn't time racing. Service is set to autorestart always, + # default 60sec is fine for real system, but is too much for a test + systemd.services.postgresql-wal-receiver-main.serviceConfig.RestartSec = lib.mkForce 5; + systemd.services.postgresql.serviceConfig.ReadWritePaths = [ "/var/cache/wals" ]; + }; + + testScript = '' + # make an initial base backup + machine.wait_for_unit("postgresql") + machine.wait_for_unit("postgresql-wal-receiver-main") + # WAL receiver healthchecks PG every 5 seconds, so let's be sure they have connected each other + # required only for 9.4 + machine.sleep(5) + machine.succeed( + "${package}/bin/pg_basebackup --dbname=${replicationConn} --pgdata=${baseBackupDir}" + ) + + # create a dummy table with 100 records + machine.succeed( + "sudo -u postgres psql --command='create table dummy as select * from generate_series(1, 100) as val;'" + ) + + # stop postgres and destroy data + machine.systemctl("stop postgresql") + machine.systemctl("stop postgresql-wal-receiver-main") + machine.succeed("rm -r ${postgresqlDataDir}/{base,global,pg_*}") + + # restore the base backup + machine.succeed( + "cp -r ${baseBackupDir}/* ${postgresqlDataDir} && chown postgres:postgres -R ${postgresqlDataDir}" + ) + + # prepare WAL and recovery + machine.succeed("chmod a+rX -R ${walBackupDir}") + machine.execute( + "for part in ${walBackupDir}/*.partial; do mv $part ''${part%%.*}; done" + ) # make use of partial segments too + machine.succeed( + "cp ${recoveryFile}/* ${postgresqlDataDir}/ && chmod 666 ${postgresqlDataDir}/recovery*" + ) + + # replay WAL + machine.systemctl("start postgresql") + machine.wait_for_file("${postgresqlDataDir}/recovery.done") + machine.systemctl("restart postgresql") + machine.wait_for_unit("postgresql") + + # check that our records have been restored + machine.succeed( + "test $(sudo -u postgres psql --pset='pager=off' --tuples-only --command='select count(distinct val) from dummy;') -eq 100" + ) + ''; + }; +in +lib.recurseIntoAttrs ( + lib.concatMapAttrs (n: p: { ${n} = makeTestFor p; }) pkgs.postgresqlVersions + // { + passthru.override = p: makeTestFor p; + } +) diff --git a/nixos/tests/postgresql/postgresql.nix b/nixos/tests/postgresql/postgresql.nix new file mode 100644 index 0000000000000..509a14411de9c --- /dev/null +++ b/nixos/tests/postgresql/postgresql.nix @@ -0,0 +1,244 @@ +{ + pkgs, + makeTest, +}: + +let + inherit (pkgs) lib; + + makeTestFor = + package: + lib.recurseIntoAttrs { + postgresql = makeTestForWithBackupAll package false; + postgresql-backup-all = makeTestForWithBackupAll package true; + postgresql-clauses = makeEnsureTestFor package; + }; + + test-sql = pkgs.writeText "postgresql-test" '' + CREATE EXTENSION pgcrypto; -- just to check if lib loading works + CREATE TABLE sth ( + id int + ); + INSERT INTO sth (id) VALUES (1); + INSERT INTO sth (id) VALUES (1); + INSERT INTO sth (id) VALUES (1); + INSERT INTO sth (id) VALUES (1); + INSERT INTO sth (id) VALUES (1); + CREATE TABLE xmltest ( doc xml ); + INSERT INTO xmltest (doc) VALUES ('ok'); -- check if libxml2 enabled + ''; + + makeTestForWithBackupAll = + package: backupAll: + makeTest { + name = "postgresql${lib.optionalString backupAll "-backup-all"}-${package.name}"; + meta = with lib.maintainers; { + maintainers = [ zagy ]; + }; + + nodes.machine = + { ... }: + { + services.postgresql = { + inherit (package) ; + enable = true; + }; + + services.postgresqlBackup = { + enable = true; + databases = lib.optional (!backupAll) "postgres"; + }; + }; + + testScript = + let + backupName = if backupAll then "all" else "postgres"; + backupService = if backupAll then "postgresqlBackup" else "postgresqlBackup-postgres"; + backupFileBase = "/var/backup/postgresql/${backupName}"; + in + '' + def check_count(statement, lines): + return 'test $(sudo -u postgres psql postgres -tAc "{}"|wc -l) -eq {}'.format( + statement, lines + ) + + + machine.start() + machine.wait_for_unit("postgresql") + + with subtest("Postgresql is available just after unit start"): + machine.succeed( + "cat ${test-sql} | sudo -u postgres psql" + ) + + with subtest("Postgresql survives restart (bug #1735)"): + machine.shutdown() + import time + time.sleep(2) + machine.start() + machine.wait_for_unit("postgresql") + + machine.fail(check_count("SELECT * FROM sth;", 3)) + machine.succeed(check_count("SELECT * FROM sth;", 5)) + machine.fail(check_count("SELECT * FROM sth;", 4)) + machine.succeed(check_count("SELECT xpath('/test/text()', doc) FROM xmltest;", 1)) + + with subtest("Backup service works"): + machine.succeed( + "systemctl start ${backupService}.service", + "zcat ${backupFileBase}.sql.gz | grep 'ok'", + "ls -hal /var/backup/postgresql/ >/dev/console", + "stat -c '%a' ${backupFileBase}.sql.gz | grep 600", + ) + with subtest("Backup service removes prev files"): + machine.succeed( + # Create dummy prev files. + "touch ${backupFileBase}.prev.sql{,.gz,.zstd}", + "chown postgres:postgres ${backupFileBase}.prev.sql{,.gz,.zstd}", + + # Run backup. + "systemctl start ${backupService}.service", + "ls -hal /var/backup/postgresql/ >/dev/console", + + # Since nothing has changed in the database, the cur and prev files + # should match. + "zcat ${backupFileBase}.sql.gz | grep 'ok'", + "cmp ${backupFileBase}.sql.gz ${backupFileBase}.prev.sql.gz", + + # The prev files with unused suffix should be removed. + "[ ! -f '${backupFileBase}.prev.sql' ]", + "[ ! -f '${backupFileBase}.prev.sql.zstd' ]", + + # Both cur and prev file should only be accessible by the postgres user. + "stat -c '%a' ${backupFileBase}.sql.gz | grep 600", + "stat -c '%a' '${backupFileBase}.prev.sql.gz' | grep 600", + ) + with subtest("Backup service fails gracefully"): + # Sabotage the backup process + machine.succeed("rm /run/postgresql/.s.PGSQL.5432") + machine.fail( + "systemctl start ${backupService}.service", + ) + machine.succeed( + "ls -hal /var/backup/postgresql/ >/dev/console", + "zcat ${backupFileBase}.prev.sql.gz | grep 'ok'", + "stat ${backupFileBase}.in-progress.sql.gz", + ) + # In a previous version, the second run would overwrite prev.sql.gz, + # so we test a second run as well. + machine.fail( + "systemctl start ${backupService}.service", + ) + machine.succeed( + "stat ${backupFileBase}.in-progress.sql.gz", + "zcat ${backupFileBase}.prev.sql.gz | grep 'ok'", + ) + + + with subtest("Initdb works"): + machine.succeed("sudo -u postgres initdb -D /tmp/testpostgres2") + + machine.log(machine.execute("systemd-analyze security postgresql.service | grep -v ✓")[1]) + + machine.shutdown() + ''; + }; + + makeEnsureTestFor = + package: + makeTest { + name = "postgresql-clauses-${package.name}"; + meta = with lib.maintainers; { + maintainers = [ zagy ]; + }; + + nodes.machine = + { ... }: + { + services.postgresql = { + inherit package; + enable = true; + ensureUsers = [ + { + name = "all-clauses"; + ensureClauses = { + superuser = true; + createdb = true; + createrole = true; + "inherit" = true; + login = true; + replication = true; + bypassrls = true; + }; + } + { + name = "default-clauses"; + } + ]; + }; + }; + + testScript = + let + getClausesQuery = + user: + lib.concatStringsSep " " [ + "SELECT row_to_json(row)" + "FROM (" + "SELECT" + "rolsuper," + "rolinherit," + "rolcreaterole," + "rolcreatedb," + "rolcanlogin," + "rolreplication," + "rolbypassrls" + "FROM pg_roles" + "WHERE rolname = '${user}'" + ") row;" + ]; + in + '' + import json + machine.start() + machine.wait_for_unit("postgresql") + + with subtest("All user permissions are set according to the ensureClauses attr"): + clauses = json.loads( + machine.succeed( + "sudo -u postgres psql -tc \"${getClausesQuery "all-clauses"}\"" + ) + ) + print(clauses) + assert clauses['rolsuper'], 'expected user with clauses to have superuser clause' + assert clauses['rolinherit'], 'expected user with clauses to have inherit clause' + assert clauses['rolcreaterole'], 'expected user with clauses to have create role clause' + assert clauses['rolcreatedb'], 'expected user with clauses to have create db clause' + assert clauses['rolcanlogin'], 'expected user with clauses to have login clause' + assert clauses['rolreplication'], 'expected user with clauses to have replication clause' + assert clauses['rolbypassrls'], 'expected user with clauses to have bypassrls clause' + + with subtest("All user permissions default when ensureClauses is not provided"): + clauses = json.loads( + machine.succeed( + "sudo -u postgres psql -tc \"${getClausesQuery "default-clauses"}\"" + ) + ) + assert not clauses['rolsuper'], 'expected user with no clauses set to have default superuser clause' + assert clauses['rolinherit'], 'expected user with no clauses set to have default inherit clause' + assert not clauses['rolcreaterole'], 'expected user with no clauses set to have default create role clause' + assert not clauses['rolcreatedb'], 'expected user with no clauses set to have default create db clause' + assert clauses['rolcanlogin'], 'expected user with no clauses set to have default login clause' + assert not clauses['rolreplication'], 'expected user with no clauses set to have default replication clause' + assert not clauses['rolbypassrls'], 'expected user with no clauses set to have default bypassrls clause' + + machine.shutdown() + ''; + }; +in +lib.recurseIntoAttrs ( + lib.concatMapAttrs (n: p: { ${n} = makeTestFor p; }) pkgs.postgresqlVersions + // { + passthru.override = p: makeTestFor p; + } +) diff --git a/nixos/tests/postgresql/timescaledb.nix b/nixos/tests/postgresql/timescaledb.nix new file mode 100644 index 0000000000000..b29d59c744f06 --- /dev/null +++ b/nixos/tests/postgresql/timescaledb.nix @@ -0,0 +1,100 @@ +{ + pkgs, + makeTest, +}: + +let + inherit (pkgs) lib; + + test-sql = pkgs.writeText "postgresql-test" '' + CREATE EXTENSION timescaledb; + CREATE EXTENSION timescaledb_toolkit; + + CREATE TABLE sth ( + time TIMESTAMPTZ NOT NULL, + value DOUBLE PRECISION + ); + + SELECT create_hypertable('sth', 'time'); + + INSERT INTO sth (time, value) VALUES + ('2003-04-12 04:05:06 America/New_York', 1.0), + ('2003-04-12 04:05:07 America/New_York', 2.0), + ('2003-04-12 04:05:08 America/New_York', 3.0), + ('2003-04-12 04:05:09 America/New_York', 4.0), + ('2003-04-12 04:05:10 America/New_York', 5.0) + ; + + WITH t AS ( + SELECT + time_bucket('1 day'::interval, time) AS dt, + stats_agg(value) AS stats + FROM sth + GROUP BY time_bucket('1 day'::interval, time) + ) + SELECT + average(stats) + FROM t; + + SELECT * FROM sth; + ''; + + makeTestFor = + package: + makeTest { + name = "timescaledb-${package.name}"; + meta = with lib.maintainers; { + maintainers = [ typetetris ]; + }; + + nodes.machine = + { ... }: + { + services.postgresql = { + inherit package; + enable = true; + extraPlugins = + ps: with ps; [ + timescaledb + timescaledb_toolkit + ]; + settings = { + shared_preload_libraries = "timescaledb, timescaledb_toolkit"; + }; + }; + }; + + testScript = '' + def check_count(statement, lines): + return 'test $(sudo -u postgres psql postgres -tAc "{}"|wc -l) -eq {}'.format( + statement, lines + ) + + + machine.start() + machine.wait_for_unit("postgresql") + + with subtest("Postgresql with extensions timescaledb and timescaledb_toolkit is available just after unit start"): + machine.succeed( + "sudo -u postgres psql -f ${test-sql}" + ) + + machine.fail(check_count("SELECT * FROM sth;", 3)) + machine.succeed(check_count("SELECT * FROM sth;", 5)) + machine.fail(check_count("SELECT * FROM sth;", 4)) + + machine.shutdown() + ''; + }; +in +# Not run by default, because this requires allowUnfree. +# To run these tests: +# NIXPKGS_ALLOW_UNFREE=1 nix-build -A nixosTests.postgresql.timescaledb +lib.dontRecurseIntoAttrs ( + lib.concatMapAttrs (n: p: { ${n} = makeTestFor p; }) ( + lib.filterAttrs (_: p: !p.pkgs.timescaledb.meta.broken) pkgs.postgresqlVersions + ) + // { + passthru.override = p: makeTestFor p; + } +) diff --git a/nixos/tests/postgresql/tsja.nix b/nixos/tests/postgresql/tsja.nix new file mode 100644 index 0000000000000..7c976da21b68f --- /dev/null +++ b/nixos/tests/postgresql/tsja.nix @@ -0,0 +1,50 @@ +{ + pkgs, + makeTest, +}: + +let + inherit (pkgs) lib; + + makeTestFor = + package: + makeTest { + name = "tsja-${package.name}"; + meta = { + maintainers = with lib.maintainers; [ chayleaf ]; + }; + + nodes.master = + { ... }: + { + services.postgresql = { + inherit package; + enable = true; + extraPlugins = + ps: with ps; [ + tsja + ]; + }; + }; + + testScript = '' + start_all() + master.wait_for_unit("postgresql") + master.succeed("sudo -u postgres psql -f /run/current-system/sw/share/postgresql/extension/libtsja_dbinit.sql") + # make sure "日本語" is parsed as a separate lexeme + master.succeed(""" + sudo -u postgres \\ + psql -c "SELECT * FROM ts_debug('japanese', 'PostgreSQLで日本語のテキスト検索ができます。')" \\ + | grep "{日本語}" + """) + ''; + }; +in +lib.recurseIntoAttrs ( + lib.concatMapAttrs (n: p: { ${n} = makeTestFor p; }) ( + lib.filterAttrs (_: p: !p.pkgs.tsja.meta.broken) pkgs.postgresqlVersions + ) + // { + passthru.override = p: makeTestFor p; + } +) diff --git a/nixos/tests/postgresql/wal2json.nix b/nixos/tests/postgresql/wal2json.nix new file mode 100644 index 0000000000000..551254a68ebda --- /dev/null +++ b/nixos/tests/postgresql/wal2json.nix @@ -0,0 +1,52 @@ +{ + pkgs, + makeTest, +}: + +let + inherit (pkgs) lib; + + makeTestFor = + package: + makeTest { + name = "wal2json-${package.name}"; + meta.maintainers = with pkgs.lib.maintainers; [ euank ]; + + nodes.machine = { + services.postgresql = { + inherit package; + enable = true; + extraPlugins = with package.pkgs; [ wal2json ]; + settings = { + wal_level = "logical"; + max_replication_slots = "10"; + max_wal_senders = "10"; + }; + }; + }; + + testScript = '' + machine.wait_for_unit("postgresql") + machine.succeed( + "sudo -u postgres psql -qAt -f ${./wal2json/example2.sql} postgres > /tmp/example2.out" + ) + machine.succeed( + "diff ${./wal2json/example2.out} /tmp/example2.out" + ) + machine.succeed( + "sudo -u postgres psql -qAt -f ${./wal2json/example3.sql} postgres > /tmp/example3.out" + ) + machine.succeed( + "diff ${./wal2json/example3.out} /tmp/example3.out" + ) + ''; + }; +in +lib.recurseIntoAttrs ( + lib.concatMapAttrs (n: p: { ${n} = makeTestFor p; }) ( + lib.filterAttrs (_: p: !p.pkgs.wal2json.meta.broken) pkgs.postgresqlVersions + ) + // { + passthru.override = p: makeTestFor p; + } +) diff --git a/nixos/tests/timescaledb.nix b/nixos/tests/timescaledb.nix deleted file mode 100644 index 8ee788daeac70..0000000000000 --- a/nixos/tests/timescaledb.nix +++ /dev/null @@ -1,93 +0,0 @@ -# mostly copied from ./postgresql.nix as it seemed unapproriate to -# test additional extensions for postgresql there. - -{ system ? builtins.currentSystem -, config ? { } -, pkgs ? import ../.. { inherit system config; } -}: - -with import ../lib/testing-python.nix { inherit system pkgs; }; -with pkgs.lib; - -let - postgresql-versions = import ../../pkgs/servers/sql/postgresql pkgs; - test-sql = pkgs.writeText "postgresql-test" '' - CREATE EXTENSION timescaledb; - CREATE EXTENSION timescaledb_toolkit; - - CREATE TABLE sth ( - time TIMESTAMPTZ NOT NULL, - value DOUBLE PRECISION - ); - - SELECT create_hypertable('sth', 'time'); - - INSERT INTO sth (time, value) VALUES - ('2003-04-12 04:05:06 America/New_York', 1.0), - ('2003-04-12 04:05:07 America/New_York', 2.0), - ('2003-04-12 04:05:08 America/New_York', 3.0), - ('2003-04-12 04:05:09 America/New_York', 4.0), - ('2003-04-12 04:05:10 America/New_York', 5.0) - ; - - WITH t AS ( - SELECT - time_bucket('1 day'::interval, time) AS dt, - stats_agg(value) AS stats - FROM sth - GROUP BY time_bucket('1 day'::interval, time) - ) - SELECT - average(stats) - FROM t; - ''; - make-postgresql-test = postgresql-name: postgresql-package: makeTest { - name = postgresql-name; - meta = with pkgs.lib.maintainers; { - maintainers = [ typetetris ]; - }; - - nodes.machine = { ... }: - { - services.postgresql = { - enable = true; - package = postgresql-package; - extraPlugins = ps: with ps; [ - timescaledb - timescaledb_toolkit - ]; - settings = { shared_preload_libraries = "timescaledb, timescaledb_toolkit"; }; - }; - }; - - testScript = '' - def check_count(statement, lines): - return 'test $(sudo -u postgres psql postgres -tAc "{}"|wc -l) -eq {}'.format( - statement, lines - ) - - - machine.start() - machine.wait_for_unit("postgresql") - - with subtest("Postgresql with extensions timescaledb and timescaledb_toolkit is available just after unit start"): - machine.succeed( - "sudo -u postgres psql -f ${test-sql}" - ) - - machine.fail(check_count("SELECT * FROM sth;", 3)) - machine.succeed(check_count("SELECT * FROM sth;", 5)) - machine.fail(check_count("SELECT * FROM sth;", 4)) - - machine.shutdown() - ''; - - }; - applicablePostgresqlVersions = filterAttrs (_: value: versionAtLeast value.version "14") postgresql-versions; -in -mapAttrs' - (name: package: { - inherit name; - value = make-postgresql-test name package; - }) - applicablePostgresqlVersions diff --git a/nixos/tests/tsja.nix b/nixos/tests/tsja.nix deleted file mode 100644 index f34358ff3f5f3..0000000000000 --- a/nixos/tests/tsja.nix +++ /dev/null @@ -1,32 +0,0 @@ -import ./make-test-python.nix ({ pkgs, lib, ...} : { - name = "tsja"; - meta = { - maintainers = with lib.maintainers; [ chayleaf ]; - }; - - nodes = { - master = - { config, ... }: - - { - services.postgresql = { - enable = true; - extraPlugins = ps: with ps; [ - tsja - ]; - }; - }; - }; - - testScript = '' - start_all() - master.wait_for_unit("postgresql") - master.succeed("sudo -u postgres psql -f /run/current-system/sw/share/postgresql/extension/libtsja_dbinit.sql") - # make sure "日本語" is parsed as a separate lexeme - master.succeed(""" - sudo -u postgres \\ - psql -c "SELECT * FROM ts_debug('japanese', 'PostgreSQLで日本語のテキスト検索ができます。')" \\ - | grep "{日本語}" - """) - ''; -}) diff --git a/pkgs/by-name/pg/pg-dump-anon/package.nix b/pkgs/by-name/pg/pg-dump-anon/package.nix index 96b6d351130f6..a097eb8aa9c71 100644 --- a/pkgs/by-name/pg/pg-dump-anon/package.nix +++ b/pkgs/by-name/pg/pg-dump-anon/package.nix @@ -14,7 +14,7 @@ buildGoModule rec { vendorHash = "sha256-CwU1zoIayxvfnGL9kPdummPJiV+ECfSz4+q6gZGb8pw="; - passthru.tests = { inherit (nixosTests) pg_anonymizer; }; + passthru.tests = { inherit (nixosTests.postgresql) anonymizer; }; nativeBuildInputs = [ makeWrapper ]; postInstall = '' diff --git a/pkgs/servers/sql/postgresql/ext/anonymizer.nix b/pkgs/servers/sql/postgresql/ext/anonymizer.nix index 4bb5aa544440e..afb517c6b5a04 100644 --- a/pkgs/servers/sql/postgresql/ext/anonymizer.nix +++ b/pkgs/servers/sql/postgresql/ext/anonymizer.nix @@ -1,9 +1,9 @@ -{ lib, stdenv, pg-dump-anon, postgresql, runtimeShell, jitSupport, llvm }: +{ lib, stdenv, pg-dump-anon, postgresql, runtimeShell, jitSupport, llvm, nixosTests }: stdenv.mkDerivation (finalAttrs: { pname = "postgresql_anonymizer"; - inherit (pg-dump-anon) version src passthru; + inherit (pg-dump-anon) version src; buildInputs = [ postgresql ]; nativeBuildInputs = [ postgresql ] ++ lib.optional jitSupport llvm; @@ -26,6 +26,8 @@ stdenv.mkDerivation (finalAttrs: { EOF ''; + passthru.tests = nixosTests.postgresql.anonymizer.passthru.override postgresql; + meta = lib.getAttrs [ "homepage" "maintainers" "license" ] pg-dump-anon.meta // { description = "Extension to mask or replace personally identifiable information (PII) or commercially sensitive data from a PostgreSQL database"; }; diff --git a/pkgs/servers/sql/postgresql/ext/apache_datasketches.nix b/pkgs/servers/sql/postgresql/ext/apache_datasketches.nix index af3df00fca644..750551a9cbfe5 100644 --- a/pkgs/servers/sql/postgresql/ext/apache_datasketches.nix +++ b/pkgs/servers/sql/postgresql/ext/apache_datasketches.nix @@ -1,4 +1,4 @@ -{ stdenv, lib, fetchFromGitHub, postgresql, boost182, nixosTests }: +{ stdenv, lib, fetchFromGitHub, postgresql, boost182, postgresqlTestExtension }: let version = "1.7.0"; @@ -20,7 +20,7 @@ let }; in -stdenv.mkDerivation { +stdenv.mkDerivation (finalAttrs: { pname = "apache_datasketches"; inherit version; @@ -61,7 +61,13 @@ stdenv.mkDerivation { runHook postInstall ''; - passthru.tests.apache_datasketches = nixosTests.apache_datasketches; + passthru.tests.extension = postgresqlTestExtension { + inherit (finalAttrs) finalPackage; + sql = '' + CREATE EXTENSION datasketches; + SELECT hll_sketch_to_string(hll_sketch_build(1)); + ''; + }; meta = { description = "PostgreSQL extension providing approximate algorithms for distinct item counts, quantile estimation and frequent items detection"; @@ -75,4 +81,4 @@ stdenv.mkDerivation { license = lib.licenses.asl20; maintainers = with lib.maintainers; [ mmusnjak ]; }; -} +}) diff --git a/pkgs/servers/sql/postgresql/ext/pgjwt.nix b/pkgs/servers/sql/postgresql/ext/pgjwt.nix index a90502c35a4e8..7bac053d920cd 100644 --- a/pkgs/servers/sql/postgresql/ext/pgjwt.nix +++ b/pkgs/servers/sql/postgresql/ext/pgjwt.nix @@ -19,8 +19,8 @@ stdenv.mkDerivation (finalAttrs: { passthru.updateScript = unstableGitUpdater { }; - passthru.tests = { - inherit (nixosTests) pgjwt; + passthru.tests = lib.recurseIntoAttrs { + pgjwt = nixosTests.postgresql.pgjwt.passthru.override postgresql; extension = postgresqlTestExtension { inherit (finalAttrs) finalPackage; diff --git a/pkgs/servers/sql/postgresql/ext/pgvecto-rs/default.nix b/pkgs/servers/sql/postgresql/ext/pgvecto-rs/default.nix index 117319e606f0f..f25afda0ca612 100644 --- a/pkgs/servers/sql/postgresql/ext/pgvecto-rs/default.nix +++ b/pkgs/servers/sql/postgresql/ext/pgvecto-rs/default.nix @@ -86,9 +86,7 @@ in passthru = { updateScript = nix-update-script { }; - tests = { - pgvecto-rs = nixosTests.pgvecto-rs; - }; + tests = nixosTests.postgresql.pgvecto-rs.passthru.override postgresql; }; meta = with lib; { diff --git a/pkgs/servers/sql/postgresql/ext/postgis.nix b/pkgs/servers/sql/postgresql/ext/postgis.nix index aa62c778a3fda..afd24f85928e9 100644 --- a/pkgs/servers/sql/postgresql/ext/postgis.nix +++ b/pkgs/servers/sql/postgresql/ext/postgis.nix @@ -18,7 +18,7 @@ docbook5, cunit, pcre2, - nixosTests, + postgresqlTestExtension, jitSupport, llvm, }: @@ -26,7 +26,7 @@ let gdal = gdalMinimal; in -stdenv.mkDerivation rec { +stdenv.mkDerivation (finalAttrs: { pname = "postgis"; version = "3.5.0"; @@ -36,7 +36,7 @@ stdenv.mkDerivation rec { ]; src = fetchurl { - url = "https://download.osgeo.org/postgis/source/postgis-${version}.tar.gz"; + url = "https://download.osgeo.org/postgis/source/postgis-${finalAttrs.version}.tar.gz"; hash = "sha256-ymmKIswrKzRnrE4GO0OihBPzAE3dUFvczddMVqZH9RA="; }; @@ -70,9 +70,9 @@ stdenv.mkDerivation rec { preConfigure = '' sed -i 's@/usr/bin/file@${file}/bin/file@' configure - configureFlags="--datadir=$out/share/postgresql --datarootdir=$out/share/postgresql --bindir=$out/bin --docdir=$doc/share/doc/${pname} --with-gdalconfig=${gdal}/bin/gdal-config --with-jsondir=${json_c.dev} --disable-extension-upgrades-install" + configureFlags="--datadir=$out/share/postgresql --datarootdir=$out/share/postgresql --bindir=$out/bin --docdir=$doc/share/doc/${finalAttrs.pname} --with-gdalconfig=${gdal}/bin/gdal-config --with-jsondir=${json_c.dev} --disable-extension-upgrades-install" - makeFlags="PERL=${perl}/bin/perl datadir=$out/share/postgresql pkglibdir=$out/lib bindir=$out/bin docdir=$doc/share/doc/${pname}" + makeFlags="PERL=${perl}/bin/perl datadir=$out/share/postgresql pkglibdir=$out/lib bindir=$out/bin docdir=$doc/share/doc/${finalAttrs.pname}" ''; postConfigure = '' sed -i "s|@mkdir -p \$(DESTDIR)\$(PGSQL_BINDIR)||g ; @@ -106,19 +106,39 @@ stdenv.mkDerivation rec { rm $out/bin/postgres for prog in $out/bin/*; do # */ - ln -s $prog $prog-${version} + ln -s $prog $prog-${finalAttrs.version} done mkdir -p $doc/share/doc/postgis mv doc/* $doc/share/doc/postgis/ ''; - passthru.tests.postgis = nixosTests.postgis; + passthru.tests.extension = postgresqlTestExtension { + inherit (finalAttrs) finalPackage; + sql = + let + expectedVersion = "${lib.versions.major finalAttrs.version}.${lib.versions.minor finalAttrs.version} USE_GEOS=1 USE_PROJ=1 USE_STATS=1"; + in + '' + CREATE EXTENSION postgis; + CREATE EXTENSION postgis_raster; + CREATE EXTENSION postgis_topology; + select postgis_version(); + do $$ + begin + if postgis_version() <> '${expectedVersion}' then + raise '"%" does not match "${expectedVersion}"', postgis_version(); + end if; + end$$; + -- st_makepoint goes through c code + select st_makepoint(1, 1); + ''; + }; meta = with lib; { description = "Geographic Objects for PostgreSQL"; homepage = "https://postgis.net/"; - changelog = "https://git.osgeo.org/gitea/postgis/postgis/raw/tag/${version}/NEWS"; + changelog = "https://git.osgeo.org/gitea/postgis/postgis/raw/tag/${finalAttrs.version}/NEWS"; license = licenses.gpl2Plus; maintainers = with maintainers; @@ -129,4 +149,4 @@ stdenv.mkDerivation rec { ]; inherit (postgresql.meta) platforms; }; -} +}) diff --git a/pkgs/servers/sql/postgresql/ext/timescaledb.nix b/pkgs/servers/sql/postgresql/ext/timescaledb.nix index 60e14e922835b..ab4736c5f7e9e 100644 --- a/pkgs/servers/sql/postgresql/ext/timescaledb.nix +++ b/pkgs/servers/sql/postgresql/ext/timescaledb.nix @@ -32,7 +32,7 @@ stdenv.mkDerivation rec { done ''; - passthru.tests = { inherit (nixosTests) timescaledb; }; + passthru.tests = nixosTests.postgresql.timescaledb.passthru.override postgresql; meta = with lib; { description = "Scales PostgreSQL for time-series data via automatic partitioning across time and space"; diff --git a/pkgs/servers/sql/postgresql/ext/timescaledb_toolkit.nix b/pkgs/servers/sql/postgresql/ext/timescaledb_toolkit.nix index ab414a400d294..eec287ddcc10d 100644 --- a/pkgs/servers/sql/postgresql/ext/timescaledb_toolkit.nix +++ b/pkgs/servers/sql/postgresql/ext/timescaledb_toolkit.nix @@ -25,9 +25,7 @@ passthru = { updateScript = nix-update-script { }; - tests = { - timescaledb_toolkit = nixosTests.timescaledb; - }; + tests = nixosTests.postgresql.timescaledb.passthru.override postgresql; }; # tests take really long diff --git a/pkgs/servers/sql/postgresql/ext/tsja.nix b/pkgs/servers/sql/postgresql/ext/tsja.nix index f0b9238f59ef2..de19b047fe9c3 100644 --- a/pkgs/servers/sql/postgresql/ext/tsja.nix +++ b/pkgs/servers/sql/postgresql/ext/tsja.nix @@ -32,7 +32,7 @@ stdenv.mkDerivation rec { mv dbinit_libtsja.txt $out/share/postgresql/extension/libtsja_dbinit.sql ''; - passthru.tests.tsja = nixosTests.tsja; + passthru.tests = nixosTests.postgresql.tsja.passthru.override postgresql; meta = with lib; { description = "PostgreSQL extension implementing Japanese text search"; diff --git a/pkgs/servers/sql/postgresql/ext/wal2json.nix b/pkgs/servers/sql/postgresql/ext/wal2json.nix index 4a1c9b18b13b5..f7d581f2c8dd4 100644 --- a/pkgs/servers/sql/postgresql/ext/wal2json.nix +++ b/pkgs/servers/sql/postgresql/ext/wal2json.nix @@ -4,6 +4,7 @@ callPackage, fetchFromGitHub, postgresql, + nixosTests, }: stdenv.mkDerivation rec { @@ -26,12 +27,7 @@ stdenv.mkDerivation rec { install -D -t $out/share/postgresql/extension sql/*.sql ''; - passthru.tests.wal2json = lib.recurseIntoAttrs ( - callPackage ../../../../../nixos/tests/postgresql-wal2json.nix { - inherit (stdenv) system; - inherit postgresql; - } - ); + passthru.tests = nixosTests.postgresql.wal2json.passthru.override postgresql; meta = with lib; { description = "PostgreSQL JSON output plugin for changeset extraction"; diff --git a/pkgs/servers/sql/postgresql/generic.nix b/pkgs/servers/sql/postgresql/generic.nix index 82a81c6798f9e..7d59f6349d57a 100644 --- a/pkgs/servers/sql/postgresql/generic.nix +++ b/pkgs/servers/sql/postgresql/generic.nix @@ -20,7 +20,7 @@ let , version, hash, muslPatches ? {} # for tests - , testers + , testers, nixosTests # JIT , jitSupport @@ -312,18 +312,12 @@ let }; tests = { - postgresql-wal-receiver = import ../../../../nixos/tests/postgresql-wal-receiver.nix { - inherit (stdenv) system; - pkgs = self; - package = this; - }; + postgresql = nixosTests.postgresql.postgresql.passthru.override finalAttrs.finalPackage; + postgresql-tls-client-cert = nixosTests.postgresql.postgresql-tls-client-cert.passthru.override finalAttrs.finalPackage; + postgresql-wal-receiver = nixosTests.postgresql.postgresql-wal-receiver.passthru.override finalAttrs.finalPackage; pkg-config = testers.testMetaPkgConfig finalAttrs.finalPackage; } // lib.optionalAttrs jitSupport { - postgresql-jit = import ../../../../nixos/tests/postgresql-jit.nix { - inherit (stdenv) system; - pkgs = self; - package = this; - }; + postgresql-jit = nixosTests.postgresql.postgresql-jit.passthru.override finalAttrs.finalPackage; }; }; diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index deabeb6e3bd4a..a2f0a06c379b6 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -12211,7 +12211,8 @@ with pkgs; asciidoc = asciidoc-full; }; - inherit (import ../servers/sql/postgresql pkgs) + postgresqlVersions = import ../servers/sql/postgresql pkgs; + inherit (postgresqlVersions) postgresql_12 postgresql_13 postgresql_14