From d3feaaebea1860ab45859f21c0b937288980ebd1 Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Sat, 2 Nov 2024 19:19:08 +0100 Subject: [PATCH 01/17] nixosTests.pgjwt: fix test This seems to have broken years ago, because "CREATE EXTENSION pgcrypto;" etc. were added to the upstream file about 6 years ago. --- nixos/tests/pgjwt.nix | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/nixos/tests/pgjwt.nix b/nixos/tests/pgjwt.nix index 3ab905cea9ac9..3ef787a96235c 100644 --- a/nixos/tests/pgjwt.nix +++ b/nixos/tests/pgjwt.nix @@ -26,10 +26,7 @@ with pkgs; { start_all() master.wait_for_unit("postgresql") master.succeed( - "${pkgs.gnused}/bin/sed -e '12 i CREATE EXTENSION pgcrypto;\\nCREATE EXTENSION pgtap;\\nSET search_path TO tap,public;' ${pgjwt.src}/test.sql > /tmp/test.sql" - ) - master.succeed( - "${pkgs.sudo}/bin/sudo -u ${sqlSU} PGOPTIONS=--search_path=tap,public ${pgProve}/bin/pg_prove -d postgres -v -f /tmp/test.sql" + "${pkgs.sudo}/bin/sudo -u ${sqlSU} ${pgProve}/bin/pg_prove -d postgres -v -f ${pgjwt.src}/test.sql" ) ''; }) From 75d51c588914be193a5e0888078ce71f6505fda4 Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Sat, 2 Nov 2024 20:13:14 +0100 Subject: [PATCH 02/17] postgresqlVersions: init Allows building all PostgreSQL versions at once with: nix-build -A postgresqlVersions Also makes it possible for nixosTests to target all PostgreSQL versions without importing the postgresql folder across the whole repo. --- pkgs/top-level/all-packages.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 38019396a0355..9523a35f0df94 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -12220,7 +12220,8 @@ with pkgs; asciidoc = asciidoc-full; }; - inherit (import ../servers/sql/postgresql pkgs) + postgresqlVersions = import ../servers/sql/postgresql pkgs; + inherit (postgresqlVersions) postgresql_12 postgresql_13 postgresql_14 From a1ae4377e090c6a79b3294c99e5743aa9af0ecd6 Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Fri, 1 Nov 2024 19:21:50 +0100 Subject: [PATCH 03/17] nixosTests.postgresql-wal-receiver: avoid manual imports Manually importing postgresql packages from the /pkgs/ folder or manually importing the test from /nixos/tests/ in generic.nix is not only ugly, but also forbidden should we ever move to pkgs/by-name. We can achieve almost the same with a slightly different setup. We allow overriding the postgresql package for the test via passthru.override, to make sure that each postgresql_xx.tests.postgresql-wal-receiver is properly teted with the right version. --- nixos/tests/postgresql-wal-receiver.nix | 19 ++++--------------- pkgs/servers/sql/postgresql/generic.nix | 8 ++------ 2 files changed, 6 insertions(+), 21 deletions(-) diff --git a/nixos/tests/postgresql-wal-receiver.nix b/nixos/tests/postgresql-wal-receiver.nix index a984f73c2be5b..9b70668e7305d 100644 --- a/nixos/tests/postgresql-wal-receiver.nix +++ b/nixos/tests/postgresql-wal-receiver.nix @@ -1,7 +1,6 @@ { system ? builtins.currentSystem, config ? {}, pkgs ? import ../.. { inherit system config; }, - package ? null }: with import ../lib/testing-python.nix { inherit system pkgs; }; @@ -9,13 +8,6 @@ with import ../lib/testing-python.nix { inherit system pkgs; }; let lib = pkgs.lib; - # Makes a test for a PostgreSQL package, given by name and looked up from `pkgs`. - makeTestAttribute = name: - { - inherit name; - value = makePostgresqlWalReceiverTest pkgs."${name}"; - }; - makePostgresqlWalReceiverTest = pkg: let postgresqlDataDir = "/var/lib/postgresql/${pkg.psqlSchema}"; @@ -114,11 +106,8 @@ let ) ''; }; - in -if package == null then - # all-tests.nix: Maps the generic function over all attributes of PostgreSQL packages - builtins.listToAttrs (map makeTestAttribute (builtins.attrNames (import ../../pkgs/servers/sql/postgresql pkgs))) -else - # Called directly from .tests - makePostgresqlWalReceiverTest package +lib.concatMapAttrs (n: p: { ${n} = makePostgresqlWalReceiverTest p; }) pkgs.postgresqlVersions +// { + passthru.override = p: makePostgresqlWalReceiverTest p; +} diff --git a/pkgs/servers/sql/postgresql/generic.nix b/pkgs/servers/sql/postgresql/generic.nix index 82a81c6798f9e..4a8daf3514cfe 100644 --- a/pkgs/servers/sql/postgresql/generic.nix +++ b/pkgs/servers/sql/postgresql/generic.nix @@ -20,7 +20,7 @@ let , version, hash, muslPatches ? {} # for tests - , testers + , testers, nixosTests # JIT , jitSupport @@ -312,11 +312,7 @@ let }; tests = { - postgresql-wal-receiver = import ../../../../nixos/tests/postgresql-wal-receiver.nix { - inherit (stdenv) system; - pkgs = self; - package = this; - }; + postgresql-wal-receiver = nixosTests.postgresql-wal-receiver.passthru.override finalAttrs.finalPackage; pkg-config = testers.testMetaPkgConfig finalAttrs.finalPackage; } // lib.optionalAttrs jitSupport { postgresql-jit = import ../../../../nixos/tests/postgresql-jit.nix { From 65ef7381c8d7108291844bef144296176b3deede Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Fri, 1 Nov 2024 19:27:43 +0100 Subject: [PATCH 04/17] nixosTests.postgresql-jit: avoid manual imports Same reasoning as commit before. --- nixos/tests/postgresql-jit.nix | 15 +++++---------- pkgs/servers/sql/postgresql/generic.nix | 6 +----- 2 files changed, 6 insertions(+), 15 deletions(-) diff --git a/nixos/tests/postgresql-jit.nix b/nixos/tests/postgresql-jit.nix index f4b1d07a7faf8..fb598972ba0fb 100644 --- a/nixos/tests/postgresql-jit.nix +++ b/nixos/tests/postgresql-jit.nix @@ -1,20 +1,15 @@ { system ? builtins.currentSystem , config ? {} , pkgs ? import ../.. { inherit system config; } -, package ? null }: with import ../lib/testing-python.nix { inherit system pkgs; }; let inherit (pkgs) lib; - packages = builtins.attrNames (import ../../pkgs/servers/sql/postgresql pkgs); - - mkJitTestFromName = name: - mkJitTest pkgs.${name}; mkJitTest = package: makeTest { - name = package.name; + name = "postgresql-jit-${package.name}"; meta.maintainers = with lib.maintainers; [ ma27 ]; nodes.machine = { pkgs, lib, ... }: { services.postgresql = { @@ -49,7 +44,7 @@ let ''; }; in -if package == null then - lib.genAttrs packages mkJitTestFromName -else - mkJitTest package +lib.concatMapAttrs (n: p: { ${n} = mkJitTest p; }) (lib.filterAttrs (n: _: lib.hasSuffix "_jit" n) pkgs.postgresqlVersions) +// { + passthru.override = p: mkJitTest p; +} diff --git a/pkgs/servers/sql/postgresql/generic.nix b/pkgs/servers/sql/postgresql/generic.nix index 4a8daf3514cfe..548fcd561e73f 100644 --- a/pkgs/servers/sql/postgresql/generic.nix +++ b/pkgs/servers/sql/postgresql/generic.nix @@ -315,11 +315,7 @@ let postgresql-wal-receiver = nixosTests.postgresql-wal-receiver.passthru.override finalAttrs.finalPackage; pkg-config = testers.testMetaPkgConfig finalAttrs.finalPackage; } // lib.optionalAttrs jitSupport { - postgresql-jit = import ../../../../nixos/tests/postgresql-jit.nix { - inherit (stdenv) system; - pkgs = self; - package = this; - }; + postgresql-jit = nixosTests.postgresql-jit.passthru.override finalAttrs.finalPackage; }; }; From 319d82d5c218514d1cd5b0b6d711c318f11d08f0 Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Fri, 1 Nov 2024 19:33:14 +0100 Subject: [PATCH 05/17] nixosTests.postgresql-wal2json: avoid manual imports Same reasoning as commit before. --- nixos/tests/postgresql-wal2json.nix | 22 +++++++------------- pkgs/servers/sql/postgresql/ext/wal2json.nix | 8 ++----- 2 files changed, 9 insertions(+), 21 deletions(-) diff --git a/nixos/tests/postgresql-wal2json.nix b/nixos/tests/postgresql-wal2json.nix index 043ad48cbc6ec..2ad1a1ee66a0a 100644 --- a/nixos/tests/postgresql-wal2json.nix +++ b/nixos/tests/postgresql-wal2json.nix @@ -2,16 +2,12 @@ system ? builtins.currentSystem, config ? { }, pkgs ? import ../.. { inherit system config; }, - postgresql ? null, }: +with import ../lib/testing-python.nix { inherit system pkgs; }; + let - makeTest = import ./make-test-python.nix; - # Makes a test for a PostgreSQL package, given by name and looked up from `pkgs`. - makeTestAttribute = name: { - inherit name; - value = makePostgresqlWal2jsonTest pkgs."${name}"; - }; + inherit (pkgs) lib; makePostgresqlWal2jsonTest = postgresqlPackage: @@ -50,11 +46,7 @@ let }; in -# By default, create one test per postgresql version -if postgresql == null then - builtins.listToAttrs ( - map makeTestAttribute (builtins.attrNames (import ../../pkgs/servers/sql/postgresql pkgs)) - ) -# but if postgresql is set, we're being made as a passthru test for a specific postgres + wal2json version, just run one -else - makePostgresqlWal2jsonTest postgresql +lib.concatMapAttrs (n: p: { ${n} = makePostgresqlWal2jsonTest p; }) pkgs.postgresqlVersions +// { + passthru.override = p: makePostgresqlWal2jsonTest p; +} diff --git a/pkgs/servers/sql/postgresql/ext/wal2json.nix b/pkgs/servers/sql/postgresql/ext/wal2json.nix index 4a1c9b18b13b5..25f56db522227 100644 --- a/pkgs/servers/sql/postgresql/ext/wal2json.nix +++ b/pkgs/servers/sql/postgresql/ext/wal2json.nix @@ -4,6 +4,7 @@ callPackage, fetchFromGitHub, postgresql, + nixosTests, }: stdenv.mkDerivation rec { @@ -26,12 +27,7 @@ stdenv.mkDerivation rec { install -D -t $out/share/postgresql/extension sql/*.sql ''; - passthru.tests.wal2json = lib.recurseIntoAttrs ( - callPackage ../../../../../nixos/tests/postgresql-wal2json.nix { - inherit (stdenv) system; - inherit postgresql; - } - ); + passthru.tests = nixosTests.postgresql-wal2json.passthru.override postgresql; meta = with lib; { description = "PostgreSQL JSON output plugin for changeset extraction"; From f6c2de926290a2d62d06876b1df7caffcac13150 Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Fri, 1 Nov 2024 19:29:09 +0100 Subject: [PATCH 06/17] postgresql: add passthru.tests.postgresql Restructuring the nixosTests.postgresql test a little bit to allow calling it with the specific versioned package from generic.nix. --- nixos/tests/postgresql.nix | 26 +++++++++++++++---------- pkgs/servers/sql/postgresql/generic.nix | 1 + 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/nixos/tests/postgresql.nix b/nixos/tests/postgresql.nix index ce16e54edf662..4f6c37643c257 100644 --- a/nixos/tests/postgresql.nix +++ b/nixos/tests/postgresql.nix @@ -7,7 +7,13 @@ with import ../lib/testing-python.nix { inherit system pkgs; }; with pkgs.lib; let - postgresql-versions = import ../../pkgs/servers/sql/postgresql pkgs; + testsForPackage = package: + recurseIntoAttrs { + postgresql = make-postgresql-test package false; + postgresql-backup-all = make-postgresql-test package true; + postgresql-clauses = mk-ensure-clauses-test package; + }; + test-sql = pkgs.writeText "postgresql-test" '' CREATE EXTENSION pgcrypto; -- just to check if lib loading works CREATE TABLE sth ( @@ -21,8 +27,9 @@ let CREATE TABLE xmltest ( doc xml ); INSERT INTO xmltest (doc) VALUES ('ok'); -- check if libxml2 enabled ''; - make-postgresql-test = postgresql-name: postgresql-package: backup-all: makeTest { - name = postgresql-name; + + make-postgresql-test = postgresql-package: backup-all: makeTest { + name = "postgresql${optionalString backup-all "-backup-all"}-${postgresql-package.name}"; meta = with pkgs.lib.maintainers; { maintainers = [ zagy ]; }; @@ -133,8 +140,8 @@ let }; - mk-ensure-clauses-test = postgresql-name: postgresql-package: makeTest { - name = postgresql-name; + mk-ensure-clauses-test = postgresql-package: makeTest { + name = "postgresql-clauses-${postgresql-package.name}"; meta = with pkgs.lib.maintainers; { maintainers = [ zagy ]; }; @@ -219,8 +226,7 @@ let ''; }; in - concatMapAttrs (name: package: { - ${name} = make-postgresql-test name package false; - ${name + "-backup-all"} = make-postgresql-test "${name + "-backup-all"}" package true; - ${name + "-clauses"} = mk-ensure-clauses-test name package; - }) postgresql-versions +concatMapAttrs (n: p: { ${n} = testsForPackage p; }) pkgs.postgresqlVersions +// { + passthru.override = p: testsForPackage p; +} diff --git a/pkgs/servers/sql/postgresql/generic.nix b/pkgs/servers/sql/postgresql/generic.nix index 548fcd561e73f..587f150bc6da2 100644 --- a/pkgs/servers/sql/postgresql/generic.nix +++ b/pkgs/servers/sql/postgresql/generic.nix @@ -312,6 +312,7 @@ let }; tests = { + postgresql = nixosTests.postgresql.passthru.override finalAttrs.finalPackage; postgresql-wal-receiver = nixosTests.postgresql-wal-receiver.passthru.override finalAttrs.finalPackage; pkg-config = testers.testMetaPkgConfig finalAttrs.finalPackage; } // lib.optionalAttrs jitSupport { From 139c5466764b62fe5f9b7b5efa8c765f757d0a71 Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Fri, 1 Nov 2024 19:29:41 +0100 Subject: [PATCH 07/17] postgresql: add passthru.tests.postgresql-tls-client-cert Same reasoning as commit before. --- nixos/tests/postgresql-tls-client-cert.nix | 18 ++++-------------- pkgs/servers/sql/postgresql/generic.nix | 1 + 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/nixos/tests/postgresql-tls-client-cert.nix b/nixos/tests/postgresql-tls-client-cert.nix index c1678ed733beb..86dada85f272b 100644 --- a/nixos/tests/postgresql-tls-client-cert.nix +++ b/nixos/tests/postgresql-tls-client-cert.nix @@ -1,7 +1,6 @@ { system ? builtins.currentSystem , config ? { } , pkgs ? import ../.. { inherit system config; } -, package ? null }: with import ../lib/testing-python.nix { inherit system pkgs; }; @@ -9,13 +8,6 @@ with import ../lib/testing-python.nix { inherit system pkgs; }; let lib = pkgs.lib; - # Makes a test for a PostgreSQL package, given by name and looked up from `pkgs`. - makeTestAttribute = name: - { - inherit name; - value = makePostgresqlTlsClientCertTest pkgs."${name}"; - }; - makePostgresqlTlsClientCertTest = pkg: let runWithOpenSSL = file: cmd: pkgs.runCommand file @@ -133,9 +125,7 @@ let }; in -if package == null then -# all-tests.nix: Maps the generic function over all attributes of PostgreSQL packages - builtins.listToAttrs (map makeTestAttribute (builtins.attrNames (import ../../pkgs/servers/sql/postgresql pkgs))) -else -# Called directly from .tests - makePostgresqlTlsClientCertTest package +lib.concatMapAttrs (n: p: { ${n} = makePostgresqlTlsClientCertTest p; }) pkgs.postgresqlVersions +// { + passthru.override = p: makePostgresqlTlsClientCertTest p; +} diff --git a/pkgs/servers/sql/postgresql/generic.nix b/pkgs/servers/sql/postgresql/generic.nix index 587f150bc6da2..6c8530ce8b54e 100644 --- a/pkgs/servers/sql/postgresql/generic.nix +++ b/pkgs/servers/sql/postgresql/generic.nix @@ -313,6 +313,7 @@ let tests = { postgresql = nixosTests.postgresql.passthru.override finalAttrs.finalPackage; + postgresql-tls-client-cert = nixosTests.postgresql-tls-client-cert.passthru.override finalAttrs.finalPackage; postgresql-wal-receiver = nixosTests.postgresql-wal-receiver.passthru.override finalAttrs.finalPackage; pkg-config = testers.testMetaPkgConfig finalAttrs.finalPackage; } // lib.optionalAttrs jitSupport { From aded718a9824b385143e46a3b68a051741609e3e Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Fri, 1 Nov 2024 19:56:51 +0100 Subject: [PATCH 08/17] postgresqlPackages.apache_datasketches: move nixosTests.apache_datasketches into package There is no need to fire up a whole VM just to run a two line test of creating the extension. We can use postgresqlTestExtension for that. This has the advantage that it runs with postgresqlTestHook, so without a VM, making it more portable. --- nixos/tests/all-tests.nix | 1 - nixos/tests/apache_datasketches.nix | 29 ------------------- .../postgresql/ext/apache_datasketches.nix | 14 ++++++--- 3 files changed, 10 insertions(+), 34 deletions(-) delete mode 100644 nixos/tests/apache_datasketches.nix diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index 1288955272aa3..d07fd0e205580 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -815,7 +815,6 @@ in { postfix-raise-smtpd-tls-security-level = handleTest ./postfix-raise-smtpd-tls-security-level.nix {}; postfixadmin = handleTest ./postfixadmin.nix {}; postgis = handleTest ./postgis.nix {}; - apache_datasketches = handleTest ./apache_datasketches.nix {}; postgresql = handleTest ./postgresql.nix {}; postgresql-jit = handleTest ./postgresql-jit.nix {}; postgresql-wal-receiver = handleTest ./postgresql-wal-receiver.nix {}; diff --git a/nixos/tests/apache_datasketches.nix b/nixos/tests/apache_datasketches.nix deleted file mode 100644 index 2bf099ac7991b..0000000000000 --- a/nixos/tests/apache_datasketches.nix +++ /dev/null @@ -1,29 +0,0 @@ -import ./make-test-python.nix ({ pkgs, ...} : { - name = "postgis"; - meta = with pkgs.lib.maintainers; { - maintainers = [ lsix ]; # TODO: Who's the maintener now? - }; - - nodes = { - master = - { pkgs, ... }: - - { - services.postgresql = let mypg = pkgs.postgresql_15; in { - enable = true; - package = mypg; - extraPlugins = with mypg.pkgs; [ - apache_datasketches - ]; - }; - }; - }; - - testScript = '' - start_all() - master.wait_for_unit("postgresql") - master.sleep(10) # Hopefully this is long enough!! - master.succeed("sudo -u postgres psql -c 'CREATE EXTENSION datasketches;'") - master.succeed("sudo -u postgres psql -c 'SELECT hll_sketch_to_string(hll_sketch_build(1));'") - ''; -}) diff --git a/pkgs/servers/sql/postgresql/ext/apache_datasketches.nix b/pkgs/servers/sql/postgresql/ext/apache_datasketches.nix index af3df00fca644..750551a9cbfe5 100644 --- a/pkgs/servers/sql/postgresql/ext/apache_datasketches.nix +++ b/pkgs/servers/sql/postgresql/ext/apache_datasketches.nix @@ -1,4 +1,4 @@ -{ stdenv, lib, fetchFromGitHub, postgresql, boost182, nixosTests }: +{ stdenv, lib, fetchFromGitHub, postgresql, boost182, postgresqlTestExtension }: let version = "1.7.0"; @@ -20,7 +20,7 @@ let }; in -stdenv.mkDerivation { +stdenv.mkDerivation (finalAttrs: { pname = "apache_datasketches"; inherit version; @@ -61,7 +61,13 @@ stdenv.mkDerivation { runHook postInstall ''; - passthru.tests.apache_datasketches = nixosTests.apache_datasketches; + passthru.tests.extension = postgresqlTestExtension { + inherit (finalAttrs) finalPackage; + sql = '' + CREATE EXTENSION datasketches; + SELECT hll_sketch_to_string(hll_sketch_build(1)); + ''; + }; meta = { description = "PostgreSQL extension providing approximate algorithms for distinct item counts, quantile estimation and frequent items detection"; @@ -75,4 +81,4 @@ stdenv.mkDerivation { license = lib.licenses.asl20; maintainers = with lib.maintainers; [ mmusnjak ]; }; -} +}) From ecffab1fdaf8727fad48cfbacbbeeb6ef6d183de Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Fri, 1 Nov 2024 19:32:30 +0100 Subject: [PATCH 09/17] postgresqlPackages.postgis: move nixosTests.postgis into package Same reasoning as commit before. --- nixos/tests/all-tests.nix | 1 - nixos/tests/postgis.nix | 38 --------------------- pkgs/servers/sql/postgresql/ext/postgis.nix | 38 ++++++++++++++++----- 3 files changed, 29 insertions(+), 48 deletions(-) delete mode 100644 nixos/tests/postgis.nix diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index d07fd0e205580..bb674910e700d 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -814,7 +814,6 @@ in { postfix = handleTest ./postfix.nix {}; postfix-raise-smtpd-tls-security-level = handleTest ./postfix-raise-smtpd-tls-security-level.nix {}; postfixadmin = handleTest ./postfixadmin.nix {}; - postgis = handleTest ./postgis.nix {}; postgresql = handleTest ./postgresql.nix {}; postgresql-jit = handleTest ./postgresql-jit.nix {}; postgresql-wal-receiver = handleTest ./postgresql-wal-receiver.nix {}; diff --git a/nixos/tests/postgis.nix b/nixos/tests/postgis.nix deleted file mode 100644 index 5b63e2ba73eb1..0000000000000 --- a/nixos/tests/postgis.nix +++ /dev/null @@ -1,38 +0,0 @@ -import ./make-test-python.nix ({ pkgs, ...} : { - name = "postgis"; - meta = with pkgs.lib.maintainers; { - maintainers = [ lsix ]; - }; - - nodes = { - master = - { pkgs, ... }: - - { - services.postgresql = { - enable = true; - package = pkgs.postgresql; - extraPlugins = ps: with ps; [ - postgis - ]; - }; - }; - }; - - testScript = '' - start_all() - master.wait_for_unit("postgresql") - master.sleep(10) # Hopefully this is long enough!! - master.succeed("sudo -u postgres psql -c 'CREATE EXTENSION postgis;'") - master.succeed("sudo -u postgres psql -c 'CREATE EXTENSION postgis_raster;'") - master.succeed("sudo -u postgres psql -c 'CREATE EXTENSION postgis_topology;'") - master.succeed("sudo -u postgres psql -c 'select postgis_version();'") - master.succeed("[ \"$(sudo -u postgres psql --no-psqlrc --tuples-only -c 'select postgis_version();')\" = \" ${ - pkgs.lib.versions.major pkgs.postgis.version - }.${ - pkgs.lib.versions.minor pkgs.postgis.version - } USE_GEOS=1 USE_PROJ=1 USE_STATS=1\" ]") - # st_makepoint goes through c code - master.succeed("sudo -u postgres psql --no-psqlrc --tuples-only -c 'select st_makepoint(1, 1)'") - ''; -}) diff --git a/pkgs/servers/sql/postgresql/ext/postgis.nix b/pkgs/servers/sql/postgresql/ext/postgis.nix index aa62c778a3fda..afd24f85928e9 100644 --- a/pkgs/servers/sql/postgresql/ext/postgis.nix +++ b/pkgs/servers/sql/postgresql/ext/postgis.nix @@ -18,7 +18,7 @@ docbook5, cunit, pcre2, - nixosTests, + postgresqlTestExtension, jitSupport, llvm, }: @@ -26,7 +26,7 @@ let gdal = gdalMinimal; in -stdenv.mkDerivation rec { +stdenv.mkDerivation (finalAttrs: { pname = "postgis"; version = "3.5.0"; @@ -36,7 +36,7 @@ stdenv.mkDerivation rec { ]; src = fetchurl { - url = "https://download.osgeo.org/postgis/source/postgis-${version}.tar.gz"; + url = "https://download.osgeo.org/postgis/source/postgis-${finalAttrs.version}.tar.gz"; hash = "sha256-ymmKIswrKzRnrE4GO0OihBPzAE3dUFvczddMVqZH9RA="; }; @@ -70,9 +70,9 @@ stdenv.mkDerivation rec { preConfigure = '' sed -i 's@/usr/bin/file@${file}/bin/file@' configure - configureFlags="--datadir=$out/share/postgresql --datarootdir=$out/share/postgresql --bindir=$out/bin --docdir=$doc/share/doc/${pname} --with-gdalconfig=${gdal}/bin/gdal-config --with-jsondir=${json_c.dev} --disable-extension-upgrades-install" + configureFlags="--datadir=$out/share/postgresql --datarootdir=$out/share/postgresql --bindir=$out/bin --docdir=$doc/share/doc/${finalAttrs.pname} --with-gdalconfig=${gdal}/bin/gdal-config --with-jsondir=${json_c.dev} --disable-extension-upgrades-install" - makeFlags="PERL=${perl}/bin/perl datadir=$out/share/postgresql pkglibdir=$out/lib bindir=$out/bin docdir=$doc/share/doc/${pname}" + makeFlags="PERL=${perl}/bin/perl datadir=$out/share/postgresql pkglibdir=$out/lib bindir=$out/bin docdir=$doc/share/doc/${finalAttrs.pname}" ''; postConfigure = '' sed -i "s|@mkdir -p \$(DESTDIR)\$(PGSQL_BINDIR)||g ; @@ -106,19 +106,39 @@ stdenv.mkDerivation rec { rm $out/bin/postgres for prog in $out/bin/*; do # */ - ln -s $prog $prog-${version} + ln -s $prog $prog-${finalAttrs.version} done mkdir -p $doc/share/doc/postgis mv doc/* $doc/share/doc/postgis/ ''; - passthru.tests.postgis = nixosTests.postgis; + passthru.tests.extension = postgresqlTestExtension { + inherit (finalAttrs) finalPackage; + sql = + let + expectedVersion = "${lib.versions.major finalAttrs.version}.${lib.versions.minor finalAttrs.version} USE_GEOS=1 USE_PROJ=1 USE_STATS=1"; + in + '' + CREATE EXTENSION postgis; + CREATE EXTENSION postgis_raster; + CREATE EXTENSION postgis_topology; + select postgis_version(); + do $$ + begin + if postgis_version() <> '${expectedVersion}' then + raise '"%" does not match "${expectedVersion}"', postgis_version(); + end if; + end$$; + -- st_makepoint goes through c code + select st_makepoint(1, 1); + ''; + }; meta = with lib; { description = "Geographic Objects for PostgreSQL"; homepage = "https://postgis.net/"; - changelog = "https://git.osgeo.org/gitea/postgis/postgis/raw/tag/${version}/NEWS"; + changelog = "https://git.osgeo.org/gitea/postgis/postgis/raw/tag/${finalAttrs.version}/NEWS"; license = licenses.gpl2Plus; maintainers = with maintainers; @@ -129,4 +149,4 @@ stdenv.mkDerivation rec { ]; inherit (postgresql.meta) platforms; }; -} +}) From 0af934adf74002875db137b9fe6dc8d25ebe1cb2 Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Fri, 1 Nov 2024 19:34:26 +0100 Subject: [PATCH 10/17] postgresqlPackages.pgjwt: make passthru.tests work with correct package Same reasoning as commit before. --- nixos/tests/pgjwt.nix | 71 ++++++++++++++--------- pkgs/servers/sql/postgresql/ext/pgjwt.nix | 4 +- 2 files changed, 45 insertions(+), 30 deletions(-) diff --git a/nixos/tests/pgjwt.nix b/nixos/tests/pgjwt.nix index 3ef787a96235c..5c14ee68b5954 100644 --- a/nixos/tests/pgjwt.nix +++ b/nixos/tests/pgjwt.nix @@ -1,32 +1,47 @@ -import ./make-test-python.nix ({ pkgs, lib, ...}: +{ system ? builtins.currentSystem +, config ? {} +, pkgs ? import ../.. { inherit system config; } +}: -with pkgs; { - name = "pgjwt"; - meta = with lib.maintainers; { - maintainers = [ spinus willibutz ]; - }; +with import ../lib/testing-python.nix { inherit system pkgs; }; - nodes = { - master = { ... }: - { - services.postgresql = { - enable = true; - extraPlugins = ps: with ps; [ pgjwt pgtap ]; +let + inherit (pkgs) lib; + + makePgjwtTest = postgresqlPackage: + makeTest { + name = "pgjwt-${postgresqlPackage.name}"; + meta = with lib.maintainers; { + maintainers = [ spinus willibutz ]; }; - }; - }; - testScript = { nodes, ... }: - let - sqlSU = "${nodes.master.services.postgresql.superUser}"; - pgProve = "${pkgs.perlPackages.TAPParserSourceHandlerpgTAP}"; - inherit (nodes.master.services.postgresql.package.pkgs) pgjwt; - in - '' - start_all() - master.wait_for_unit("postgresql") - master.succeed( - "${pkgs.sudo}/bin/sudo -u ${sqlSU} ${pgProve}/bin/pg_prove -d postgres -v -f ${pgjwt.src}/test.sql" - ) - ''; -}) + nodes = { + master = { ... }: + { + services.postgresql = { + enable = true; + package = postgresqlPackage; + extraPlugins = ps: with ps; [ pgjwt pgtap ]; + }; + }; + }; + + testScript = { nodes, ... }: + let + sqlSU = "${nodes.master.services.postgresql.superUser}"; + pgProve = "${pkgs.perlPackages.TAPParserSourceHandlerpgTAP}"; + inherit (nodes.master.services.postgresql.package.pkgs) pgjwt; + in + '' + start_all() + master.wait_for_unit("postgresql") + master.succeed( + "${pkgs.sudo}/bin/sudo -u ${sqlSU} ${pgProve}/bin/pg_prove -d postgres -v -f ${pgjwt.src}/test.sql" + ) + ''; + }; +in +lib.concatMapAttrs (n: p: { ${n} = makePgjwtTest p; }) pkgs.postgresqlVersions +// { + passthru.override = p: makePgjwtTest p; +} diff --git a/pkgs/servers/sql/postgresql/ext/pgjwt.nix b/pkgs/servers/sql/postgresql/ext/pgjwt.nix index a90502c35a4e8..0209ab27d59e0 100644 --- a/pkgs/servers/sql/postgresql/ext/pgjwt.nix +++ b/pkgs/servers/sql/postgresql/ext/pgjwt.nix @@ -19,8 +19,8 @@ stdenv.mkDerivation (finalAttrs: { passthru.updateScript = unstableGitUpdater { }; - passthru.tests = { - inherit (nixosTests) pgjwt; + passthru.tests = lib.recurseIntoAttrs { + pgjwt = nixosTests.pgjwt.passthru.override postgresql; extension = postgresqlTestExtension { inherit (finalAttrs) finalPackage; From a5c41ae80a2f20e61acbce984553562b3af16782 Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Fri, 1 Nov 2024 19:47:08 +0100 Subject: [PATCH 11/17] postgresqlPackages.pgvecto-rs: make passthru.tests work with correct package Same reasoning as commit before. --- nixos/tests/pgvecto-rs.nix | 30 +++++++++---------- .../sql/postgresql/ext/pgvecto-rs/default.nix | 4 +-- 2 files changed, 15 insertions(+), 19 deletions(-) diff --git a/nixos/tests/pgvecto-rs.nix b/nixos/tests/pgvecto-rs.nix index 8d9d6c0b88f51..4a2b75132ed5f 100644 --- a/nixos/tests/pgvecto-rs.nix +++ b/nixos/tests/pgvecto-rs.nix @@ -1,6 +1,3 @@ -# mostly copied from ./timescaledb.nix which was copied from ./postgresql.nix -# as it seemed unapproriate to test additional extensions for postgresql there. - { system ? builtins.currentSystem , config ? { } , pkgs ? import ../.. { inherit system config; } @@ -10,7 +7,6 @@ with import ../lib/testing-python.nix { inherit system pkgs; }; with pkgs.lib; let - postgresql-versions = import ../../pkgs/servers/sql/postgresql pkgs; # Test cases from https://docs.pgvecto.rs/use-cases/hybrid-search.html test-sql = pkgs.writeText "postgresql-test" '' CREATE EXTENSION vectors; @@ -27,8 +23,9 @@ let ('a thin cat sat on a mat and ate a thin rat', '[7, 8, 9]'), ('a thin dog sat on a mat and ate a thin rat', '[10, 11, 12]'); ''; - make-postgresql-test = postgresql-name: postgresql-package: makeTest { - name = postgresql-name; + + makePgVectorsTest = postgresqlPackage: makeTest { + name = "pgvecto-rs-${postgresqlPackage.name}"; meta = with pkgs.lib.maintainers; { maintainers = [ diogotcorreia ]; }; @@ -37,7 +34,7 @@ let { services.postgresql = { enable = true; - package = postgresql-package; + package = postgresqlPackage; extraPlugins = ps: with ps; [ pgvecto-rs ]; @@ -45,7 +42,11 @@ let }; }; - testScript = '' + testScript = { nodes, ... }: + let + inherit (nodes.machine.services.postgresql.package.pkgs) pgvecto-rs; + in + '' def check_count(statement, lines): return 'test $(sudo -u postgres psql postgres -tAc "{}"|wc -l) -eq {}'.format( statement, lines @@ -56,7 +57,7 @@ let machine.wait_for_unit("postgresql") with subtest("Postgresql with extension vectors is available just after unit start"): - machine.succeed(check_count("SELECT * FROM pg_available_extensions WHERE name = 'vectors' AND default_version = '${postgresql-package.pkgs.pgvecto-rs.version}';", 1)) + machine.succeed(check_count("SELECT * FROM pg_available_extensions WHERE name = 'vectors' AND default_version = '${pgvecto-rs.version}';", 1)) machine.succeed("sudo -u postgres psql -f ${test-sql}") @@ -66,11 +67,8 @@ let ''; }; - applicablePostgresqlVersions = filterAttrs (_: value: versionAtLeast value.version "14") postgresql-versions; in -mapAttrs' - (name: package: { - inherit name; - value = make-postgresql-test name package; - }) - applicablePostgresqlVersions +concatMapAttrs (n: p: { ${n} = makePgVectorsTest p; }) (filterAttrs (n: p: !p.pkgs.pgvecto-rs.meta.broken) pkgs.postgresqlVersions) +// { + passthru.override = p: makePgVectorsTest p; +} diff --git a/pkgs/servers/sql/postgresql/ext/pgvecto-rs/default.nix b/pkgs/servers/sql/postgresql/ext/pgvecto-rs/default.nix index 117319e606f0f..40cde8eeae0ba 100644 --- a/pkgs/servers/sql/postgresql/ext/pgvecto-rs/default.nix +++ b/pkgs/servers/sql/postgresql/ext/pgvecto-rs/default.nix @@ -86,9 +86,7 @@ in passthru = { updateScript = nix-update-script { }; - tests = { - pgvecto-rs = nixosTests.pgvecto-rs; - }; + tests = nixosTests.pgvecto-rs.passthru.override postgresql; }; meta = with lib; { From 6d7da20a9044a6a37d5c83ca7a467e4193b3cca9 Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Fri, 1 Nov 2024 19:49:55 +0100 Subject: [PATCH 12/17] postgresqlPackages.tsja: make passthru.tests work with correct package Same reasoning as commit before. --- nixos/tests/tsja.nix | 71 ++++++++++++++---------- pkgs/servers/sql/postgresql/ext/tsja.nix | 2 +- 2 files changed, 44 insertions(+), 29 deletions(-) diff --git a/nixos/tests/tsja.nix b/nixos/tests/tsja.nix index f34358ff3f5f3..493b6d56165e7 100644 --- a/nixos/tests/tsja.nix +++ b/nixos/tests/tsja.nix @@ -1,32 +1,47 @@ -import ./make-test-python.nix ({ pkgs, lib, ...} : { - name = "tsja"; - meta = { - maintainers = with lib.maintainers; [ chayleaf ]; - }; +{ system ? builtins.currentSystem +, config ? {} +, pkgs ? import ../.. { inherit system config; } +}: - nodes = { - master = - { config, ... }: +with import ../lib/testing-python.nix { inherit system pkgs; }; - { - services.postgresql = { - enable = true; - extraPlugins = ps: with ps; [ - tsja - ]; - }; +let + makeTsjaTest = postgresqlPackage: + makeTest { + name = "tsja-${postgresqlPackage.name}"; + meta = { + maintainers = with lib.maintainers; [ chayleaf ]; }; - }; - testScript = '' - start_all() - master.wait_for_unit("postgresql") - master.succeed("sudo -u postgres psql -f /run/current-system/sw/share/postgresql/extension/libtsja_dbinit.sql") - # make sure "日本語" is parsed as a separate lexeme - master.succeed(""" - sudo -u postgres \\ - psql -c "SELECT * FROM ts_debug('japanese', 'PostgreSQLで日本語のテキスト検索ができます。')" \\ - | grep "{日本語}" - """) - ''; -}) + nodes = { + master = + { config, ... }: + + { + services.postgresql = { + enable = true; + package = postgresqlPackage; + extraPlugins = ps: with ps; [ + tsja + ]; + }; + }; + }; + + testScript = '' + start_all() + master.wait_for_unit("postgresql") + master.succeed("sudo -u postgres psql -f /run/current-system/sw/share/postgresql/extension/libtsja_dbinit.sql") + # make sure "日本語" is parsed as a separate lexeme + master.succeed(""" + sudo -u postgres \\ + psql -c "SELECT * FROM ts_debug('japanese', 'PostgreSQLで日本語のテキスト検索ができます。')" \\ + | grep "{日本語}" + """) + ''; + }; +in +pkgs.lib.concatMapAttrs (n: p: { ${n} = makeTsjaTest p; }) pkgs.postgresqlVersions +// { + passthru.override = p: makeTsjaTest p; +} diff --git a/pkgs/servers/sql/postgresql/ext/tsja.nix b/pkgs/servers/sql/postgresql/ext/tsja.nix index f0b9238f59ef2..38df785e7f7c3 100644 --- a/pkgs/servers/sql/postgresql/ext/tsja.nix +++ b/pkgs/servers/sql/postgresql/ext/tsja.nix @@ -32,7 +32,7 @@ stdenv.mkDerivation rec { mv dbinit_libtsja.txt $out/share/postgresql/extension/libtsja_dbinit.sql ''; - passthru.tests.tsja = nixosTests.tsja; + passthru.tests = nixosTests.tsja.passthru.override postgresql; meta = with lib; { description = "PostgreSQL extension implementing Japanese text search"; From 23c19a255fab8dc09105851748583272755c5db7 Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Fri, 1 Nov 2024 20:48:25 +0100 Subject: [PATCH 13/17] postgresqlPackages.timescaledb: make passthru.tests work with correct package Same reasoning as commit before. --- nixos/tests/timescaledb.nix | 24 ++++++++----------- .../sql/postgresql/ext/timescaledb.nix | 2 +- .../postgresql/ext/timescaledb_toolkit.nix | 4 +--- 3 files changed, 12 insertions(+), 18 deletions(-) diff --git a/nixos/tests/timescaledb.nix b/nixos/tests/timescaledb.nix index 8ee788daeac70..6c6ba8ab6c6dd 100644 --- a/nixos/tests/timescaledb.nix +++ b/nixos/tests/timescaledb.nix @@ -1,6 +1,3 @@ -# mostly copied from ./postgresql.nix as it seemed unapproriate to -# test additional extensions for postgresql there. - { system ? builtins.currentSystem , config ? { } , pkgs ? import ../.. { inherit system config; } @@ -10,7 +7,6 @@ with import ../lib/testing-python.nix { inherit system pkgs; }; with pkgs.lib; let - postgresql-versions = import ../../pkgs/servers/sql/postgresql pkgs; test-sql = pkgs.writeText "postgresql-test" '' CREATE EXTENSION timescaledb; CREATE EXTENSION timescaledb_toolkit; @@ -40,9 +36,12 @@ let SELECT average(stats) FROM t; + + SELECT * FROM sth; ''; - make-postgresql-test = postgresql-name: postgresql-package: makeTest { - name = postgresql-name; + + makeTimescaleDbTest = postgresqlPackage: makeTest { + name = "timescaledb-${postgresqlPackage.name}"; meta = with pkgs.lib.maintainers; { maintainers = [ typetetris ]; }; @@ -51,7 +50,7 @@ let { services.postgresql = { enable = true; - package = postgresql-package; + package = postgresqlPackage; extraPlugins = ps: with ps; [ timescaledb timescaledb_toolkit @@ -83,11 +82,8 @@ let ''; }; - applicablePostgresqlVersions = filterAttrs (_: value: versionAtLeast value.version "14") postgresql-versions; in -mapAttrs' - (name: package: { - inherit name; - value = make-postgresql-test name package; - }) - applicablePostgresqlVersions +pkgs.lib.concatMapAttrs (n: p: { ${n} = makeTimescaleDbTest p; }) (filterAttrs (n: p: !p.pkgs.timescaledb.meta.broken) pkgs.postgresqlVersions) +// { + passthru.override = p: makeTimescaleDbTest p; +} diff --git a/pkgs/servers/sql/postgresql/ext/timescaledb.nix b/pkgs/servers/sql/postgresql/ext/timescaledb.nix index 60e14e922835b..7ddb887ec8eff 100644 --- a/pkgs/servers/sql/postgresql/ext/timescaledb.nix +++ b/pkgs/servers/sql/postgresql/ext/timescaledb.nix @@ -32,7 +32,7 @@ stdenv.mkDerivation rec { done ''; - passthru.tests = { inherit (nixosTests) timescaledb; }; + passthru.tests = nixosTests.timescaledb.passthru.override postgresql; meta = with lib; { description = "Scales PostgreSQL for time-series data via automatic partitioning across time and space"; diff --git a/pkgs/servers/sql/postgresql/ext/timescaledb_toolkit.nix b/pkgs/servers/sql/postgresql/ext/timescaledb_toolkit.nix index ab414a400d294..7665a4a155781 100644 --- a/pkgs/servers/sql/postgresql/ext/timescaledb_toolkit.nix +++ b/pkgs/servers/sql/postgresql/ext/timescaledb_toolkit.nix @@ -25,9 +25,7 @@ passthru = { updateScript = nix-update-script { }; - tests = { - timescaledb_toolkit = nixosTests.timescaledb; - }; + tests = nixosTests.timescaledb.passthru.override postgresql; }; # tests take really long From db2d6a00abe5ec01c9e016d1ac0bc449d06e51ad Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Sat, 2 Nov 2024 19:49:16 +0100 Subject: [PATCH 14/17] postgresqlPackages.anonymizer: make passthru.tests work with correct package Same reasoning as commit before. --- nixos/tests/pg_anonymizer.nix | 203 ++++++++++-------- .../servers/sql/postgresql/ext/anonymizer.nix | 6 +- 2 files changed, 114 insertions(+), 95 deletions(-) diff --git a/nixos/tests/pg_anonymizer.nix b/nixos/tests/pg_anonymizer.nix index b26e4dca05809..080b8deabbe88 100644 --- a/nixos/tests/pg_anonymizer.nix +++ b/nixos/tests/pg_anonymizer.nix @@ -1,94 +1,111 @@ -import ./make-test-python.nix ({ pkgs, lib, ... }: { - name = "pg_anonymizer"; - meta.maintainers = lib.teams.flyingcircus.members; - - nodes.machine = { pkgs, ... }: { - environment.systemPackages = [ pkgs.pg-dump-anon ]; - services.postgresql = { - enable = true; - extraPlugins = ps: [ ps.anonymizer ]; - settings.shared_preload_libraries = [ "anon" ]; +{ system ? builtins.currentSystem +, config ? {} +, pkgs ? import ../.. { inherit system config; } +}: + +with import ../lib/testing-python.nix { inherit system pkgs; }; + +let + inherit (pkgs) lib; + + makeAnonymizerTest = postgresqlPackage: + makeTest { + name = "pg_anonymizer-${postgresqlPackage.name}"; + meta.maintainers = lib.teams.flyingcircus.members; + + nodes.machine = { pkgs, ... }: { + environment.systemPackages = [ pkgs.pg-dump-anon ]; + services.postgresql = { + enable = true; + package = postgresqlPackage; + extraPlugins = ps: [ ps.anonymizer ]; + settings.shared_preload_libraries = [ "anon" ]; + }; + }; + + testScript = '' + start_all() + machine.wait_for_unit("multi-user.target") + machine.wait_for_unit("postgresql.service") + + with subtest("Setup"): + machine.succeed("sudo -u postgres psql --command 'create database demo'") + machine.succeed( + "sudo -u postgres psql -d demo -f ${pkgs.writeText "init.sql" '' + create extension anon cascade; + select anon.init(); + create table player(id serial, name text, points int); + insert into player(id,name,points) values (1,'Foo', 23); + insert into player(id,name,points) values (2,'Bar',42); + security label for anon on column player.name is 'MASKED WITH FUNCTION anon.fake_last_name();'; + security label for anon on column player.points is 'MASKED WITH VALUE NULL'; + ''}" + ) + + def get_player_table_contents(): + return [ + x.split(',') for x in machine.succeed("sudo -u postgres psql -d demo --csv --command 'select * from player'").splitlines()[1:] + ] + + def check_anonymized_row(row, id, original_name): + assert row[0] == id, f"Expected first row to have ID {id}, but got {row[0]}" + assert row[1] != original_name, f"Expected first row to have a name other than {original_name}" + assert not bool(row[2]), "Expected points to be NULL in first row" + + def find_xsv_in_dump(dump, sep=','): + """ + Expecting to find a CSV (for pg_dump_anon) or TSV (for pg_dump) structure, looking like + + COPY public.player ... + 1,Shields, + 2,Salazar, + \. + + in the given dump (the commas are tabs in case of pg_dump). + Extract the CSV lines and split by `sep`. + """ + + try: + from itertools import dropwhile, takewhile + return [x.split(sep) for x in list(takewhile( + lambda x: x != "\\.", + dropwhile( + lambda x: not x.startswith("COPY public.player"), + dump.splitlines() + ) + ))[1:]] + except: + print(f"Dump to process: {dump}") + raise + + def check_original_data(output): + assert output[0] == ['1','Foo','23'], f"Expected first row from player table to be 1,Foo,23; got {output[0]}" + assert output[1] == ['2','Bar','42'], f"Expected first row from player table to be 2,Bar,42; got {output[1]}" + + def check_anonymized_rows(output): + check_anonymized_row(output[0], '1', 'Foo') + check_anonymized_row(output[1], '2', 'Bar') + + with subtest("Check initial state"): + check_original_data(get_player_table_contents()) + + with subtest("Anonymous dumps"): + check_original_data(find_xsv_in_dump( + machine.succeed("sudo -u postgres pg_dump demo"), + sep='\t' + )) + check_anonymized_rows(find_xsv_in_dump( + machine.succeed("sudo -u postgres pg_dump_anon -U postgres -h /run/postgresql -d demo"), + sep=',' + )) + + with subtest("Anonymize"): + machine.succeed("sudo -u postgres psql -d demo --command 'select anon.anonymize_database();'") + check_anonymized_rows(get_player_table_contents()) + ''; }; - }; - - testScript = '' - start_all() - machine.wait_for_unit("multi-user.target") - machine.wait_for_unit("postgresql.service") - - with subtest("Setup"): - machine.succeed("sudo -u postgres psql --command 'create database demo'") - machine.succeed( - "sudo -u postgres psql -d demo -f ${pkgs.writeText "init.sql" '' - create extension anon cascade; - select anon.init(); - create table player(id serial, name text, points int); - insert into player(id,name,points) values (1,'Foo', 23); - insert into player(id,name,points) values (2,'Bar',42); - security label for anon on column player.name is 'MASKED WITH FUNCTION anon.fake_last_name();'; - security label for anon on column player.points is 'MASKED WITH VALUE NULL'; - ''}" - ) - - def get_player_table_contents(): - return [ - x.split(',') for x in machine.succeed("sudo -u postgres psql -d demo --csv --command 'select * from player'").splitlines()[1:] - ] - - def check_anonymized_row(row, id, original_name): - assert row[0] == id, f"Expected first row to have ID {id}, but got {row[0]}" - assert row[1] != original_name, f"Expected first row to have a name other than {original_name}" - assert not bool(row[2]), "Expected points to be NULL in first row" - - def find_xsv_in_dump(dump, sep=','): - """ - Expecting to find a CSV (for pg_dump_anon) or TSV (for pg_dump) structure, looking like - - COPY public.player ... - 1,Shields, - 2,Salazar, - \. - - in the given dump (the commas are tabs in case of pg_dump). - Extract the CSV lines and split by `sep`. - """ - - try: - from itertools import dropwhile, takewhile - return [x.split(sep) for x in list(takewhile( - lambda x: x != "\\.", - dropwhile( - lambda x: not x.startswith("COPY public.player"), - dump.splitlines() - ) - ))[1:]] - except: - print(f"Dump to process: {dump}") - raise - - def check_original_data(output): - assert output[0] == ['1','Foo','23'], f"Expected first row from player table to be 1,Foo,23; got {output[0]}" - assert output[1] == ['2','Bar','42'], f"Expected first row from player table to be 2,Bar,42; got {output[1]}" - - def check_anonymized_rows(output): - check_anonymized_row(output[0], '1', 'Foo') - check_anonymized_row(output[1], '2', 'Bar') - - with subtest("Check initial state"): - check_original_data(get_player_table_contents()) - - with subtest("Anonymous dumps"): - check_original_data(find_xsv_in_dump( - machine.succeed("sudo -u postgres pg_dump demo"), - sep='\t' - )) - check_anonymized_rows(find_xsv_in_dump( - machine.succeed("sudo -u postgres pg_dump_anon -U postgres -h /run/postgresql -d demo"), - sep=',' - )) - - with subtest("Anonymize"): - machine.succeed("sudo -u postgres psql -d demo --command 'select anon.anonymize_database();'") - check_anonymized_rows(get_player_table_contents()) - ''; -}) +in +pkgs.lib.concatMapAttrs (n: p: { ${n} = makeAnonymizerTest p; }) pkgs.postgresqlVersions +// { + passthru.override = p: makeAnonymizerTest p; +} diff --git a/pkgs/servers/sql/postgresql/ext/anonymizer.nix b/pkgs/servers/sql/postgresql/ext/anonymizer.nix index 4bb5aa544440e..559f7db3f675a 100644 --- a/pkgs/servers/sql/postgresql/ext/anonymizer.nix +++ b/pkgs/servers/sql/postgresql/ext/anonymizer.nix @@ -1,9 +1,9 @@ -{ lib, stdenv, pg-dump-anon, postgresql, runtimeShell, jitSupport, llvm }: +{ lib, stdenv, pg-dump-anon, postgresql, runtimeShell, jitSupport, llvm, nixosTests }: stdenv.mkDerivation (finalAttrs: { pname = "postgresql_anonymizer"; - inherit (pg-dump-anon) version src passthru; + inherit (pg-dump-anon) version src; buildInputs = [ postgresql ]; nativeBuildInputs = [ postgresql ] ++ lib.optional jitSupport llvm; @@ -26,6 +26,8 @@ stdenv.mkDerivation (finalAttrs: { EOF ''; + passthru.tests = nixosTests.pg_anonymizer.passthru.override postgresql; + meta = lib.getAttrs [ "homepage" "maintainers" "license" ] pg-dump-anon.meta // { description = "Extension to mask or replace personally identifiable information (PII) or commercially sensitive data from a PostgreSQL database"; }; From 9035573855d9d1bd3c0e7512d361b9d95e47c033 Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Sat, 2 Nov 2024 18:58:52 +0100 Subject: [PATCH 15/17] nixosTests.postgresql: move all postgresql related nixosTests into one folder This makes it possible to run all those tests at once by building nixosTests.postgresql and allow a simple entry to ci/OWNERS for all tests. --- ci/OWNERS | 2 +- nixos/tests/all-tests.nix | 11 +--- nixos/tests/postgresql-wal2json.nix | 52 ------------------- .../anonymizer.nix} | 19 ++++--- nixos/tests/postgresql/default.nix | 26 ++++++++++ nixos/tests/{ => postgresql}/pgjwt.nix | 17 +++--- nixos/tests/{ => postgresql}/pgvecto-rs.nix | 16 +++--- .../tests/{ => postgresql}/postgresql-jit.nix | 17 +++--- .../postgresql-tls-client-cert.nix | 17 +++--- .../postgresql-wal-receiver.nix | 17 +++--- nixos/tests/{ => postgresql}/postgresql.nix | 16 +++--- nixos/tests/{ => postgresql}/timescaledb.nix | 19 ++++--- nixos/tests/{ => postgresql}/tsja.nix | 17 +++--- nixos/tests/postgresql/wal2json.nix | 50 ++++++++++++++++++ pkgs/by-name/pg/pg-dump-anon/package.nix | 2 +- .../servers/sql/postgresql/ext/anonymizer.nix | 2 +- pkgs/servers/sql/postgresql/ext/pgjwt.nix | 2 +- .../sql/postgresql/ext/pgvecto-rs/default.nix | 2 +- .../sql/postgresql/ext/timescaledb.nix | 2 +- .../postgresql/ext/timescaledb_toolkit.nix | 2 +- pkgs/servers/sql/postgresql/ext/tsja.nix | 2 +- pkgs/servers/sql/postgresql/ext/wal2json.nix | 2 +- pkgs/servers/sql/postgresql/generic.nix | 8 +-- 23 files changed, 166 insertions(+), 154 deletions(-) delete mode 100644 nixos/tests/postgresql-wal2json.nix rename nixos/tests/{pg_anonymizer.nix => postgresql/anonymizer.nix} (91%) create mode 100644 nixos/tests/postgresql/default.nix rename nixos/tests/{ => postgresql}/pgjwt.nix (76%) rename nixos/tests/{ => postgresql}/pgvecto-rs.nix (85%) rename nixos/tests/{ => postgresql}/postgresql-jit.nix (78%) rename nixos/tests/{ => postgresql}/postgresql-tls-client-cert.nix (92%) rename nixos/tests/{ => postgresql}/postgresql-wal-receiver.nix (92%) rename nixos/tests/{ => postgresql}/postgresql.nix (96%) rename nixos/tests/{ => postgresql}/timescaledb.nix (83%) rename nixos/tests/{ => postgresql}/tsja.nix (77%) create mode 100644 nixos/tests/postgresql/wal2json.nix diff --git a/ci/OWNERS b/ci/OWNERS index 8e227b1525668..b8aa53105fc94 100644 --- a/ci/OWNERS +++ b/ci/OWNERS @@ -229,7 +229,7 @@ pkgs/development/python-modules/buildcatrust/ @ajs124 @lukegb @mweinelt /pkgs/servers/sql/postgresql @NixOS/postgres /nixos/modules/services/databases/postgresql.md @NixOS/postgres /nixos/modules/services/databases/postgresql.nix @NixOS/postgres -/nixos/tests/postgresql.nix @NixOS/postgres +/nixos/tests/postgresql @NixOS/postgres # Hardened profile & related modules /nixos/modules/profiles/hardened.nix @joachifm diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index bb674910e700d..37e005f128a2e 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -775,13 +775,10 @@ in { peering-manager = handleTest ./web-apps/peering-manager.nix {}; peertube = handleTestOn ["x86_64-linux"] ./web-apps/peertube.nix {}; peroxide = handleTest ./peroxide.nix {}; - pg_anonymizer = handleTest ./pg_anonymizer.nix {}; pgadmin4 = handleTest ./pgadmin4.nix {}; pgbouncer = handleTest ./pgbouncer.nix {}; pghero = runTest ./pghero.nix; - pgjwt = handleTest ./pgjwt.nix {}; pgmanage = handleTest ./pgmanage.nix {}; - pgvecto-rs = handleTest ./pgvecto-rs.nix {}; phosh = handleTest ./phosh.nix {}; photonvision = handleTest ./photonvision.nix {}; photoprism = handleTest ./photoprism.nix {}; @@ -814,11 +811,7 @@ in { postfix = handleTest ./postfix.nix {}; postfix-raise-smtpd-tls-security-level = handleTest ./postfix-raise-smtpd-tls-security-level.nix {}; postfixadmin = handleTest ./postfixadmin.nix {}; - postgresql = handleTest ./postgresql.nix {}; - postgresql-jit = handleTest ./postgresql-jit.nix {}; - postgresql-wal-receiver = handleTest ./postgresql-wal-receiver.nix {}; - postgresql-tls-client-cert = handleTest ./postgresql-tls-client-cert.nix {}; - postgresql-wal2json = handleTest ./postgresql-wal2json.nix {}; + postgresql = handleTest ./postgresql {}; powerdns = handleTest ./powerdns.nix {}; powerdns-admin = handleTest ./powerdns-admin.nix {}; power-profiles-daemon = handleTest ./power-profiles-daemon.nix {}; @@ -1045,7 +1038,6 @@ in { tiddlywiki = handleTest ./tiddlywiki.nix {}; tigervnc = handleTest ./tigervnc.nix {}; tika = runTest ./tika.nix; - timescaledb = handleTest ./timescaledb.nix {}; timezone = handleTest ./timezone.nix {}; timidity = handleTestOn ["aarch64-linux" "x86_64-linux"] ./timidity {}; tinc = handleTest ./tinc {}; @@ -1065,7 +1057,6 @@ in { trezord = handleTest ./trezord.nix {}; trickster = handleTest ./trickster.nix {}; trilium-server = handleTestOn ["x86_64-linux"] ./trilium-server.nix {}; - tsja = handleTest ./tsja.nix {}; tsm-client-gui = handleTest ./tsm-client-gui.nix {}; ttyd = handleTest ./web-servers/ttyd.nix {}; txredisapi = handleTest ./txredisapi.nix {}; diff --git a/nixos/tests/postgresql-wal2json.nix b/nixos/tests/postgresql-wal2json.nix deleted file mode 100644 index 2ad1a1ee66a0a..0000000000000 --- a/nixos/tests/postgresql-wal2json.nix +++ /dev/null @@ -1,52 +0,0 @@ -{ - system ? builtins.currentSystem, - config ? { }, - pkgs ? import ../.. { inherit system config; }, -}: - -with import ../lib/testing-python.nix { inherit system pkgs; }; - -let - inherit (pkgs) lib; - - makePostgresqlWal2jsonTest = - postgresqlPackage: - makeTest { - name = "postgresql-wal2json-${postgresqlPackage.name}"; - meta.maintainers = with pkgs.lib.maintainers; [ euank ]; - - nodes.machine = { - services.postgresql = { - package = postgresqlPackage; - enable = true; - extraPlugins = with postgresqlPackage.pkgs; [ wal2json ]; - settings = { - wal_level = "logical"; - max_replication_slots = "10"; - max_wal_senders = "10"; - }; - }; - }; - - testScript = '' - machine.wait_for_unit("postgresql") - machine.succeed( - "sudo -u postgres psql -qAt -f ${./postgresql/wal2json/example2.sql} postgres > /tmp/example2.out" - ) - machine.succeed( - "diff ${./postgresql/wal2json/example2.out} /tmp/example2.out" - ) - machine.succeed( - "sudo -u postgres psql -qAt -f ${./postgresql/wal2json/example3.sql} postgres > /tmp/example3.out" - ) - machine.succeed( - "diff ${./postgresql/wal2json/example3.out} /tmp/example3.out" - ) - ''; - }; - -in -lib.concatMapAttrs (n: p: { ${n} = makePostgresqlWal2jsonTest p; }) pkgs.postgresqlVersions -// { - passthru.override = p: makePostgresqlWal2jsonTest p; -} diff --git a/nixos/tests/pg_anonymizer.nix b/nixos/tests/postgresql/anonymizer.nix similarity index 91% rename from nixos/tests/pg_anonymizer.nix rename to nixos/tests/postgresql/anonymizer.nix index 080b8deabbe88..7b9d84ca8aff5 100644 --- a/nixos/tests/pg_anonymizer.nix +++ b/nixos/tests/postgresql/anonymizer.nix @@ -1,16 +1,13 @@ -{ system ? builtins.currentSystem -, config ? {} -, pkgs ? import ../.. { inherit system config; } +{ pkgs +, makeTest }: -with import ../lib/testing-python.nix { inherit system pkgs; }; - let inherit (pkgs) lib; makeAnonymizerTest = postgresqlPackage: makeTest { - name = "pg_anonymizer-${postgresqlPackage.name}"; + name = "postgresql_anonymizer-${postgresqlPackage.name}"; meta.maintainers = lib.teams.flyingcircus.members; nodes.machine = { pkgs, ... }: { @@ -105,7 +102,9 @@ let ''; }; in -pkgs.lib.concatMapAttrs (n: p: { ${n} = makeAnonymizerTest p; }) pkgs.postgresqlVersions -// { - passthru.override = p: makeAnonymizerTest p; -} +pkgs.lib.recurseIntoAttrs ( + pkgs.lib.concatMapAttrs (n: p: { ${n} = makeAnonymizerTest p; }) pkgs.postgresqlVersions + // { + passthru.override = p: makeAnonymizerTest p; + } +) diff --git a/nixos/tests/postgresql/default.nix b/nixos/tests/postgresql/default.nix new file mode 100644 index 0000000000000..4fe7e7a37e7ef --- /dev/null +++ b/nixos/tests/postgresql/default.nix @@ -0,0 +1,26 @@ +{ + system ? builtins.currentSystem, + config ? { }, + pkgs ? import ../../.. { inherit system config; }, +}: + +with import ../../lib/testing-python.nix { inherit system pkgs; }; + +let + importWithArgs = path: import path { inherit pkgs makeTest; }; +in +{ + # postgresql + postgresql = importWithArgs ./postgresql.nix; + postgresql-jit = importWithArgs ./postgresql-jit.nix; + postgresql-wal-receiver = importWithArgs ./postgresql-wal-receiver.nix; + postgresql-tls-client-cert = importWithArgs ./postgresql-tls-client-cert.nix; + + # extensions + anonymizer = importWithArgs ./anonymizer.nix; + pgjwt = importWithArgs ./pgjwt.nix; + pgvecto-rs = importWithArgs ./pgvecto-rs.nix; + timescaledb = importWithArgs ./timescaledb.nix; + tsja = importWithArgs ./tsja.nix; + wal2json = importWithArgs ./wal2json.nix; +} diff --git a/nixos/tests/pgjwt.nix b/nixos/tests/postgresql/pgjwt.nix similarity index 76% rename from nixos/tests/pgjwt.nix rename to nixos/tests/postgresql/pgjwt.nix index 5c14ee68b5954..e8beb6cbee532 100644 --- a/nixos/tests/pgjwt.nix +++ b/nixos/tests/postgresql/pgjwt.nix @@ -1,10 +1,7 @@ -{ system ? builtins.currentSystem -, config ? {} -, pkgs ? import ../.. { inherit system config; } +{ pkgs +, makeTest }: -with import ../lib/testing-python.nix { inherit system pkgs; }; - let inherit (pkgs) lib; @@ -41,7 +38,9 @@ let ''; }; in -lib.concatMapAttrs (n: p: { ${n} = makePgjwtTest p; }) pkgs.postgresqlVersions -// { - passthru.override = p: makePgjwtTest p; -} +lib.recurseIntoAttrs ( + lib.concatMapAttrs (n: p: { ${n} = makePgjwtTest p; }) pkgs.postgresqlVersions + // { + passthru.override = p: makePgjwtTest p; + } +) diff --git a/nixos/tests/pgvecto-rs.nix b/nixos/tests/postgresql/pgvecto-rs.nix similarity index 85% rename from nixos/tests/pgvecto-rs.nix rename to nixos/tests/postgresql/pgvecto-rs.nix index 4a2b75132ed5f..4d14dfdb6bb1f 100644 --- a/nixos/tests/pgvecto-rs.nix +++ b/nixos/tests/postgresql/pgvecto-rs.nix @@ -1,9 +1,7 @@ -{ system ? builtins.currentSystem -, config ? { } -, pkgs ? import ../.. { inherit system config; } +{ pkgs +, makeTest }: -with import ../lib/testing-python.nix { inherit system pkgs; }; with pkgs.lib; let @@ -68,7 +66,9 @@ let }; in -concatMapAttrs (n: p: { ${n} = makePgVectorsTest p; }) (filterAttrs (n: p: !p.pkgs.pgvecto-rs.meta.broken) pkgs.postgresqlVersions) -// { - passthru.override = p: makePgVectorsTest p; -} +recurseIntoAttrs ( + concatMapAttrs (n: p: { ${n} = makePgVectorsTest p; }) (filterAttrs (n: p: !p.pkgs.pgvecto-rs.meta.broken) pkgs.postgresqlVersions) + // { + passthru.override = p: makePgVectorsTest p; + } +) diff --git a/nixos/tests/postgresql-jit.nix b/nixos/tests/postgresql/postgresql-jit.nix similarity index 78% rename from nixos/tests/postgresql-jit.nix rename to nixos/tests/postgresql/postgresql-jit.nix index fb598972ba0fb..40329d038fefe 100644 --- a/nixos/tests/postgresql-jit.nix +++ b/nixos/tests/postgresql/postgresql-jit.nix @@ -1,10 +1,7 @@ -{ system ? builtins.currentSystem -, config ? {} -, pkgs ? import ../.. { inherit system config; } +{ pkgs +, makeTest }: -with import ../lib/testing-python.nix { inherit system pkgs; }; - let inherit (pkgs) lib; @@ -44,7 +41,9 @@ let ''; }; in -lib.concatMapAttrs (n: p: { ${n} = mkJitTest p; }) (lib.filterAttrs (n: _: lib.hasSuffix "_jit" n) pkgs.postgresqlVersions) -// { - passthru.override = p: mkJitTest p; -} +lib.recurseIntoAttrs ( + lib.concatMapAttrs (n: p: { ${n} = mkJitTest p; }) (lib.filterAttrs (n: _: lib.hasSuffix "_jit" n) pkgs.postgresqlVersions) + // { + passthru.override = p: mkJitTest p; + } +) diff --git a/nixos/tests/postgresql-tls-client-cert.nix b/nixos/tests/postgresql/postgresql-tls-client-cert.nix similarity index 92% rename from nixos/tests/postgresql-tls-client-cert.nix rename to nixos/tests/postgresql/postgresql-tls-client-cert.nix index 86dada85f272b..503bc77268c20 100644 --- a/nixos/tests/postgresql-tls-client-cert.nix +++ b/nixos/tests/postgresql/postgresql-tls-client-cert.nix @@ -1,10 +1,7 @@ -{ system ? builtins.currentSystem -, config ? { } -, pkgs ? import ../.. { inherit system config; } +{ pkgs +, makeTest }: -with import ../lib/testing-python.nix { inherit system pkgs; }; - let lib = pkgs.lib; @@ -125,7 +122,9 @@ let }; in -lib.concatMapAttrs (n: p: { ${n} = makePostgresqlTlsClientCertTest p; }) pkgs.postgresqlVersions -// { - passthru.override = p: makePostgresqlTlsClientCertTest p; -} +lib.recurseIntoAttrs ( + lib.concatMapAttrs (n: p: { ${n} = makePostgresqlTlsClientCertTest p; }) pkgs.postgresqlVersions + // { + passthru.override = p: makePostgresqlTlsClientCertTest p; + } +) diff --git a/nixos/tests/postgresql-wal-receiver.nix b/nixos/tests/postgresql/postgresql-wal-receiver.nix similarity index 92% rename from nixos/tests/postgresql-wal-receiver.nix rename to nixos/tests/postgresql/postgresql-wal-receiver.nix index 9b70668e7305d..d46b3fcbdcf30 100644 --- a/nixos/tests/postgresql-wal-receiver.nix +++ b/nixos/tests/postgresql/postgresql-wal-receiver.nix @@ -1,10 +1,7 @@ -{ system ? builtins.currentSystem, - config ? {}, - pkgs ? import ../.. { inherit system config; }, +{ pkgs +, makeTest }: -with import ../lib/testing-python.nix { inherit system pkgs; }; - let lib = pkgs.lib; @@ -107,7 +104,9 @@ let ''; }; in -lib.concatMapAttrs (n: p: { ${n} = makePostgresqlWalReceiverTest p; }) pkgs.postgresqlVersions -// { - passthru.override = p: makePostgresqlWalReceiverTest p; -} +lib.recurseIntoAttrs ( + lib.concatMapAttrs (n: p: { ${n} = makePostgresqlWalReceiverTest p; }) pkgs.postgresqlVersions + // { + passthru.override = p: makePostgresqlWalReceiverTest p; + } +) diff --git a/nixos/tests/postgresql.nix b/nixos/tests/postgresql/postgresql.nix similarity index 96% rename from nixos/tests/postgresql.nix rename to nixos/tests/postgresql/postgresql.nix index 4f6c37643c257..56ec43d0a3b69 100644 --- a/nixos/tests/postgresql.nix +++ b/nixos/tests/postgresql/postgresql.nix @@ -1,9 +1,7 @@ -{ system ? builtins.currentSystem, - config ? {}, - pkgs ? import ../.. { inherit system config; } +{ pkgs +, makeTest }: -with import ../lib/testing-python.nix { inherit system pkgs; }; with pkgs.lib; let @@ -226,7 +224,9 @@ let ''; }; in -concatMapAttrs (n: p: { ${n} = testsForPackage p; }) pkgs.postgresqlVersions -// { - passthru.override = p: testsForPackage p; -} +recurseIntoAttrs ( + concatMapAttrs (n: p: { ${n} = testsForPackage p; }) pkgs.postgresqlVersions + // { + passthru.override = p: testsForPackage p; + } +) diff --git a/nixos/tests/timescaledb.nix b/nixos/tests/postgresql/timescaledb.nix similarity index 83% rename from nixos/tests/timescaledb.nix rename to nixos/tests/postgresql/timescaledb.nix index 6c6ba8ab6c6dd..7f37e79bf0946 100644 --- a/nixos/tests/timescaledb.nix +++ b/nixos/tests/postgresql/timescaledb.nix @@ -1,9 +1,7 @@ -{ system ? builtins.currentSystem -, config ? { } -, pkgs ? import ../.. { inherit system config; } +{ pkgs +, makeTest }: -with import ../lib/testing-python.nix { inherit system pkgs; }; with pkgs.lib; let @@ -83,7 +81,12 @@ let }; in -pkgs.lib.concatMapAttrs (n: p: { ${n} = makeTimescaleDbTest p; }) (filterAttrs (n: p: !p.pkgs.timescaledb.meta.broken) pkgs.postgresqlVersions) -// { - passthru.override = p: makeTimescaleDbTest p; -} +# Not run by default, because this requires allowUnfree. +# To run these tests: +# NIXPKGS_ALLOW_UNFREE=1 nix-build -A nixosTests.postgresql.timescaledb +dontRecurseIntoAttrs ( + pkgs.lib.concatMapAttrs (n: p: { ${n} = makeTimescaleDbTest p; }) (filterAttrs (n: p: !p.pkgs.timescaledb.meta.broken) pkgs.postgresqlVersions) + // { + passthru.override = p: makeTimescaleDbTest p; + } +) diff --git a/nixos/tests/tsja.nix b/nixos/tests/postgresql/tsja.nix similarity index 77% rename from nixos/tests/tsja.nix rename to nixos/tests/postgresql/tsja.nix index 493b6d56165e7..dedda35540ace 100644 --- a/nixos/tests/tsja.nix +++ b/nixos/tests/postgresql/tsja.nix @@ -1,10 +1,7 @@ -{ system ? builtins.currentSystem -, config ? {} -, pkgs ? import ../.. { inherit system config; } +{ pkgs +, makeTest }: -with import ../lib/testing-python.nix { inherit system pkgs; }; - let makeTsjaTest = postgresqlPackage: makeTest { @@ -41,7 +38,9 @@ let ''; }; in -pkgs.lib.concatMapAttrs (n: p: { ${n} = makeTsjaTest p; }) pkgs.postgresqlVersions -// { - passthru.override = p: makeTsjaTest p; -} +pkgs.lib.recurseIntoAttrs ( + pkgs.lib.concatMapAttrs (n: p: { ${n} = makeTsjaTest p; }) pkgs.postgresqlVersions + // { + passthru.override = p: makeTsjaTest p; + } +) diff --git a/nixos/tests/postgresql/wal2json.nix b/nixos/tests/postgresql/wal2json.nix new file mode 100644 index 0000000000000..782fc5c94f66b --- /dev/null +++ b/nixos/tests/postgresql/wal2json.nix @@ -0,0 +1,50 @@ +{ pkgs +, makeTest +}: + +let + inherit (pkgs) lib; + + makePostgresqlWal2jsonTest = + postgresqlPackage: + makeTest { + name = "wal2json-${postgresqlPackage.name}"; + meta.maintainers = with pkgs.lib.maintainers; [ euank ]; + + nodes.machine = { + services.postgresql = { + package = postgresqlPackage; + enable = true; + extraPlugins = with postgresqlPackage.pkgs; [ wal2json ]; + settings = { + wal_level = "logical"; + max_replication_slots = "10"; + max_wal_senders = "10"; + }; + }; + }; + + testScript = '' + machine.wait_for_unit("postgresql") + machine.succeed( + "sudo -u postgres psql -qAt -f ${./wal2json/example2.sql} postgres > /tmp/example2.out" + ) + machine.succeed( + "diff ${./wal2json/example2.out} /tmp/example2.out" + ) + machine.succeed( + "sudo -u postgres psql -qAt -f ${./wal2json/example3.sql} postgres > /tmp/example3.out" + ) + machine.succeed( + "diff ${./wal2json/example3.out} /tmp/example3.out" + ) + ''; + }; + +in +lib.recurseIntoAttrs ( + lib.concatMapAttrs (n: p: { ${n} = makePostgresqlWal2jsonTest p; }) pkgs.postgresqlVersions + // { + passthru.override = p: makePostgresqlWal2jsonTest p; + } +) diff --git a/pkgs/by-name/pg/pg-dump-anon/package.nix b/pkgs/by-name/pg/pg-dump-anon/package.nix index 96b6d351130f6..a097eb8aa9c71 100644 --- a/pkgs/by-name/pg/pg-dump-anon/package.nix +++ b/pkgs/by-name/pg/pg-dump-anon/package.nix @@ -14,7 +14,7 @@ buildGoModule rec { vendorHash = "sha256-CwU1zoIayxvfnGL9kPdummPJiV+ECfSz4+q6gZGb8pw="; - passthru.tests = { inherit (nixosTests) pg_anonymizer; }; + passthru.tests = { inherit (nixosTests.postgresql) anonymizer; }; nativeBuildInputs = [ makeWrapper ]; postInstall = '' diff --git a/pkgs/servers/sql/postgresql/ext/anonymizer.nix b/pkgs/servers/sql/postgresql/ext/anonymizer.nix index 559f7db3f675a..afb517c6b5a04 100644 --- a/pkgs/servers/sql/postgresql/ext/anonymizer.nix +++ b/pkgs/servers/sql/postgresql/ext/anonymizer.nix @@ -26,7 +26,7 @@ stdenv.mkDerivation (finalAttrs: { EOF ''; - passthru.tests = nixosTests.pg_anonymizer.passthru.override postgresql; + passthru.tests = nixosTests.postgresql.anonymizer.passthru.override postgresql; meta = lib.getAttrs [ "homepage" "maintainers" "license" ] pg-dump-anon.meta // { description = "Extension to mask or replace personally identifiable information (PII) or commercially sensitive data from a PostgreSQL database"; diff --git a/pkgs/servers/sql/postgresql/ext/pgjwt.nix b/pkgs/servers/sql/postgresql/ext/pgjwt.nix index 0209ab27d59e0..7bac053d920cd 100644 --- a/pkgs/servers/sql/postgresql/ext/pgjwt.nix +++ b/pkgs/servers/sql/postgresql/ext/pgjwt.nix @@ -20,7 +20,7 @@ stdenv.mkDerivation (finalAttrs: { passthru.updateScript = unstableGitUpdater { }; passthru.tests = lib.recurseIntoAttrs { - pgjwt = nixosTests.pgjwt.passthru.override postgresql; + pgjwt = nixosTests.postgresql.pgjwt.passthru.override postgresql; extension = postgresqlTestExtension { inherit (finalAttrs) finalPackage; diff --git a/pkgs/servers/sql/postgresql/ext/pgvecto-rs/default.nix b/pkgs/servers/sql/postgresql/ext/pgvecto-rs/default.nix index 40cde8eeae0ba..f25afda0ca612 100644 --- a/pkgs/servers/sql/postgresql/ext/pgvecto-rs/default.nix +++ b/pkgs/servers/sql/postgresql/ext/pgvecto-rs/default.nix @@ -86,7 +86,7 @@ in passthru = { updateScript = nix-update-script { }; - tests = nixosTests.pgvecto-rs.passthru.override postgresql; + tests = nixosTests.postgresql.pgvecto-rs.passthru.override postgresql; }; meta = with lib; { diff --git a/pkgs/servers/sql/postgresql/ext/timescaledb.nix b/pkgs/servers/sql/postgresql/ext/timescaledb.nix index 7ddb887ec8eff..ab4736c5f7e9e 100644 --- a/pkgs/servers/sql/postgresql/ext/timescaledb.nix +++ b/pkgs/servers/sql/postgresql/ext/timescaledb.nix @@ -32,7 +32,7 @@ stdenv.mkDerivation rec { done ''; - passthru.tests = nixosTests.timescaledb.passthru.override postgresql; + passthru.tests = nixosTests.postgresql.timescaledb.passthru.override postgresql; meta = with lib; { description = "Scales PostgreSQL for time-series data via automatic partitioning across time and space"; diff --git a/pkgs/servers/sql/postgresql/ext/timescaledb_toolkit.nix b/pkgs/servers/sql/postgresql/ext/timescaledb_toolkit.nix index 7665a4a155781..eec287ddcc10d 100644 --- a/pkgs/servers/sql/postgresql/ext/timescaledb_toolkit.nix +++ b/pkgs/servers/sql/postgresql/ext/timescaledb_toolkit.nix @@ -25,7 +25,7 @@ passthru = { updateScript = nix-update-script { }; - tests = nixosTests.timescaledb.passthru.override postgresql; + tests = nixosTests.postgresql.timescaledb.passthru.override postgresql; }; # tests take really long diff --git a/pkgs/servers/sql/postgresql/ext/tsja.nix b/pkgs/servers/sql/postgresql/ext/tsja.nix index 38df785e7f7c3..de19b047fe9c3 100644 --- a/pkgs/servers/sql/postgresql/ext/tsja.nix +++ b/pkgs/servers/sql/postgresql/ext/tsja.nix @@ -32,7 +32,7 @@ stdenv.mkDerivation rec { mv dbinit_libtsja.txt $out/share/postgresql/extension/libtsja_dbinit.sql ''; - passthru.tests = nixosTests.tsja.passthru.override postgresql; + passthru.tests = nixosTests.postgresql.tsja.passthru.override postgresql; meta = with lib; { description = "PostgreSQL extension implementing Japanese text search"; diff --git a/pkgs/servers/sql/postgresql/ext/wal2json.nix b/pkgs/servers/sql/postgresql/ext/wal2json.nix index 25f56db522227..f7d581f2c8dd4 100644 --- a/pkgs/servers/sql/postgresql/ext/wal2json.nix +++ b/pkgs/servers/sql/postgresql/ext/wal2json.nix @@ -27,7 +27,7 @@ stdenv.mkDerivation rec { install -D -t $out/share/postgresql/extension sql/*.sql ''; - passthru.tests = nixosTests.postgresql-wal2json.passthru.override postgresql; + passthru.tests = nixosTests.postgresql.wal2json.passthru.override postgresql; meta = with lib; { description = "PostgreSQL JSON output plugin for changeset extraction"; diff --git a/pkgs/servers/sql/postgresql/generic.nix b/pkgs/servers/sql/postgresql/generic.nix index 6c8530ce8b54e..7d59f6349d57a 100644 --- a/pkgs/servers/sql/postgresql/generic.nix +++ b/pkgs/servers/sql/postgresql/generic.nix @@ -312,12 +312,12 @@ let }; tests = { - postgresql = nixosTests.postgresql.passthru.override finalAttrs.finalPackage; - postgresql-tls-client-cert = nixosTests.postgresql-tls-client-cert.passthru.override finalAttrs.finalPackage; - postgresql-wal-receiver = nixosTests.postgresql-wal-receiver.passthru.override finalAttrs.finalPackage; + postgresql = nixosTests.postgresql.postgresql.passthru.override finalAttrs.finalPackage; + postgresql-tls-client-cert = nixosTests.postgresql.postgresql-tls-client-cert.passthru.override finalAttrs.finalPackage; + postgresql-wal-receiver = nixosTests.postgresql.postgresql-wal-receiver.passthru.override finalAttrs.finalPackage; pkg-config = testers.testMetaPkgConfig finalAttrs.finalPackage; } // lib.optionalAttrs jitSupport { - postgresql-jit = nixosTests.postgresql-jit.passthru.override finalAttrs.finalPackage; + postgresql-jit = nixosTests.postgresql.postgresql-jit.passthru.override finalAttrs.finalPackage; }; }; From 128244b5981822d457f245986b284fa563ab4934 Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Sat, 2 Nov 2024 21:24:48 +0100 Subject: [PATCH 16/17] nixosTests.postgresql: use a common pattern throughout all tests Avoiding "with", using the same names and basic structure in each test. Consistency is key! --- nixos/tests/postgresql/anonymizer.nix | 12 +- nixos/tests/postgresql/pgjwt.nix | 14 +- nixos/tests/postgresql/pgvecto-rs.nix | 78 ++-- nixos/tests/postgresql/postgresql-jit.nix | 69 +-- .../postgresql/postgresql-tls-client-cert.nix | 75 ++-- .../postgresql/postgresql-wal-receiver.nix | 23 +- nixos/tests/postgresql/postgresql.nix | 395 +++++++++--------- nixos/tests/postgresql/timescaledb.nix | 78 ++-- nixos/tests/postgresql/tsja.nix | 34 +- nixos/tests/postgresql/wal2json.nix | 14 +- 10 files changed, 393 insertions(+), 399 deletions(-) diff --git a/nixos/tests/postgresql/anonymizer.nix b/nixos/tests/postgresql/anonymizer.nix index 7b9d84ca8aff5..ec971aa710253 100644 --- a/nixos/tests/postgresql/anonymizer.nix +++ b/nixos/tests/postgresql/anonymizer.nix @@ -5,16 +5,16 @@ let inherit (pkgs) lib; - makeAnonymizerTest = postgresqlPackage: + makeTestFor = package: makeTest { - name = "postgresql_anonymizer-${postgresqlPackage.name}"; + name = "postgresql_anonymizer-${package.name}"; meta.maintainers = lib.teams.flyingcircus.members; nodes.machine = { pkgs, ... }: { environment.systemPackages = [ pkgs.pg-dump-anon ]; services.postgresql = { + inherit package; enable = true; - package = postgresqlPackage; extraPlugins = ps: [ ps.anonymizer ]; settings.shared_preload_libraries = [ "anon" ]; }; @@ -102,9 +102,9 @@ let ''; }; in -pkgs.lib.recurseIntoAttrs ( - pkgs.lib.concatMapAttrs (n: p: { ${n} = makeAnonymizerTest p; }) pkgs.postgresqlVersions +lib.recurseIntoAttrs ( + lib.concatMapAttrs (n: p: { ${n} = makeTestFor p; }) (lib.filterAttrs (_: p: !p.pkgs.anonymizer.meta.broken) pkgs.postgresqlVersions) // { - passthru.override = p: makeAnonymizerTest p; + passthru.override = p: makeTestFor p; } ) diff --git a/nixos/tests/postgresql/pgjwt.nix b/nixos/tests/postgresql/pgjwt.nix index e8beb6cbee532..2c0b77f180ff8 100644 --- a/nixos/tests/postgresql/pgjwt.nix +++ b/nixos/tests/postgresql/pgjwt.nix @@ -5,23 +5,21 @@ let inherit (pkgs) lib; - makePgjwtTest = postgresqlPackage: + makeTestFor = package: makeTest { - name = "pgjwt-${postgresqlPackage.name}"; + name = "pgjwt-${package.name}"; meta = with lib.maintainers; { maintainers = [ spinus willibutz ]; }; - nodes = { - master = { ... }: + nodes.master = { ... }: { services.postgresql = { + inherit package; enable = true; - package = postgresqlPackage; extraPlugins = ps: with ps; [ pgjwt pgtap ]; }; }; - }; testScript = { nodes, ... }: let @@ -39,8 +37,8 @@ let }; in lib.recurseIntoAttrs ( - lib.concatMapAttrs (n: p: { ${n} = makePgjwtTest p; }) pkgs.postgresqlVersions + lib.concatMapAttrs (n: p: { ${n} = makeTestFor p; }) (lib.filterAttrs (_: p: !p.pkgs.pgjwt.meta.broken) pkgs.postgresqlVersions) // { - passthru.override = p: makePgjwtTest p; + passthru.override = p: makeTestFor p; } ) diff --git a/nixos/tests/postgresql/pgvecto-rs.nix b/nixos/tests/postgresql/pgvecto-rs.nix index 4d14dfdb6bb1f..8f8c3fdd84368 100644 --- a/nixos/tests/postgresql/pgvecto-rs.nix +++ b/nixos/tests/postgresql/pgvecto-rs.nix @@ -2,9 +2,9 @@ , makeTest }: -with pkgs.lib; - let + inherit (pkgs) lib; + # Test cases from https://docs.pgvecto.rs/use-cases/hybrid-search.html test-sql = pkgs.writeText "postgresql-test" '' CREATE EXTENSION vectors; @@ -22,53 +22,53 @@ let ('a thin dog sat on a mat and ate a thin rat', '[10, 11, 12]'); ''; - makePgVectorsTest = postgresqlPackage: makeTest { - name = "pgvecto-rs-${postgresqlPackage.name}"; - meta = with pkgs.lib.maintainers; { - maintainers = [ diogotcorreia ]; - }; - - nodes.machine = { ... }: - { - services.postgresql = { - enable = true; - package = postgresqlPackage; - extraPlugins = ps: with ps; [ - pgvecto-rs - ]; - settings.shared_preload_libraries = "vectors"; - }; + makeTestFor = postgresqlPackage: + makeTest { + name = "pgvecto-rs-${postgresqlPackage.name}"; + meta = with lib.maintainers; { + maintainers = [ diogotcorreia ]; }; - testScript = { nodes, ... }: - let - inherit (nodes.machine.services.postgresql.package.pkgs) pgvecto-rs; - in - '' - def check_count(statement, lines): - return 'test $(sudo -u postgres psql postgres -tAc "{}"|wc -l) -eq {}'.format( - statement, lines - ) + nodes.machine = { ... }: + { + services.postgresql = { + enable = true; + package = postgresqlPackage; + extraPlugins = ps: with ps; [ + pgvecto-rs + ]; + settings.shared_preload_libraries = "vectors"; + }; + }; + testScript = { nodes, ... }: + let + inherit (nodes.machine.services.postgresql.package.pkgs) pgvecto-rs; + in + '' + def check_count(statement, lines): + return 'test $(sudo -u postgres psql postgres -tAc "{}"|wc -l) -eq {}'.format( + statement, lines + ) - machine.start() - machine.wait_for_unit("postgresql") - with subtest("Postgresql with extension vectors is available just after unit start"): - machine.succeed(check_count("SELECT * FROM pg_available_extensions WHERE name = 'vectors' AND default_version = '${pgvecto-rs.version}';", 1)) + machine.start() + machine.wait_for_unit("postgresql") - machine.succeed("sudo -u postgres psql -f ${test-sql}") + with subtest("Postgresql with extension vectors is available just after unit start"): + machine.succeed(check_count("SELECT * FROM pg_available_extensions WHERE name = 'vectors' AND default_version = '${pgvecto-rs.version}';", 1)) - machine.succeed(check_count("SELECT content, embedding FROM items WHERE to_tsvector('english', content) @@ 'cat & rat'::tsquery;", 2)) + machine.succeed("sudo -u postgres psql -f ${test-sql}") - machine.shutdown() - ''; + machine.succeed(check_count("SELECT content, embedding FROM items WHERE to_tsvector('english', content) @@ 'cat & rat'::tsquery;", 2)) - }; + machine.shutdown() + ''; + }; in -recurseIntoAttrs ( - concatMapAttrs (n: p: { ${n} = makePgVectorsTest p; }) (filterAttrs (n: p: !p.pkgs.pgvecto-rs.meta.broken) pkgs.postgresqlVersions) +lib.recurseIntoAttrs ( + lib.concatMapAttrs (n: p: { ${n} = makeTestFor p; }) (lib.filterAttrs (_: p: !p.pkgs.pgvecto-rs.meta.broken) pkgs.postgresqlVersions) // { - passthru.override = p: makePgVectorsTest p; + passthru.override = p: makeTestFor p; } ) diff --git a/nixos/tests/postgresql/postgresql-jit.nix b/nixos/tests/postgresql/postgresql-jit.nix index 40329d038fefe..69967f091312b 100644 --- a/nixos/tests/postgresql/postgresql-jit.nix +++ b/nixos/tests/postgresql/postgresql-jit.nix @@ -5,45 +5,48 @@ let inherit (pkgs) lib; - mkJitTest = package: makeTest { - name = "postgresql-jit-${package.name}"; - meta.maintainers = with lib.maintainers; [ ma27 ]; - nodes.machine = { pkgs, lib, ... }: { - services.postgresql = { - inherit package; - enable = true; - enableJIT = true; - initialScript = pkgs.writeText "init.sql" '' - create table demo (id int); - insert into demo (id) select generate_series(1, 5); - ''; + makeTestFor = package: + makeTest { + name = "postgresql-jit-${package.name}"; + meta.maintainers = with lib.maintainers; [ ma27 ]; + + nodes.machine = { pkgs, ... }: { + services.postgresql = { + inherit package; + enable = true; + enableJIT = true; + initialScript = pkgs.writeText "init.sql" '' + create table demo (id int); + insert into demo (id) select generate_series(1, 5); + ''; + }; }; - }; - testScript = '' - machine.start() - machine.wait_for_unit("postgresql.service") - with subtest("JIT is enabled"): - machine.succeed("sudo -u postgres psql <<<'show jit;' | grep 'on'") + testScript = '' + machine.start() + machine.wait_for_unit("postgresql.service") - with subtest("Test JIT works fine"): - output = machine.succeed( - "cat ${pkgs.writeText "test.sql" '' - set jit_above_cost = 1; - EXPLAIN ANALYZE SELECT CONCAT('jit result = ', SUM(id)) FROM demo; - SELECT CONCAT('jit result = ', SUM(id)) from demo; - ''} | sudo -u postgres psql" - ) - assert "JIT:" in output - assert "jit result = 15" in output + with subtest("JIT is enabled"): + machine.succeed("sudo -u postgres psql <<<'show jit;' | grep 'on'") - machine.shutdown() - ''; - }; + with subtest("Test JIT works fine"): + output = machine.succeed( + "cat ${pkgs.writeText "test.sql" '' + set jit_above_cost = 1; + EXPLAIN ANALYZE SELECT CONCAT('jit result = ', SUM(id)) FROM demo; + SELECT CONCAT('jit result = ', SUM(id)) from demo; + ''} | sudo -u postgres psql" + ) + assert "JIT:" in output + assert "jit result = 15" in output + + machine.shutdown() + ''; + }; in lib.recurseIntoAttrs ( - lib.concatMapAttrs (n: p: { ${n} = mkJitTest p; }) (lib.filterAttrs (n: _: lib.hasSuffix "_jit" n) pkgs.postgresqlVersions) + lib.concatMapAttrs (n: p: { ${n} = makeTestFor p; }) (lib.filterAttrs (n: _: lib.hasSuffix "_jit" n) pkgs.postgresqlVersions) // { - passthru.override = p: mkJitTest p; + passthru.override = p: makeTestFor p; } ) diff --git a/nixos/tests/postgresql/postgresql-tls-client-cert.nix b/nixos/tests/postgresql/postgresql-tls-client-cert.nix index 503bc77268c20..4939c96b2b84d 100644 --- a/nixos/tests/postgresql/postgresql-tls-client-cert.nix +++ b/nixos/tests/postgresql/postgresql-tls-client-cert.nix @@ -3,43 +3,41 @@ }: let - lib = pkgs.lib; + inherit (pkgs) lib; - makePostgresqlTlsClientCertTest = pkg: - let - runWithOpenSSL = file: cmd: pkgs.runCommand file - { - buildInputs = [ pkgs.openssl ]; - } - cmd; - caKey = runWithOpenSSL "ca.key" "openssl ecparam -name prime256v1 -genkey -noout -out $out"; - caCert = runWithOpenSSL - "ca.crt" - '' - openssl req -new -x509 -sha256 -key ${caKey} -out $out -subj "/CN=test.example" -days 36500 - ''; - serverKey = - runWithOpenSSL "server.key" "openssl ecparam -name prime256v1 -genkey -noout -out $out"; - serverKeyPath = "/var/lib/postgresql"; - serverCert = - runWithOpenSSL "server.crt" '' - openssl req -new -sha256 -key ${serverKey} -out server.csr -subj "/CN=db.test.example" - openssl x509 -req -in server.csr -CA ${caCert} -CAkey ${caKey} \ - -CAcreateserial -out $out -days 36500 -sha256 - ''; - clientKey = - runWithOpenSSL "client.key" "openssl ecparam -name prime256v1 -genkey -noout -out $out"; - clientCert = - runWithOpenSSL "client.crt" '' - openssl req -new -sha256 -key ${clientKey} -out client.csr -subj "/CN=test" - openssl x509 -req -in client.csr -CA ${caCert} -CAkey ${caKey} \ - -CAcreateserial -out $out -days 36500 -sha256 - ''; - clientKeyPath = "/root"; + runWithOpenSSL = file: cmd: pkgs.runCommand file + { + buildInputs = [ pkgs.openssl ]; + } + cmd; + caKey = runWithOpenSSL "ca.key" "openssl ecparam -name prime256v1 -genkey -noout -out $out"; + caCert = runWithOpenSSL + "ca.crt" + '' + openssl req -new -x509 -sha256 -key ${caKey} -out $out -subj "/CN=test.example" -days 36500 + ''; + serverKey = + runWithOpenSSL "server.key" "openssl ecparam -name prime256v1 -genkey -noout -out $out"; + serverKeyPath = "/var/lib/postgresql"; + serverCert = + runWithOpenSSL "server.crt" '' + openssl req -new -sha256 -key ${serverKey} -out server.csr -subj "/CN=db.test.example" + openssl x509 -req -in server.csr -CA ${caCert} -CAkey ${caKey} \ + -CAcreateserial -out $out -days 36500 -sha256 + ''; + clientKey = + runWithOpenSSL "client.key" "openssl ecparam -name prime256v1 -genkey -noout -out $out"; + clientCert = + runWithOpenSSL "client.crt" '' + openssl req -new -sha256 -key ${clientKey} -out client.csr -subj "/CN=test" + openssl x509 -req -in client.csr -CA ${caCert} -CAkey ${caKey} \ + -CAcreateserial -out $out -days 36500 -sha256 + ''; + clientKeyPath = "/root"; - in + makeTestFor = package: makeTest { - name = "postgresql-tls-client-cert-${pkg.name}"; + name = "postgresql-tls-client-cert-${package.name}"; meta.maintainers = with lib.maintainers; [ erictapen ]; nodes.server = { ... }: { @@ -52,7 +50,7 @@ let ''; }; services.postgresql = { - package = pkg; + inherit package; enable = true; enableTCPIP = true; ensureUsers = [ @@ -102,7 +100,7 @@ let PGSSLKEY = "${clientKeyPath}/client.key"; PGSSLROOTCERT = caCert; }; - systemPackages = [ pkg ]; + systemPackages = [ package ]; }; networking = { interfaces.eth1 = { @@ -120,11 +118,10 @@ let client.succeed("psql -c \"SELECT 1;\"") ''; }; - in lib.recurseIntoAttrs ( - lib.concatMapAttrs (n: p: { ${n} = makePostgresqlTlsClientCertTest p; }) pkgs.postgresqlVersions + lib.concatMapAttrs (n: p: { ${n} = makeTestFor p; }) pkgs.postgresqlVersions // { - passthru.override = p: makePostgresqlTlsClientCertTest p; + passthru.override = p: makeTestFor p; } ) diff --git a/nixos/tests/postgresql/postgresql-wal-receiver.nix b/nixos/tests/postgresql/postgresql-wal-receiver.nix index d46b3fcbdcf30..b9d69cf00684d 100644 --- a/nixos/tests/postgresql/postgresql-wal-receiver.nix +++ b/nixos/tests/postgresql/postgresql-wal-receiver.nix @@ -3,21 +3,20 @@ }: let - lib = pkgs.lib; + inherit (pkgs) lib; - makePostgresqlWalReceiverTest = pkg: + makeTestFor = package: let - postgresqlDataDir = "/var/lib/postgresql/${pkg.psqlSchema}"; + postgresqlDataDir = "/var/lib/postgresql/${package.psqlSchema}"; replicationUser = "wal_receiver_user"; replicationSlot = "wal_receiver_slot"; replicationConn = "postgresql://${replicationUser}@localhost"; baseBackupDir = "/var/cache/wals/pg_basebackup"; walBackupDir = "/var/cache/wals/pg_wal"; - recoveryFile = pkgs.writeTextDir "recovery.signal" ""; - - in makeTest { - name = "postgresql-wal-receiver-${pkg.name}"; + in + makeTest { + name = "postgresql-wal-receiver-${package.name}"; meta.maintainers = with lib.maintainers; [ pacien ]; nodes.machine = { ... }: { @@ -26,7 +25,7 @@ let ]; services.postgresql = { - package = pkg; + inherit package; enable = true; settings = { max_replication_slots = 10; @@ -45,7 +44,7 @@ let }; services.postgresqlWalReceiver.receivers.main = { - postgresqlPackage = pkg; + postgresqlPackage = package; connection = replicationConn; slot = replicationSlot; directory = walBackupDir; @@ -64,7 +63,7 @@ let # required only for 9.4 machine.sleep(5) machine.succeed( - "${pkg}/bin/pg_basebackup --dbname=${replicationConn} --pgdata=${baseBackupDir}" + "${package}/bin/pg_basebackup --dbname=${replicationConn} --pgdata=${baseBackupDir}" ) # create a dummy table with 100 records @@ -105,8 +104,8 @@ let }; in lib.recurseIntoAttrs ( - lib.concatMapAttrs (n: p: { ${n} = makePostgresqlWalReceiverTest p; }) pkgs.postgresqlVersions + lib.concatMapAttrs (n: p: { ${n} = makeTestFor p; }) pkgs.postgresqlVersions // { - passthru.override = p: makePostgresqlWalReceiverTest p; + passthru.override = p: makeTestFor p; } ) diff --git a/nixos/tests/postgresql/postgresql.nix b/nixos/tests/postgresql/postgresql.nix index 56ec43d0a3b69..13a768d9a8a9a 100644 --- a/nixos/tests/postgresql/postgresql.nix +++ b/nixos/tests/postgresql/postgresql.nix @@ -2,14 +2,14 @@ , makeTest }: -with pkgs.lib; - let - testsForPackage = package: - recurseIntoAttrs { - postgresql = make-postgresql-test package false; - postgresql-backup-all = make-postgresql-test package true; - postgresql-clauses = mk-ensure-clauses-test package; + inherit (pkgs) lib; + + makeTestFor = package: + lib.recurseIntoAttrs { + postgresql = makeTestForWithBackupAll package false; + postgresql-backup-all = makeTestForWithBackupAll package true; + postgresql-clauses = makeEnsureTestFor package; }; test-sql = pkgs.writeText "postgresql-test" '' @@ -26,207 +26,208 @@ let INSERT INTO xmltest (doc) VALUES ('ok'); -- check if libxml2 enabled ''; - make-postgresql-test = postgresql-package: backup-all: makeTest { - name = "postgresql${optionalString backup-all "-backup-all"}-${postgresql-package.name}"; - meta = with pkgs.lib.maintainers; { - maintainers = [ zagy ]; - }; + makeTestForWithBackupAll = package: backupAll: + makeTest { + name = "postgresql${lib.optionalString backupAll "-backup-all"}-${package.name}"; + meta = with lib.maintainers; { + maintainers = [ zagy ]; + }; - nodes.machine = {...}: - { - services.postgresql = { - enable = true; - package = postgresql-package; + nodes.machine = {...}: + { + services.postgresql = { + inherit (package); + enable = true; + }; + + services.postgresqlBackup = { + enable = true; + databases = lib.optional (!backupAll) "postgres"; + }; }; - services.postgresqlBackup = { - enable = true; - databases = optional (!backup-all) "postgres"; - }; - }; + testScript = let + backupName = if backupAll then "all" else "postgres"; + backupService = if backupAll then "postgresqlBackup" else "postgresqlBackup-postgres"; + backupFileBase = "/var/backup/postgresql/${backupName}"; + in '' + def check_count(statement, lines): + return 'test $(sudo -u postgres psql postgres -tAc "{}"|wc -l) -eq {}'.format( + statement, lines + ) - testScript = let - backupName = if backup-all then "all" else "postgres"; - backupService = if backup-all then "postgresqlBackup" else "postgresqlBackup-postgres"; - backupFileBase = "/var/backup/postgresql/${backupName}"; - in '' - def check_count(statement, lines): - return 'test $(sudo -u postgres psql postgres -tAc "{}"|wc -l) -eq {}'.format( - statement, lines - ) - - - machine.start() - machine.wait_for_unit("postgresql") - - with subtest("Postgresql is available just after unit start"): - machine.succeed( - "cat ${test-sql} | sudo -u postgres psql" - ) - - with subtest("Postgresql survives restart (bug #1735)"): - machine.shutdown() - import time - time.sleep(2) - machine.start() - machine.wait_for_unit("postgresql") - - machine.fail(check_count("SELECT * FROM sth;", 3)) - machine.succeed(check_count("SELECT * FROM sth;", 5)) - machine.fail(check_count("SELECT * FROM sth;", 4)) - machine.succeed(check_count("SELECT xpath('/test/text()', doc) FROM xmltest;", 1)) - - with subtest("Backup service works"): - machine.succeed( - "systemctl start ${backupService}.service", - "zcat ${backupFileBase}.sql.gz | grep 'ok'", - "ls -hal /var/backup/postgresql/ >/dev/console", - "stat -c '%a' ${backupFileBase}.sql.gz | grep 600", - ) - with subtest("Backup service removes prev files"): - machine.succeed( - # Create dummy prev files. - "touch ${backupFileBase}.prev.sql{,.gz,.zstd}", - "chown postgres:postgres ${backupFileBase}.prev.sql{,.gz,.zstd}", - - # Run backup. - "systemctl start ${backupService}.service", - "ls -hal /var/backup/postgresql/ >/dev/console", - - # Since nothing has changed in the database, the cur and prev files - # should match. - "zcat ${backupFileBase}.sql.gz | grep 'ok'", - "cmp ${backupFileBase}.sql.gz ${backupFileBase}.prev.sql.gz", - - # The prev files with unused suffix should be removed. - "[ ! -f '${backupFileBase}.prev.sql' ]", - "[ ! -f '${backupFileBase}.prev.sql.zstd' ]", - - # Both cur and prev file should only be accessible by the postgres user. - "stat -c '%a' ${backupFileBase}.sql.gz | grep 600", - "stat -c '%a' '${backupFileBase}.prev.sql.gz' | grep 600", - ) - with subtest("Backup service fails gracefully"): - # Sabotage the backup process - machine.succeed("rm /run/postgresql/.s.PGSQL.5432") - machine.fail( - "systemctl start ${backupService}.service", - ) - machine.succeed( - "ls -hal /var/backup/postgresql/ >/dev/console", - "zcat ${backupFileBase}.prev.sql.gz | grep 'ok'", - "stat ${backupFileBase}.in-progress.sql.gz", - ) - # In a previous version, the second run would overwrite prev.sql.gz, - # so we test a second run as well. - machine.fail( - "systemctl start ${backupService}.service", - ) - machine.succeed( - "stat ${backupFileBase}.in-progress.sql.gz", - "zcat ${backupFileBase}.prev.sql.gz | grep 'ok'", - ) - - - with subtest("Initdb works"): - machine.succeed("sudo -u postgres initdb -D /tmp/testpostgres2") - - machine.log(machine.execute("systemd-analyze security postgresql.service | grep -v ✓")[1]) - - machine.shutdown() - ''; - - }; - - mk-ensure-clauses-test = postgresql-package: makeTest { - name = "postgresql-clauses-${postgresql-package.name}"; - meta = with pkgs.lib.maintainers; { - maintainers = [ zagy ]; - }; - nodes.machine = {...}: - { - services.postgresql = { - enable = true; - package = postgresql-package; - ensureUsers = [ - { - name = "all-clauses"; - ensureClauses = { - superuser = true; - createdb = true; - createrole = true; - "inherit" = true; - login = true; - replication = true; - bypassrls = true; - }; - } - { - name = "default-clauses"; - } - ]; - }; - }; + machine.start() + machine.wait_for_unit("postgresql") + + with subtest("Postgresql is available just after unit start"): + machine.succeed( + "cat ${test-sql} | sudo -u postgres psql" + ) - testScript = let - getClausesQuery = user: pkgs.lib.concatStringsSep " " - [ - "SELECT row_to_json(row)" - "FROM (" - "SELECT" - "rolsuper," - "rolinherit," - "rolcreaterole," - "rolcreatedb," - "rolcanlogin," - "rolreplication," - "rolbypassrls" - "FROM pg_roles" - "WHERE rolname = '${user}'" - ") row;" - ]; - in '' - import json - machine.start() - machine.wait_for_unit("postgresql") - - with subtest("All user permissions are set according to the ensureClauses attr"): - clauses = json.loads( + with subtest("Postgresql survives restart (bug #1735)"): + machine.shutdown() + import time + time.sleep(2) + machine.start() + machine.wait_for_unit("postgresql") + + machine.fail(check_count("SELECT * FROM sth;", 3)) + machine.succeed(check_count("SELECT * FROM sth;", 5)) + machine.fail(check_count("SELECT * FROM sth;", 4)) + machine.succeed(check_count("SELECT xpath('/test/text()', doc) FROM xmltest;", 1)) + + with subtest("Backup service works"): + machine.succeed( + "systemctl start ${backupService}.service", + "zcat ${backupFileBase}.sql.gz | grep 'ok'", + "ls -hal /var/backup/postgresql/ >/dev/console", + "stat -c '%a' ${backupFileBase}.sql.gz | grep 600", + ) + with subtest("Backup service removes prev files"): machine.succeed( - "sudo -u postgres psql -tc \"${getClausesQuery "all-clauses"}\"" + # Create dummy prev files. + "touch ${backupFileBase}.prev.sql{,.gz,.zstd}", + "chown postgres:postgres ${backupFileBase}.prev.sql{,.gz,.zstd}", + + # Run backup. + "systemctl start ${backupService}.service", + "ls -hal /var/backup/postgresql/ >/dev/console", + + # Since nothing has changed in the database, the cur and prev files + # should match. + "zcat ${backupFileBase}.sql.gz | grep 'ok'", + "cmp ${backupFileBase}.sql.gz ${backupFileBase}.prev.sql.gz", + + # The prev files with unused suffix should be removed. + "[ ! -f '${backupFileBase}.prev.sql' ]", + "[ ! -f '${backupFileBase}.prev.sql.zstd' ]", + + # Both cur and prev file should only be accessible by the postgres user. + "stat -c '%a' ${backupFileBase}.sql.gz | grep 600", + "stat -c '%a' '${backupFileBase}.prev.sql.gz' | grep 600", + ) + with subtest("Backup service fails gracefully"): + # Sabotage the backup process + machine.succeed("rm /run/postgresql/.s.PGSQL.5432") + machine.fail( + "systemctl start ${backupService}.service", ) - ) - print(clauses) - assert clauses['rolsuper'], 'expected user with clauses to have superuser clause' - assert clauses['rolinherit'], 'expected user with clauses to have inherit clause' - assert clauses['rolcreaterole'], 'expected user with clauses to have create role clause' - assert clauses['rolcreatedb'], 'expected user with clauses to have create db clause' - assert clauses['rolcanlogin'], 'expected user with clauses to have login clause' - assert clauses['rolreplication'], 'expected user with clauses to have replication clause' - assert clauses['rolbypassrls'], 'expected user with clauses to have bypassrls clause' - - with subtest("All user permissions default when ensureClauses is not provided"): - clauses = json.loads( machine.succeed( - "sudo -u postgres psql -tc \"${getClausesQuery "default-clauses"}\"" + "ls -hal /var/backup/postgresql/ >/dev/console", + "zcat ${backupFileBase}.prev.sql.gz | grep 'ok'", + "stat ${backupFileBase}.in-progress.sql.gz", + ) + # In a previous version, the second run would overwrite prev.sql.gz, + # so we test a second run as well. + machine.fail( + "systemctl start ${backupService}.service", + ) + machine.succeed( + "stat ${backupFileBase}.in-progress.sql.gz", + "zcat ${backupFileBase}.prev.sql.gz | grep 'ok'", + ) + + + with subtest("Initdb works"): + machine.succeed("sudo -u postgres initdb -D /tmp/testpostgres2") + + machine.log(machine.execute("systemd-analyze security postgresql.service | grep -v ✓")[1]) + + machine.shutdown() + ''; + }; + + makeEnsureTestFor = package: + makeTest { + name = "postgresql-clauses-${package.name}"; + meta = with lib.maintainers; { + maintainers = [ zagy ]; + }; + + nodes.machine = {...}: + { + services.postgresql = { + inherit package; + enable = true; + ensureUsers = [ + { + name = "all-clauses"; + ensureClauses = { + superuser = true; + createdb = true; + createrole = true; + "inherit" = true; + login = true; + replication = true; + bypassrls = true; + }; + } + { + name = "default-clauses"; + } + ]; + }; + }; + + testScript = let + getClausesQuery = user: lib.concatStringsSep " " + [ + "SELECT row_to_json(row)" + "FROM (" + "SELECT" + "rolsuper," + "rolinherit," + "rolcreaterole," + "rolcreatedb," + "rolcanlogin," + "rolreplication," + "rolbypassrls" + "FROM pg_roles" + "WHERE rolname = '${user}'" + ") row;" + ]; + in '' + import json + machine.start() + machine.wait_for_unit("postgresql") + + with subtest("All user permissions are set according to the ensureClauses attr"): + clauses = json.loads( + machine.succeed( + "sudo -u postgres psql -tc \"${getClausesQuery "all-clauses"}\"" + ) ) - ) - assert not clauses['rolsuper'], 'expected user with no clauses set to have default superuser clause' - assert clauses['rolinherit'], 'expected user with no clauses set to have default inherit clause' - assert not clauses['rolcreaterole'], 'expected user with no clauses set to have default create role clause' - assert not clauses['rolcreatedb'], 'expected user with no clauses set to have default create db clause' - assert clauses['rolcanlogin'], 'expected user with no clauses set to have default login clause' - assert not clauses['rolreplication'], 'expected user with no clauses set to have default replication clause' - assert not clauses['rolbypassrls'], 'expected user with no clauses set to have default bypassrls clause' - - machine.shutdown() - ''; - }; + print(clauses) + assert clauses['rolsuper'], 'expected user with clauses to have superuser clause' + assert clauses['rolinherit'], 'expected user with clauses to have inherit clause' + assert clauses['rolcreaterole'], 'expected user with clauses to have create role clause' + assert clauses['rolcreatedb'], 'expected user with clauses to have create db clause' + assert clauses['rolcanlogin'], 'expected user with clauses to have login clause' + assert clauses['rolreplication'], 'expected user with clauses to have replication clause' + assert clauses['rolbypassrls'], 'expected user with clauses to have bypassrls clause' + + with subtest("All user permissions default when ensureClauses is not provided"): + clauses = json.loads( + machine.succeed( + "sudo -u postgres psql -tc \"${getClausesQuery "default-clauses"}\"" + ) + ) + assert not clauses['rolsuper'], 'expected user with no clauses set to have default superuser clause' + assert clauses['rolinherit'], 'expected user with no clauses set to have default inherit clause' + assert not clauses['rolcreaterole'], 'expected user with no clauses set to have default create role clause' + assert not clauses['rolcreatedb'], 'expected user with no clauses set to have default create db clause' + assert clauses['rolcanlogin'], 'expected user with no clauses set to have default login clause' + assert not clauses['rolreplication'], 'expected user with no clauses set to have default replication clause' + assert not clauses['rolbypassrls'], 'expected user with no clauses set to have default bypassrls clause' + + machine.shutdown() + ''; + }; in -recurseIntoAttrs ( - concatMapAttrs (n: p: { ${n} = testsForPackage p; }) pkgs.postgresqlVersions +lib.recurseIntoAttrs ( + lib.concatMapAttrs (n: p: { ${n} = makeTestFor p; }) pkgs.postgresqlVersions // { - passthru.override = p: testsForPackage p; + passthru.override = p: makeTestFor p; } ) diff --git a/nixos/tests/postgresql/timescaledb.nix b/nixos/tests/postgresql/timescaledb.nix index 7f37e79bf0946..a01f890a12c6c 100644 --- a/nixos/tests/postgresql/timescaledb.nix +++ b/nixos/tests/postgresql/timescaledb.nix @@ -2,9 +2,9 @@ , makeTest }: -with pkgs.lib; - let + inherit (pkgs) lib; + test-sql = pkgs.writeText "postgresql-test" '' CREATE EXTENSION timescaledb; CREATE EXTENSION timescaledb_toolkit; @@ -38,55 +38,55 @@ let SELECT * FROM sth; ''; - makeTimescaleDbTest = postgresqlPackage: makeTest { - name = "timescaledb-${postgresqlPackage.name}"; - meta = with pkgs.lib.maintainers; { - maintainers = [ typetetris ]; - }; - - nodes.machine = { ... }: - { - services.postgresql = { - enable = true; - package = postgresqlPackage; - extraPlugins = ps: with ps; [ - timescaledb - timescaledb_toolkit - ]; - settings = { shared_preload_libraries = "timescaledb, timescaledb_toolkit"; }; - }; + makeTestFor = package: + makeTest { + name = "timescaledb-${package.name}"; + meta = with lib.maintainers; { + maintainers = [ typetetris ]; }; - testScript = '' - def check_count(statement, lines): - return 'test $(sudo -u postgres psql postgres -tAc "{}"|wc -l) -eq {}'.format( - statement, lines - ) + nodes.machine = { ... }: + { + services.postgresql = { + inherit package; + enable = true; + extraPlugins = ps: with ps; [ + timescaledb + timescaledb_toolkit + ]; + settings = { shared_preload_libraries = "timescaledb, timescaledb_toolkit"; }; + }; + }; + testScript = '' + def check_count(statement, lines): + return 'test $(sudo -u postgres psql postgres -tAc "{}"|wc -l) -eq {}'.format( + statement, lines + ) - machine.start() - machine.wait_for_unit("postgresql") - with subtest("Postgresql with extensions timescaledb and timescaledb_toolkit is available just after unit start"): - machine.succeed( - "sudo -u postgres psql -f ${test-sql}" - ) + machine.start() + machine.wait_for_unit("postgresql") - machine.fail(check_count("SELECT * FROM sth;", 3)) - machine.succeed(check_count("SELECT * FROM sth;", 5)) - machine.fail(check_count("SELECT * FROM sth;", 4)) + with subtest("Postgresql with extensions timescaledb and timescaledb_toolkit is available just after unit start"): + machine.succeed( + "sudo -u postgres psql -f ${test-sql}" + ) - machine.shutdown() - ''; + machine.fail(check_count("SELECT * FROM sth;", 3)) + machine.succeed(check_count("SELECT * FROM sth;", 5)) + machine.fail(check_count("SELECT * FROM sth;", 4)) - }; + machine.shutdown() + ''; + }; in # Not run by default, because this requires allowUnfree. # To run these tests: # NIXPKGS_ALLOW_UNFREE=1 nix-build -A nixosTests.postgresql.timescaledb -dontRecurseIntoAttrs ( - pkgs.lib.concatMapAttrs (n: p: { ${n} = makeTimescaleDbTest p; }) (filterAttrs (n: p: !p.pkgs.timescaledb.meta.broken) pkgs.postgresqlVersions) +lib.dontRecurseIntoAttrs ( + lib.concatMapAttrs (n: p: { ${n} = makeTestFor p; }) (lib.filterAttrs (_: p: !p.pkgs.timescaledb.meta.broken) pkgs.postgresqlVersions) // { - passthru.override = p: makeTimescaleDbTest p; + passthru.override = p: makeTestFor p; } ) diff --git a/nixos/tests/postgresql/tsja.nix b/nixos/tests/postgresql/tsja.nix index dedda35540ace..8d78093136df7 100644 --- a/nixos/tests/postgresql/tsja.nix +++ b/nixos/tests/postgresql/tsja.nix @@ -3,27 +3,25 @@ }: let - makeTsjaTest = postgresqlPackage: + inherit (pkgs) lib; + + makeTestFor = package: makeTest { - name = "tsja-${postgresqlPackage.name}"; + name = "tsja-${package.name}"; meta = { maintainers = with lib.maintainers; [ chayleaf ]; }; - nodes = { - master = - { config, ... }: - - { - services.postgresql = { - enable = true; - package = postgresqlPackage; - extraPlugins = ps: with ps; [ - tsja - ]; - }; + nodes.master = { ... }: + { + services.postgresql = { + inherit package; + enable = true; + extraPlugins = ps: with ps; [ + tsja + ]; }; - }; + }; testScript = '' start_all() @@ -38,9 +36,9 @@ let ''; }; in -pkgs.lib.recurseIntoAttrs ( - pkgs.lib.concatMapAttrs (n: p: { ${n} = makeTsjaTest p; }) pkgs.postgresqlVersions +lib.recurseIntoAttrs ( + lib.concatMapAttrs (n: p: { ${n} = makeTestFor p; }) (lib.filterAttrs (_: p: !p.pkgs.tsja.meta.broken) pkgs.postgresqlVersions) // { - passthru.override = p: makeTsjaTest p; + passthru.override = p: makeTestFor p; } ) diff --git a/nixos/tests/postgresql/wal2json.nix b/nixos/tests/postgresql/wal2json.nix index 782fc5c94f66b..0333369722f0c 100644 --- a/nixos/tests/postgresql/wal2json.nix +++ b/nixos/tests/postgresql/wal2json.nix @@ -5,17 +5,16 @@ let inherit (pkgs) lib; - makePostgresqlWal2jsonTest = - postgresqlPackage: + makeTestFor = package: makeTest { - name = "wal2json-${postgresqlPackage.name}"; + name = "wal2json-${package.name}"; meta.maintainers = with pkgs.lib.maintainers; [ euank ]; nodes.machine = { services.postgresql = { - package = postgresqlPackage; + inherit package; enable = true; - extraPlugins = with postgresqlPackage.pkgs; [ wal2json ]; + extraPlugins = with package.pkgs; [ wal2json ]; settings = { wal_level = "logical"; max_replication_slots = "10"; @@ -40,11 +39,10 @@ let ) ''; }; - in lib.recurseIntoAttrs ( - lib.concatMapAttrs (n: p: { ${n} = makePostgresqlWal2jsonTest p; }) pkgs.postgresqlVersions + lib.concatMapAttrs (n: p: { ${n} = makeTestFor p; }) (lib.filterAttrs (_: p: !p.pkgs.wal2json.meta.broken) pkgs.postgresqlVersions) // { - passthru.override = p: makePostgresqlWal2jsonTest p; + passthru.override = p: makeTestFor p; } ) From 45cef36e39b2b910a1fdebc76412add1e01cf6bc Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Sat, 2 Nov 2024 23:04:45 +0100 Subject: [PATCH 17/17] nixosTests.postgresql: run nixfmt Because with as many changes as in here anybody working on those test files will have merge conflicts anyway. --- nixos/tests/postgresql/anonymizer.nix | 30 +- nixos/tests/postgresql/pgjwt.nix | 53 ++-- nixos/tests/postgresql/pgvecto-rs.nix | 57 ++-- nixos/tests/postgresql/postgresql-jit.nix | 34 +- .../postgresql/postgresql-tls-client-cert.nix | 190 +++++------ .../postgresql/postgresql-wal-receiver.nix | 72 +++-- nixos/tests/postgresql/postgresql.nix | 295 +++++++++--------- nixos/tests/postgresql/timescaledb.nix | 28 +- nixos/tests/postgresql/tsja.nix | 22 +- nixos/tests/postgresql/wal2json.nix | 12 +- 10 files changed, 433 insertions(+), 360 deletions(-) diff --git a/nixos/tests/postgresql/anonymizer.nix b/nixos/tests/postgresql/anonymizer.nix index ec971aa710253..3a5f69086eaac 100644 --- a/nixos/tests/postgresql/anonymizer.nix +++ b/nixos/tests/postgresql/anonymizer.nix @@ -1,24 +1,28 @@ -{ pkgs -, makeTest +{ + pkgs, + makeTest, }: let inherit (pkgs) lib; - makeTestFor = package: + makeTestFor = + package: makeTest { name = "postgresql_anonymizer-${package.name}"; meta.maintainers = lib.teams.flyingcircus.members; - nodes.machine = { pkgs, ... }: { - environment.systemPackages = [ pkgs.pg-dump-anon ]; - services.postgresql = { - inherit package; - enable = true; - extraPlugins = ps: [ ps.anonymizer ]; - settings.shared_preload_libraries = [ "anon" ]; + nodes.machine = + { pkgs, ... }: + { + environment.systemPackages = [ pkgs.pg-dump-anon ]; + services.postgresql = { + inherit package; + enable = true; + extraPlugins = ps: [ ps.anonymizer ]; + settings.shared_preload_libraries = [ "anon" ]; + }; }; - }; testScript = '' start_all() @@ -103,7 +107,9 @@ let }; in lib.recurseIntoAttrs ( - lib.concatMapAttrs (n: p: { ${n} = makeTestFor p; }) (lib.filterAttrs (_: p: !p.pkgs.anonymizer.meta.broken) pkgs.postgresqlVersions) + lib.concatMapAttrs (n: p: { ${n} = makeTestFor p; }) ( + lib.filterAttrs (_: p: !p.pkgs.anonymizer.meta.broken) pkgs.postgresqlVersions + ) // { passthru.override = p: makeTestFor p; } diff --git a/nixos/tests/postgresql/pgjwt.nix b/nixos/tests/postgresql/pgjwt.nix index 2c0b77f180ff8..81e5dac41adae 100644 --- a/nixos/tests/postgresql/pgjwt.nix +++ b/nixos/tests/postgresql/pgjwt.nix @@ -1,43 +1,56 @@ -{ pkgs -, makeTest +{ + pkgs, + makeTest, }: let inherit (pkgs) lib; - makeTestFor = package: + makeTestFor = + package: makeTest { name = "pgjwt-${package.name}"; meta = with lib.maintainers; { - maintainers = [ spinus willibutz ]; + maintainers = [ + spinus + willibutz + ]; }; - nodes.master = { ... }: + nodes.master = + { ... }: { services.postgresql = { inherit package; enable = true; - extraPlugins = ps: with ps; [ pgjwt pgtap ]; + extraPlugins = + ps: with ps; [ + pgjwt + pgtap + ]; }; }; - testScript = { nodes, ... }: - let - sqlSU = "${nodes.master.services.postgresql.superUser}"; - pgProve = "${pkgs.perlPackages.TAPParserSourceHandlerpgTAP}"; - inherit (nodes.master.services.postgresql.package.pkgs) pgjwt; - in - '' - start_all() - master.wait_for_unit("postgresql") - master.succeed( - "${pkgs.sudo}/bin/sudo -u ${sqlSU} ${pgProve}/bin/pg_prove -d postgres -v -f ${pgjwt.src}/test.sql" - ) - ''; + testScript = + { nodes, ... }: + let + sqlSU = "${nodes.master.services.postgresql.superUser}"; + pgProve = "${pkgs.perlPackages.TAPParserSourceHandlerpgTAP}"; + inherit (nodes.master.services.postgresql.package.pkgs) pgjwt; + in + '' + start_all() + master.wait_for_unit("postgresql") + master.succeed( + "${pkgs.sudo}/bin/sudo -u ${sqlSU} ${pgProve}/bin/pg_prove -d postgres -v -f ${pgjwt.src}/test.sql" + ) + ''; }; in lib.recurseIntoAttrs ( - lib.concatMapAttrs (n: p: { ${n} = makeTestFor p; }) (lib.filterAttrs (_: p: !p.pkgs.pgjwt.meta.broken) pkgs.postgresqlVersions) + lib.concatMapAttrs (n: p: { ${n} = makeTestFor p; }) ( + lib.filterAttrs (_: p: !p.pkgs.pgjwt.meta.broken) pkgs.postgresqlVersions + ) // { passthru.override = p: makeTestFor p; } diff --git a/nixos/tests/postgresql/pgvecto-rs.nix b/nixos/tests/postgresql/pgvecto-rs.nix index 8f8c3fdd84368..9d8389eecf99a 100644 --- a/nixos/tests/postgresql/pgvecto-rs.nix +++ b/nixos/tests/postgresql/pgvecto-rs.nix @@ -1,5 +1,6 @@ -{ pkgs -, makeTest +{ + pkgs, + makeTest, }: let @@ -22,52 +23,58 @@ let ('a thin dog sat on a mat and ate a thin rat', '[10, 11, 12]'); ''; - makeTestFor = postgresqlPackage: + makeTestFor = + postgresqlPackage: makeTest { name = "pgvecto-rs-${postgresqlPackage.name}"; meta = with lib.maintainers; { maintainers = [ diogotcorreia ]; }; - nodes.machine = { ... }: + nodes.machine = + { ... }: { services.postgresql = { enable = true; package = postgresqlPackage; - extraPlugins = ps: with ps; [ - pgvecto-rs - ]; + extraPlugins = + ps: with ps; [ + pgvecto-rs + ]; settings.shared_preload_libraries = "vectors"; }; }; - testScript = { nodes, ... }: - let - inherit (nodes.machine.services.postgresql.package.pkgs) pgvecto-rs; - in - '' - def check_count(statement, lines): - return 'test $(sudo -u postgres psql postgres -tAc "{}"|wc -l) -eq {}'.format( - statement, lines - ) + testScript = + { nodes, ... }: + let + inherit (nodes.machine.services.postgresql.package.pkgs) pgvecto-rs; + in + '' + def check_count(statement, lines): + return 'test $(sudo -u postgres psql postgres -tAc "{}"|wc -l) -eq {}'.format( + statement, lines + ) - machine.start() - machine.wait_for_unit("postgresql") + machine.start() + machine.wait_for_unit("postgresql") - with subtest("Postgresql with extension vectors is available just after unit start"): - machine.succeed(check_count("SELECT * FROM pg_available_extensions WHERE name = 'vectors' AND default_version = '${pgvecto-rs.version}';", 1)) + with subtest("Postgresql with extension vectors is available just after unit start"): + machine.succeed(check_count("SELECT * FROM pg_available_extensions WHERE name = 'vectors' AND default_version = '${pgvecto-rs.version}';", 1)) - machine.succeed("sudo -u postgres psql -f ${test-sql}") + machine.succeed("sudo -u postgres psql -f ${test-sql}") - machine.succeed(check_count("SELECT content, embedding FROM items WHERE to_tsvector('english', content) @@ 'cat & rat'::tsquery;", 2)) + machine.succeed(check_count("SELECT content, embedding FROM items WHERE to_tsvector('english', content) @@ 'cat & rat'::tsquery;", 2)) - machine.shutdown() - ''; + machine.shutdown() + ''; }; in lib.recurseIntoAttrs ( - lib.concatMapAttrs (n: p: { ${n} = makeTestFor p; }) (lib.filterAttrs (_: p: !p.pkgs.pgvecto-rs.meta.broken) pkgs.postgresqlVersions) + lib.concatMapAttrs (n: p: { ${n} = makeTestFor p; }) ( + lib.filterAttrs (_: p: !p.pkgs.pgvecto-rs.meta.broken) pkgs.postgresqlVersions + ) // { passthru.override = p: makeTestFor p; } diff --git a/nixos/tests/postgresql/postgresql-jit.nix b/nixos/tests/postgresql/postgresql-jit.nix index 69967f091312b..5d0406062eae3 100644 --- a/nixos/tests/postgresql/postgresql-jit.nix +++ b/nixos/tests/postgresql/postgresql-jit.nix @@ -1,26 +1,30 @@ -{ pkgs -, makeTest +{ + pkgs, + makeTest, }: let inherit (pkgs) lib; - makeTestFor = package: + makeTestFor = + package: makeTest { name = "postgresql-jit-${package.name}"; meta.maintainers = with lib.maintainers; [ ma27 ]; - nodes.machine = { pkgs, ... }: { - services.postgresql = { - inherit package; - enable = true; - enableJIT = true; - initialScript = pkgs.writeText "init.sql" '' - create table demo (id int); - insert into demo (id) select generate_series(1, 5); - ''; + nodes.machine = + { pkgs, ... }: + { + services.postgresql = { + inherit package; + enable = true; + enableJIT = true; + initialScript = pkgs.writeText "init.sql" '' + create table demo (id int); + insert into demo (id) select generate_series(1, 5); + ''; + }; }; - }; testScript = '' machine.start() @@ -45,7 +49,9 @@ let }; in lib.recurseIntoAttrs ( - lib.concatMapAttrs (n: p: { ${n} = makeTestFor p; }) (lib.filterAttrs (n: _: lib.hasSuffix "_jit" n) pkgs.postgresqlVersions) + lib.concatMapAttrs (n: p: { ${n} = makeTestFor p; }) ( + lib.filterAttrs (n: _: lib.hasSuffix "_jit" n) pkgs.postgresqlVersions + ) // { passthru.override = p: makeTestFor p; } diff --git a/nixos/tests/postgresql/postgresql-tls-client-cert.nix b/nixos/tests/postgresql/postgresql-tls-client-cert.nix index 4939c96b2b84d..d7cddb625256c 100644 --- a/nixos/tests/postgresql/postgresql-tls-client-cert.nix +++ b/nixos/tests/postgresql/postgresql-tls-client-cert.nix @@ -1,116 +1,124 @@ -{ pkgs -, makeTest +{ + pkgs, + makeTest, }: let inherit (pkgs) lib; - runWithOpenSSL = file: cmd: pkgs.runCommand file - { + runWithOpenSSL = + file: cmd: + pkgs.runCommand file { buildInputs = [ pkgs.openssl ]; - } - cmd; + } cmd; caKey = runWithOpenSSL "ca.key" "openssl ecparam -name prime256v1 -genkey -noout -out $out"; - caCert = runWithOpenSSL - "ca.crt" - '' - openssl req -new -x509 -sha256 -key ${caKey} -out $out -subj "/CN=test.example" -days 36500 - ''; - serverKey = - runWithOpenSSL "server.key" "openssl ecparam -name prime256v1 -genkey -noout -out $out"; + caCert = runWithOpenSSL "ca.crt" '' + openssl req -new -x509 -sha256 -key ${caKey} -out $out -subj "/CN=test.example" -days 36500 + ''; + serverKey = runWithOpenSSL "server.key" "openssl ecparam -name prime256v1 -genkey -noout -out $out"; serverKeyPath = "/var/lib/postgresql"; - serverCert = - runWithOpenSSL "server.crt" '' - openssl req -new -sha256 -key ${serverKey} -out server.csr -subj "/CN=db.test.example" - openssl x509 -req -in server.csr -CA ${caCert} -CAkey ${caKey} \ - -CAcreateserial -out $out -days 36500 -sha256 - ''; - clientKey = - runWithOpenSSL "client.key" "openssl ecparam -name prime256v1 -genkey -noout -out $out"; - clientCert = - runWithOpenSSL "client.crt" '' - openssl req -new -sha256 -key ${clientKey} -out client.csr -subj "/CN=test" - openssl x509 -req -in client.csr -CA ${caCert} -CAkey ${caKey} \ - -CAcreateserial -out $out -days 36500 -sha256 - ''; + serverCert = runWithOpenSSL "server.crt" '' + openssl req -new -sha256 -key ${serverKey} -out server.csr -subj "/CN=db.test.example" + openssl x509 -req -in server.csr -CA ${caCert} -CAkey ${caKey} \ + -CAcreateserial -out $out -days 36500 -sha256 + ''; + clientKey = runWithOpenSSL "client.key" "openssl ecparam -name prime256v1 -genkey -noout -out $out"; + clientCert = runWithOpenSSL "client.crt" '' + openssl req -new -sha256 -key ${clientKey} -out client.csr -subj "/CN=test" + openssl x509 -req -in client.csr -CA ${caCert} -CAkey ${caKey} \ + -CAcreateserial -out $out -days 36500 -sha256 + ''; clientKeyPath = "/root"; - makeTestFor = package: + makeTestFor = + package: makeTest { name = "postgresql-tls-client-cert-${package.name}"; meta.maintainers = with lib.maintainers; [ erictapen ]; - nodes.server = { ... }: { - system.activationScripts = { - keyPlacement.text = '' - mkdir -p '${serverKeyPath}' - cp '${serverKey}' '${serverKeyPath}/server.key' - chown postgres:postgres '${serverKeyPath}/server.key' - chmod 600 '${serverKeyPath}/server.key' - ''; - }; - services.postgresql = { - inherit package; - enable = true; - enableTCPIP = true; - ensureUsers = [ - { - name = "test"; - ensureDBOwnership = true; - } - ]; - ensureDatabases = [ "test" ]; - settings = { - ssl = "on"; - ssl_ca_file = toString caCert; - ssl_cert_file = toString serverCert; - ssl_key_file = "${serverKeyPath}/server.key"; + nodes.server = + { ... }: + { + system.activationScripts = { + keyPlacement.text = '' + mkdir -p '${serverKeyPath}' + cp '${serverKey}' '${serverKeyPath}/server.key' + chown postgres:postgres '${serverKeyPath}/server.key' + chmod 600 '${serverKeyPath}/server.key' + ''; }; - authentication = '' - hostssl test test ::/0 cert clientcert=verify-full - ''; - }; - networking = { - interfaces.eth1 = { - ipv6.addresses = [ - { address = "fc00::1"; prefixLength = 120; } + services.postgresql = { + inherit package; + enable = true; + enableTCPIP = true; + ensureUsers = [ + { + name = "test"; + ensureDBOwnership = true; + } ]; + ensureDatabases = [ "test" ]; + settings = { + ssl = "on"; + ssl_ca_file = toString caCert; + ssl_cert_file = toString serverCert; + ssl_key_file = "${serverKeyPath}/server.key"; + }; + authentication = '' + hostssl test test ::/0 cert clientcert=verify-full + ''; + }; + networking = { + interfaces.eth1 = { + ipv6.addresses = [ + { + address = "fc00::1"; + prefixLength = 120; + } + ]; + }; + firewall.allowedTCPPorts = [ 5432 ]; }; - firewall.allowedTCPPorts = [ 5432 ]; }; - }; - nodes.client = { ... }: { - system.activationScripts = { - keyPlacement.text = '' - mkdir -p '${clientKeyPath}' - cp '${clientKey}' '${clientKeyPath}/client.key' - chown root:root '${clientKeyPath}/client.key' - chmod 600 '${clientKeyPath}/client.key' - ''; - }; - environment = { - variables = { - PGHOST = "db.test.example"; - PGPORT = "5432"; - PGDATABASE = "test"; - PGUSER = "test"; - PGSSLMODE = "verify-full"; - PGSSLCERT = clientCert; - PGSSLKEY = "${clientKeyPath}/client.key"; - PGSSLROOTCERT = caCert; + nodes.client = + { ... }: + { + system.activationScripts = { + keyPlacement.text = '' + mkdir -p '${clientKeyPath}' + cp '${clientKey}' '${clientKeyPath}/client.key' + chown root:root '${clientKeyPath}/client.key' + chmod 600 '${clientKeyPath}/client.key' + ''; }; - systemPackages = [ package ]; - }; - networking = { - interfaces.eth1 = { - ipv6.addresses = [ - { address = "fc00::2"; prefixLength = 120; } - ]; + environment = { + variables = { + PGHOST = "db.test.example"; + PGPORT = "5432"; + PGDATABASE = "test"; + PGUSER = "test"; + PGSSLMODE = "verify-full"; + PGSSLCERT = clientCert; + PGSSLKEY = "${clientKeyPath}/client.key"; + PGSSLROOTCERT = caCert; + }; + systemPackages = [ package ]; + }; + networking = { + interfaces.eth1 = { + ipv6.addresses = [ + { + address = "fc00::2"; + prefixLength = 120; + } + ]; + }; + hosts = { + "fc00::1" = [ "db.test.example" ]; + }; }; - hosts = { "fc00::1" = [ "db.test.example" ]; }; }; - }; testScript = '' server.wait_for_unit("multi-user.target") diff --git a/nixos/tests/postgresql/postgresql-wal-receiver.nix b/nixos/tests/postgresql/postgresql-wal-receiver.nix index b9d69cf00684d..5c1551c5f2fd2 100644 --- a/nixos/tests/postgresql/postgresql-wal-receiver.nix +++ b/nixos/tests/postgresql/postgresql-wal-receiver.nix @@ -1,11 +1,13 @@ -{ pkgs -, makeTest +{ + pkgs, + makeTest, }: let inherit (pkgs) lib; - makeTestFor = package: + makeTestFor = + package: let postgresqlDataDir = "/var/lib/postgresql/${package.psqlSchema}"; replicationUser = "wal_receiver_user"; @@ -19,41 +21,43 @@ let name = "postgresql-wal-receiver-${package.name}"; meta.maintainers = with lib.maintainers; [ pacien ]; - nodes.machine = { ... }: { - systemd.tmpfiles.rules = [ - "d /var/cache/wals 0750 postgres postgres - -" - ]; + nodes.machine = + { ... }: + { + systemd.tmpfiles.rules = [ + "d /var/cache/wals 0750 postgres postgres - -" + ]; - services.postgresql = { - inherit package; - enable = true; - settings = { - max_replication_slots = 10; - max_wal_senders = 10; - recovery_end_command = "touch recovery.done"; - restore_command = "cp ${walBackupDir}/%f %p"; - wal_level = "archive"; # alias for replica on pg >= 9.6 + services.postgresql = { + inherit package; + enable = true; + settings = { + max_replication_slots = 10; + max_wal_senders = 10; + recovery_end_command = "touch recovery.done"; + restore_command = "cp ${walBackupDir}/%f %p"; + wal_level = "archive"; # alias for replica on pg >= 9.6 + }; + authentication = '' + host replication ${replicationUser} all trust + ''; + initialScript = pkgs.writeText "init.sql" '' + create user ${replicationUser} replication; + select * from pg_create_physical_replication_slot('${replicationSlot}'); + ''; }; - authentication = '' - host replication ${replicationUser} all trust - ''; - initialScript = pkgs.writeText "init.sql" '' - create user ${replicationUser} replication; - select * from pg_create_physical_replication_slot('${replicationSlot}'); - ''; - }; - services.postgresqlWalReceiver.receivers.main = { - postgresqlPackage = package; - connection = replicationConn; - slot = replicationSlot; - directory = walBackupDir; + services.postgresqlWalReceiver.receivers.main = { + postgresqlPackage = package; + connection = replicationConn; + slot = replicationSlot; + directory = walBackupDir; + }; + # This is only to speedup test, it isn't time racing. Service is set to autorestart always, + # default 60sec is fine for real system, but is too much for a test + systemd.services.postgresql-wal-receiver-main.serviceConfig.RestartSec = lib.mkForce 5; + systemd.services.postgresql.serviceConfig.ReadWritePaths = [ "/var/cache/wals" ]; }; - # This is only to speedup test, it isn't time racing. Service is set to autorestart always, - # default 60sec is fine for real system, but is too much for a test - systemd.services.postgresql-wal-receiver-main.serviceConfig.RestartSec = lib.mkForce 5; - systemd.services.postgresql.serviceConfig.ReadWritePaths = [ "/var/cache/wals" ]; - }; testScript = '' # make an initial base backup diff --git a/nixos/tests/postgresql/postgresql.nix b/nixos/tests/postgresql/postgresql.nix index 13a768d9a8a9a..509a14411de9c 100644 --- a/nixos/tests/postgresql/postgresql.nix +++ b/nixos/tests/postgresql/postgresql.nix @@ -1,11 +1,13 @@ -{ pkgs -, makeTest +{ + pkgs, + makeTest, }: let inherit (pkgs) lib; - makeTestFor = package: + makeTestFor = + package: lib.recurseIntoAttrs { postgresql = makeTestForWithBackupAll package false; postgresql-backup-all = makeTestForWithBackupAll package true; @@ -26,17 +28,19 @@ let INSERT INTO xmltest (doc) VALUES ('ok'); -- check if libxml2 enabled ''; - makeTestForWithBackupAll = package: backupAll: + makeTestForWithBackupAll = + package: backupAll: makeTest { name = "postgresql${lib.optionalString backupAll "-backup-all"}-${package.name}"; meta = with lib.maintainers; { maintainers = [ zagy ]; }; - nodes.machine = {...}: + nodes.machine = + { ... }: { services.postgresql = { - inherit (package); + inherit (package) ; enable = true; }; @@ -46,106 +50,110 @@ let }; }; - testScript = let - backupName = if backupAll then "all" else "postgres"; - backupService = if backupAll then "postgresqlBackup" else "postgresqlBackup-postgres"; - backupFileBase = "/var/backup/postgresql/${backupName}"; - in '' - def check_count(statement, lines): - return 'test $(sudo -u postgres psql postgres -tAc "{}"|wc -l) -eq {}'.format( - statement, lines - ) - - - machine.start() - machine.wait_for_unit("postgresql") - - with subtest("Postgresql is available just after unit start"): - machine.succeed( - "cat ${test-sql} | sudo -u postgres psql" - ) - - with subtest("Postgresql survives restart (bug #1735)"): - machine.shutdown() - import time - time.sleep(2) - machine.start() - machine.wait_for_unit("postgresql") - - machine.fail(check_count("SELECT * FROM sth;", 3)) - machine.succeed(check_count("SELECT * FROM sth;", 5)) - machine.fail(check_count("SELECT * FROM sth;", 4)) - machine.succeed(check_count("SELECT xpath('/test/text()', doc) FROM xmltest;", 1)) - - with subtest("Backup service works"): - machine.succeed( - "systemctl start ${backupService}.service", - "zcat ${backupFileBase}.sql.gz | grep 'ok'", - "ls -hal /var/backup/postgresql/ >/dev/console", - "stat -c '%a' ${backupFileBase}.sql.gz | grep 600", - ) - with subtest("Backup service removes prev files"): - machine.succeed( - # Create dummy prev files. - "touch ${backupFileBase}.prev.sql{,.gz,.zstd}", - "chown postgres:postgres ${backupFileBase}.prev.sql{,.gz,.zstd}", - - # Run backup. - "systemctl start ${backupService}.service", - "ls -hal /var/backup/postgresql/ >/dev/console", - - # Since nothing has changed in the database, the cur and prev files - # should match. - "zcat ${backupFileBase}.sql.gz | grep 'ok'", - "cmp ${backupFileBase}.sql.gz ${backupFileBase}.prev.sql.gz", - - # The prev files with unused suffix should be removed. - "[ ! -f '${backupFileBase}.prev.sql' ]", - "[ ! -f '${backupFileBase}.prev.sql.zstd' ]", - - # Both cur and prev file should only be accessible by the postgres user. - "stat -c '%a' ${backupFileBase}.sql.gz | grep 600", - "stat -c '%a' '${backupFileBase}.prev.sql.gz' | grep 600", - ) - with subtest("Backup service fails gracefully"): - # Sabotage the backup process - machine.succeed("rm /run/postgresql/.s.PGSQL.5432") - machine.fail( - "systemctl start ${backupService}.service", - ) - machine.succeed( - "ls -hal /var/backup/postgresql/ >/dev/console", - "zcat ${backupFileBase}.prev.sql.gz | grep 'ok'", - "stat ${backupFileBase}.in-progress.sql.gz", - ) - # In a previous version, the second run would overwrite prev.sql.gz, - # so we test a second run as well. - machine.fail( - "systemctl start ${backupService}.service", - ) - machine.succeed( - "stat ${backupFileBase}.in-progress.sql.gz", - "zcat ${backupFileBase}.prev.sql.gz | grep 'ok'", - ) - - - with subtest("Initdb works"): - machine.succeed("sudo -u postgres initdb -D /tmp/testpostgres2") - - machine.log(machine.execute("systemd-analyze security postgresql.service | grep -v ✓")[1]) - - machine.shutdown() - ''; + testScript = + let + backupName = if backupAll then "all" else "postgres"; + backupService = if backupAll then "postgresqlBackup" else "postgresqlBackup-postgres"; + backupFileBase = "/var/backup/postgresql/${backupName}"; + in + '' + def check_count(statement, lines): + return 'test $(sudo -u postgres psql postgres -tAc "{}"|wc -l) -eq {}'.format( + statement, lines + ) + + + machine.start() + machine.wait_for_unit("postgresql") + + with subtest("Postgresql is available just after unit start"): + machine.succeed( + "cat ${test-sql} | sudo -u postgres psql" + ) + + with subtest("Postgresql survives restart (bug #1735)"): + machine.shutdown() + import time + time.sleep(2) + machine.start() + machine.wait_for_unit("postgresql") + + machine.fail(check_count("SELECT * FROM sth;", 3)) + machine.succeed(check_count("SELECT * FROM sth;", 5)) + machine.fail(check_count("SELECT * FROM sth;", 4)) + machine.succeed(check_count("SELECT xpath('/test/text()', doc) FROM xmltest;", 1)) + + with subtest("Backup service works"): + machine.succeed( + "systemctl start ${backupService}.service", + "zcat ${backupFileBase}.sql.gz | grep 'ok'", + "ls -hal /var/backup/postgresql/ >/dev/console", + "stat -c '%a' ${backupFileBase}.sql.gz | grep 600", + ) + with subtest("Backup service removes prev files"): + machine.succeed( + # Create dummy prev files. + "touch ${backupFileBase}.prev.sql{,.gz,.zstd}", + "chown postgres:postgres ${backupFileBase}.prev.sql{,.gz,.zstd}", + + # Run backup. + "systemctl start ${backupService}.service", + "ls -hal /var/backup/postgresql/ >/dev/console", + + # Since nothing has changed in the database, the cur and prev files + # should match. + "zcat ${backupFileBase}.sql.gz | grep 'ok'", + "cmp ${backupFileBase}.sql.gz ${backupFileBase}.prev.sql.gz", + + # The prev files with unused suffix should be removed. + "[ ! -f '${backupFileBase}.prev.sql' ]", + "[ ! -f '${backupFileBase}.prev.sql.zstd' ]", + + # Both cur and prev file should only be accessible by the postgres user. + "stat -c '%a' ${backupFileBase}.sql.gz | grep 600", + "stat -c '%a' '${backupFileBase}.prev.sql.gz' | grep 600", + ) + with subtest("Backup service fails gracefully"): + # Sabotage the backup process + machine.succeed("rm /run/postgresql/.s.PGSQL.5432") + machine.fail( + "systemctl start ${backupService}.service", + ) + machine.succeed( + "ls -hal /var/backup/postgresql/ >/dev/console", + "zcat ${backupFileBase}.prev.sql.gz | grep 'ok'", + "stat ${backupFileBase}.in-progress.sql.gz", + ) + # In a previous version, the second run would overwrite prev.sql.gz, + # so we test a second run as well. + machine.fail( + "systemctl start ${backupService}.service", + ) + machine.succeed( + "stat ${backupFileBase}.in-progress.sql.gz", + "zcat ${backupFileBase}.prev.sql.gz | grep 'ok'", + ) + + + with subtest("Initdb works"): + machine.succeed("sudo -u postgres initdb -D /tmp/testpostgres2") + + machine.log(machine.execute("systemd-analyze security postgresql.service | grep -v ✓")[1]) + + machine.shutdown() + ''; }; - makeEnsureTestFor = package: + makeEnsureTestFor = + package: makeTest { name = "postgresql-clauses-${package.name}"; meta = with lib.maintainers; { maintainers = [ zagy ]; }; - nodes.machine = {...}: + nodes.machine = + { ... }: { services.postgresql = { inherit package; @@ -170,12 +178,14 @@ let }; }; - testScript = let - getClausesQuery = user: lib.concatStringsSep " " - [ - "SELECT row_to_json(row)" - "FROM (" - "SELECT" + testScript = + let + getClausesQuery = + user: + lib.concatStringsSep " " [ + "SELECT row_to_json(row)" + "FROM (" + "SELECT" "rolsuper," "rolinherit," "rolcreaterole," @@ -183,46 +193,47 @@ let "rolcanlogin," "rolreplication," "rolbypassrls" - "FROM pg_roles" - "WHERE rolname = '${user}'" - ") row;" - ]; - in '' - import json - machine.start() - machine.wait_for_unit("postgresql") - - with subtest("All user permissions are set according to the ensureClauses attr"): - clauses = json.loads( - machine.succeed( - "sudo -u postgres psql -tc \"${getClausesQuery "all-clauses"}\"" + "FROM pg_roles" + "WHERE rolname = '${user}'" + ") row;" + ]; + in + '' + import json + machine.start() + machine.wait_for_unit("postgresql") + + with subtest("All user permissions are set according to the ensureClauses attr"): + clauses = json.loads( + machine.succeed( + "sudo -u postgres psql -tc \"${getClausesQuery "all-clauses"}\"" + ) ) - ) - print(clauses) - assert clauses['rolsuper'], 'expected user with clauses to have superuser clause' - assert clauses['rolinherit'], 'expected user with clauses to have inherit clause' - assert clauses['rolcreaterole'], 'expected user with clauses to have create role clause' - assert clauses['rolcreatedb'], 'expected user with clauses to have create db clause' - assert clauses['rolcanlogin'], 'expected user with clauses to have login clause' - assert clauses['rolreplication'], 'expected user with clauses to have replication clause' - assert clauses['rolbypassrls'], 'expected user with clauses to have bypassrls clause' - - with subtest("All user permissions default when ensureClauses is not provided"): - clauses = json.loads( - machine.succeed( - "sudo -u postgres psql -tc \"${getClausesQuery "default-clauses"}\"" + print(clauses) + assert clauses['rolsuper'], 'expected user with clauses to have superuser clause' + assert clauses['rolinherit'], 'expected user with clauses to have inherit clause' + assert clauses['rolcreaterole'], 'expected user with clauses to have create role clause' + assert clauses['rolcreatedb'], 'expected user with clauses to have create db clause' + assert clauses['rolcanlogin'], 'expected user with clauses to have login clause' + assert clauses['rolreplication'], 'expected user with clauses to have replication clause' + assert clauses['rolbypassrls'], 'expected user with clauses to have bypassrls clause' + + with subtest("All user permissions default when ensureClauses is not provided"): + clauses = json.loads( + machine.succeed( + "sudo -u postgres psql -tc \"${getClausesQuery "default-clauses"}\"" + ) ) - ) - assert not clauses['rolsuper'], 'expected user with no clauses set to have default superuser clause' - assert clauses['rolinherit'], 'expected user with no clauses set to have default inherit clause' - assert not clauses['rolcreaterole'], 'expected user with no clauses set to have default create role clause' - assert not clauses['rolcreatedb'], 'expected user with no clauses set to have default create db clause' - assert clauses['rolcanlogin'], 'expected user with no clauses set to have default login clause' - assert not clauses['rolreplication'], 'expected user with no clauses set to have default replication clause' - assert not clauses['rolbypassrls'], 'expected user with no clauses set to have default bypassrls clause' - - machine.shutdown() - ''; + assert not clauses['rolsuper'], 'expected user with no clauses set to have default superuser clause' + assert clauses['rolinherit'], 'expected user with no clauses set to have default inherit clause' + assert not clauses['rolcreaterole'], 'expected user with no clauses set to have default create role clause' + assert not clauses['rolcreatedb'], 'expected user with no clauses set to have default create db clause' + assert clauses['rolcanlogin'], 'expected user with no clauses set to have default login clause' + assert not clauses['rolreplication'], 'expected user with no clauses set to have default replication clause' + assert not clauses['rolbypassrls'], 'expected user with no clauses set to have default bypassrls clause' + + machine.shutdown() + ''; }; in lib.recurseIntoAttrs ( diff --git a/nixos/tests/postgresql/timescaledb.nix b/nixos/tests/postgresql/timescaledb.nix index a01f890a12c6c..b29d59c744f06 100644 --- a/nixos/tests/postgresql/timescaledb.nix +++ b/nixos/tests/postgresql/timescaledb.nix @@ -1,5 +1,6 @@ -{ pkgs -, makeTest +{ + pkgs, + makeTest, }: let @@ -38,23 +39,28 @@ let SELECT * FROM sth; ''; - makeTestFor = package: + makeTestFor = + package: makeTest { name = "timescaledb-${package.name}"; meta = with lib.maintainers; { maintainers = [ typetetris ]; }; - nodes.machine = { ... }: + nodes.machine = + { ... }: { services.postgresql = { inherit package; enable = true; - extraPlugins = ps: with ps; [ - timescaledb - timescaledb_toolkit - ]; - settings = { shared_preload_libraries = "timescaledb, timescaledb_toolkit"; }; + extraPlugins = + ps: with ps; [ + timescaledb + timescaledb_toolkit + ]; + settings = { + shared_preload_libraries = "timescaledb, timescaledb_toolkit"; + }; }; }; @@ -85,7 +91,9 @@ in # To run these tests: # NIXPKGS_ALLOW_UNFREE=1 nix-build -A nixosTests.postgresql.timescaledb lib.dontRecurseIntoAttrs ( - lib.concatMapAttrs (n: p: { ${n} = makeTestFor p; }) (lib.filterAttrs (_: p: !p.pkgs.timescaledb.meta.broken) pkgs.postgresqlVersions) + lib.concatMapAttrs (n: p: { ${n} = makeTestFor p; }) ( + lib.filterAttrs (_: p: !p.pkgs.timescaledb.meta.broken) pkgs.postgresqlVersions + ) // { passthru.override = p: makeTestFor p; } diff --git a/nixos/tests/postgresql/tsja.nix b/nixos/tests/postgresql/tsja.nix index 8d78093136df7..7c976da21b68f 100644 --- a/nixos/tests/postgresql/tsja.nix +++ b/nixos/tests/postgresql/tsja.nix @@ -1,25 +1,29 @@ -{ pkgs -, makeTest +{ + pkgs, + makeTest, }: let inherit (pkgs) lib; - makeTestFor = package: + makeTestFor = + package: makeTest { name = "tsja-${package.name}"; meta = { maintainers = with lib.maintainers; [ chayleaf ]; }; - nodes.master = { ... }: + nodes.master = + { ... }: { services.postgresql = { inherit package; enable = true; - extraPlugins = ps: with ps; [ - tsja - ]; + extraPlugins = + ps: with ps; [ + tsja + ]; }; }; @@ -37,7 +41,9 @@ let }; in lib.recurseIntoAttrs ( - lib.concatMapAttrs (n: p: { ${n} = makeTestFor p; }) (lib.filterAttrs (_: p: !p.pkgs.tsja.meta.broken) pkgs.postgresqlVersions) + lib.concatMapAttrs (n: p: { ${n} = makeTestFor p; }) ( + lib.filterAttrs (_: p: !p.pkgs.tsja.meta.broken) pkgs.postgresqlVersions + ) // { passthru.override = p: makeTestFor p; } diff --git a/nixos/tests/postgresql/wal2json.nix b/nixos/tests/postgresql/wal2json.nix index 0333369722f0c..551254a68ebda 100644 --- a/nixos/tests/postgresql/wal2json.nix +++ b/nixos/tests/postgresql/wal2json.nix @@ -1,11 +1,13 @@ -{ pkgs -, makeTest +{ + pkgs, + makeTest, }: let inherit (pkgs) lib; - makeTestFor = package: + makeTestFor = + package: makeTest { name = "wal2json-${package.name}"; meta.maintainers = with pkgs.lib.maintainers; [ euank ]; @@ -41,7 +43,9 @@ let }; in lib.recurseIntoAttrs ( - lib.concatMapAttrs (n: p: { ${n} = makeTestFor p; }) (lib.filterAttrs (_: p: !p.pkgs.wal2json.meta.broken) pkgs.postgresqlVersions) + lib.concatMapAttrs (n: p: { ${n} = makeTestFor p; }) ( + lib.filterAttrs (_: p: !p.pkgs.wal2json.meta.broken) pkgs.postgresqlVersions + ) // { passthru.override = p: makeTestFor p; }