Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

postgresql: Remove dev dependencies from closure #179962

Closed
wants to merge 8 commits into from
8 changes: 3 additions & 5 deletions doc/languages-frameworks/ruby.section.md
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,7 @@ let
defaultGemConfig = pkgs.defaultGemConfig // {
pg = attrs: {
buildFlags =
[ "--with-pg-config=${pkgs."postgresql_${pg_version}"}/bin/pg_config" ];
[ "--with-pg-config=${lib.getDev pkgs."postgresql_${pg_version}"}/bin/pg_config" ];
};
};
};
Expand All @@ -170,7 +170,7 @@ let
gemConfig = pkgs.defaultGemConfig // {
pg = attrs: {
buildFlags =
[ "--with-pg-config=${pkgs."postgresql_${pg_version}"}/bin/pg_config" ];
[ "--with-pg-config=${lib.getDev pkgs."postgresql_${pg_version}"}/bin/pg_config" ];
};
};
};
Expand All @@ -188,9 +188,7 @@ let
defaultGemConfig = super.defaultGemConfig // {
pg = attrs: {
buildFlags = [
"--with-pg-config=${
pkgs."postgresql_${pg_version}"
}/bin/pg_config"
"--with-pg-config=${lib.getDev pkgs."postgresql_${pg_version}"}/bin/pg_config"
];
};
};
Expand Down
2 changes: 1 addition & 1 deletion pkgs/applications/networking/misc/zammad/default.nix
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ let
];
gemConfig = defaultGemConfig // {
pg = attrs: {
buildFlags = [ "--with-pg-config=${postgresql}/bin/pg_config" ];
buildFlags = [ "--with-pg-config=${lib.getDev postgresql}/bin/pg_config" ];
};
rszr = attrs: {
buildInputs = [ imlib2 imlib2.dev ];
Expand Down
11 changes: 9 additions & 2 deletions pkgs/development/python-modules/psycopg/default.nix
Original file line number Diff line number Diff line change
Expand Up @@ -82,11 +82,18 @@ let

nativeBuildInputs = [
cython_3
# needed to find pg_config with strictDeps
# TODO: switch to pkg-config upstream
# https://github.com/psycopg/psycopg2/issues/1001
postgresql
setuptools
tomli
];

buildInputs = [
postgresql
];
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you elaborate why this is now needed?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is just a proper place for this as a library dependency. Ideally, we would remove it from nativeBuildInputs but that will probably not work until psycopg/psycopg2#1001. cc @SuperSandro2000 who made this change.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, IIRC they use pg_config

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Added a comment.


# tested in psycopg
doCheck = false;

Expand Down Expand Up @@ -181,7 +188,7 @@ buildPythonPackage rec {
pytestCheckHook
postgresql
]
++ lib.optional (stdenv.isLinux) postgresqlTestHook
++ lib.optional stdenv.isLinux postgresqlTestHook
++ passthru.optional-dependencies.c
++ passthru.optional-dependencies.pool;

Expand All @@ -193,7 +200,7 @@ buildPythonPackage rec {

preCheck = ''
cd ..
'' + lib.optionalString (stdenv.isLinux) ''
'' + lib.optionalString stdenv.isLinux ''
export PSYCOPG_TEST_DSN="host=/build/run/postgresql user=$PGUSER"
'';

Expand Down
2 changes: 1 addition & 1 deletion pkgs/development/python-modules/psycopg2/default.nix
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ buildPythonPackage rec {
# some linker flags are added but the linker ignores them because they're incompatible
# https://github.com/psycopg/psycopg2/blob/89005ac5b849c6428c05660b23c5a266c96e677d/setup.py
substituteInPlace setup.py \
--replace "self.pg_config_exe = self.build_ext.pg_config" 'self.pg_config_exe = "${lib.getExe' buildPackages.postgresql "pg_config"}"'
--replace "self.pg_config_exe = self.build_ext.pg_config" 'self.pg_config_exe = "${lib.getDev buildPackages.postgresql}/bin/pg_config"'
'';

nativeBuildInputs = [
Expand Down
12 changes: 11 additions & 1 deletion pkgs/development/python-modules/psycopg2cffi/default.nix
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,17 @@ buildPythonPackage rec {
sha256 = "09hsnjkix1c0vlhmfvrp8pchpnz2ya4xrchyq15czj527nx2dmy2";
};

nativeBuildInputs = [ postgresql ];
nativeBuildInputs = [
# needed to find pg_config with strictDeps
# TODO: switch to pkg-config upstream
# https://github.com/psycopg/psycopg2/issues/1001
postgresql
];

buildInputs = [
postgresql
];

propagatedBuildInputs = [ six cffi ];
nativeCheckInputs = [ pytestCheckHook ];

Expand Down
2 changes: 1 addition & 1 deletion pkgs/development/ruby-modules/gem-config/default.nix
Original file line number Diff line number Diff line change
Expand Up @@ -624,7 +624,7 @@ in
# Force pkg-config lookup for libpq.
# See https://github.com/ged/ruby-pg/blob/6629dec6656f7ca27619e4675b45225d9e422112/ext/extconf.rb#L34-L55
#
# Note that setting --with-pg-config=${postgresql}/bin/pg_config would add
# Note that setting --with-pg-config=${lib.getDev postgresql}/bin/pg_config would add
# an unnecessary reference to the entire postgresql package.
buildFlags = [ "--with-pg-config=ignore" ];
nativeBuildInputs = [ pkg-config ];
Expand Down
4 changes: 2 additions & 2 deletions pkgs/development/tools/rust/cargo-pgrx/buildPgrxExtension.nix
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ let
preBuildAndTest = ''
export PGRX_HOME=$(mktemp -d)
export PGDATA="$PGRX_HOME/data-${pgrxPostgresMajor}/"
cargo-pgrx pgrx init "--pg${pgrxPostgresMajor}" ${postgresql}/bin/pg_config
cargo-pgrx pgrx init "--pg${pgrxPostgresMajor}" ${lib.getDev postgresql}/bin/pg_config
echo "unix_socket_directories = '$(mktemp -d)'" > "$PGDATA/postgresql.conf"

# This is primarily for Mac or other Nix systems that don't use the nixbld user.
Expand Down Expand Up @@ -120,7 +120,7 @@ let
NIX_PGLIBDIR="${postgresql}/lib" \
PGRX_BUILD_FLAGS="--frozen -j $NIX_BUILD_CORES ${builtins.concatStringsSep " " cargoBuildFlags}" \
cargo-pgrx pgrx package \
--pg-config ${postgresql}/bin/pg_config \
--pg-config ${lib.getDev postgresql}/bin/pg_config \
${maybeDebugFlag} \
--features "${builtins.concatStringsSep " " buildFeatures}" \
--out-dir "$out"
Expand Down
4 changes: 2 additions & 2 deletions pkgs/development/tools/rust/cargo-pgx/buildPgxExtension.nix
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ let
preBuildAndTest = ''
export PGX_HOME=$(mktemp -d)
export PGDATA="$PGX_HOME/data-${pgxPostgresMajor}/"
cargo-pgx pgx init "--pg${pgxPostgresMajor}" ${postgresql}/bin/pg_config
cargo-pgx pgx init "--pg${pgxPostgresMajor}" ${lib.getDev postgresql}/bin/pg_config
echo "unix_socket_directories = '$(mktemp -d)'" > "$PGDATA/postgresql.conf"

# This is primarily for Mac or other Nix systems that don't use the nixbld user.
Expand Down Expand Up @@ -120,7 +120,7 @@ let
NIX_PGLIBDIR="${postgresql}/lib" \
PGX_BUILD_FLAGS="--frozen -j $NIX_BUILD_CORES ${builtins.concatStringsSep " " cargoBuildFlags}" \
cargo-pgx pgx package \
--pg-config ${postgresql}/bin/pg_config \
--pg-config ${lib.getDev postgresql}/bin/pg_config \
${maybeDebugFlag} \
--features "${builtins.concatStringsSep " " buildFeatures}" \
--out-dir "$out"
Expand Down
50 changes: 43 additions & 7 deletions pkgs/servers/sql/postgresql/default.nix
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,13 @@ let
generic =
# dependencies
{ stdenv, lib, fetchurl, makeWrapper, fetchpatch
, autoreconfHook269
, perl
, docbook_xml_dtd_45
, docbook_xml_dtd_42
, libxslt
, docbook-xsl-nons
, runCommand
, glibc, zlib, readline, openssl, icu, lz4, zstd, systemd, libossp_uuid
, pkg-config, libxml2, tzdata, libkrb5, substituteAll, darwin

Expand Down Expand Up @@ -47,8 +54,7 @@ let

hardeningEnable = lib.optionals (!stdenv'.cc.isClang) [ "pie" ];

outputs = [ "out" "lib" "doc" "man" ];
setOutputFlags = false; # $out retains configureFlags :-/
outputs = [ "out" "dev" "lib" "doc" "man" ];

buildInputs = [
zlib
Expand All @@ -68,6 +74,12 @@ let
nativeBuildInputs = [
makeWrapper
pkg-config
# We are patching configure.ac
autoreconfHook269
perl
(if atLeast "13" then docbook_xml_dtd_45 else docbook_xml_dtd_42)
libxslt
docbook-xsl-nons
]
++ lib.optionals jitSupport [ llvmPackages.llvm.dev nukeReferences patchelf ];

Expand All @@ -87,7 +99,6 @@ let
"--with-libxml"
"--with-icu"
"--sysconfdir=/etc"
"--libdir=$(lib)/lib"
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm probably missing something, but how are .so files moved to the lib output now?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We previously passed setOutputFlags = false; so we had to set it explicitly. Now that I removed it, multiple-outputs hook passes this for us.

"--with-system-tzdata=${tzdata}/share/zoneinfo"
"--enable-debug"
(lib.optionalString enableSystemd "--with-systemd")
Expand All @@ -102,7 +113,12 @@ let
(if atLeast "16" then ./patches/disable-normalize_exec_path.patch
else ./patches/disable-resolve_symlinks.patch)
./patches/less-is-more.patch
./patches/hardcode-pgxs-path.patch

# Hardcode the path to pgxs and other dirs so that pg_config returns the path in the package,
# rather than one relative to the location pg_config was executed in.
# The placeholders are resolved in postPatch.
./patches/hardcode-pg_config-paths.patch
Ma27 marked this conversation as resolved.
Show resolved Hide resolved

./patches/specify_pkglibdir_at_runtime.patch
./patches/findstring.patch

Expand All @@ -111,6 +127,23 @@ let
locale = "${if stdenv.isDarwin then darwin.adv_cmds else lib.getBin stdenv.cc.libc}/bin/locale";
})

# Remove references to output paths from configure flags
# recorded in the software.
(if atLeast "14" then ./patches/remove-refs-from-configure-flags.patch
else if atLeast "13" then
runCommand
"remove-refs-from-configure-flags.patch"
{
patch = ./patches/remove-refs-from-configure-flags.patch;
}
''
substitute "$patch" "$out" --replace "configure.ac" "configure.in"
''
else ./patches/remove-refs-from-configure-flags-upto-12.patch)

# Patch out includedir path references in libraries and programs.
./patches/prevent-output-cycle.patch

] ++ lib.optionals stdenv'.hostPlatform.isMusl (
let
self = {
Expand Down Expand Up @@ -163,8 +196,9 @@ let
LC_ALL = "C";

postPatch = ''
# Hardcode the path to pgxs so pg_config returns the path in $out
substituteInPlace "src/common/config_info.c" --replace HARDCODED_PGXS_PATH "$out/lib"
# Substitute placeholders from hardcode-pg_config-paths.patch
substituteInPlace "src/common/config_info.c" --subst-var out
substituteInPlace "src/bin/pg_config/pg_config.c" --subst-var dev
'' + lib.optionalString jitSupport ''
# Force lookup of jit stuff in $out instead of $lib
substituteInPlace src/backend/jit/jit.c --replace pkglib_path \"$out/lib\"
Expand All @@ -178,9 +212,11 @@ let
moveToOutput "lib/libpgcommon*.a" "$out"
moveToOutput "lib/libpgport*.a" "$out"
moveToOutput "lib/libecpg*" "$out"
moveToOutput "bin/pg_config" "$dev"
moveToOutput "lib/pgxs" "$dev"

# Prevent a retained dependency on gcc-wrapper.
substituteInPlace "$out/lib/pgxs/src/Makefile.global" --replace ${stdenv'.cc}/bin/ld ld
substituteInPlace "$dev/lib/pgxs/src/Makefile.global" --replace ${stdenv'.cc}/bin/ld ld

if [ -z "''${dontDisableStatic:-}" ]; then
# Remove static libraries in case dynamic are available.
Expand Down
70 changes: 70 additions & 0 deletions pkgs/servers/sql/postgresql/patches/hardcode-pg_config-paths.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
diff --git a/src/bin/pg_config/pg_config.c b/src/bin/pg_config/pg_config.c
index 62b97af..e5b71e5 100644
--- a/src/bin/pg_config/pg_config.c
+++ b/src/bin/pg_config/pg_config.c
@@ -112,6 +112,11 @@ advice(void)
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
}

+static char*
+get_config_val(ConfigData configdata) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Also why is this passed by reference here?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It is passed by value, is not it? We could pass a pointer to it if we want to avoid copying the whole struct but I do not think this is worth it – the functions will probably only be called at extension build time and compiler might inline the function anyway.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

meh, I mixed them up, I meant pass-by-value and wanted to ask why 🙃

+ return strcmp(configdata.name, "PGXS") == 0 ? "@dev@/lib/pgxs/src/makefiles/pgxs.mk" : configdata.setting;
+}
+
static void
show_item(const char *configname,
ConfigData *configdata,
@@ -122,7 +127,7 @@ show_item(const char *configname,
for (i = 0; i < configdata_len; i++)
{
if (strcmp(configname, configdata[i].name) == 0)
- printf("%s\n", configdata[i].setting);
+ printf("%s\n", get_config_val(configdata[i]));
}
}

@@ -160,7 +165,7 @@ main(int argc, char **argv)
if (argc < 2)
{
for (i = 0; i < configdata_len; i++)
- printf("%s = %s\n", configdata[i].name, configdata[i].setting);
+ printf("%s = %s\n", configdata[i].name, get_config_val(configdata[i]));
exit(0);
}

diff --git a/src/common/config_info.c b/src/common/config_info.c
index aa643b6..3750c14 100644
--- a/src/common/config_info.c
+++ b/src/common/config_info.c
@@ -42,11 +42,7 @@ get_configdata(const char *my_exec_path, size_t *configdata_len)
configdata = (ConfigData *) palloc(*configdata_len * sizeof(ConfigData));

configdata[i].name = pstrdup("BINDIR");
- strlcpy(path, my_exec_path, sizeof(path));
- lastsep = strrchr(path, '/');
- if (lastsep)
- *lastsep = '\0';
- cleanup_path(path);
+ strlcpy(path, "@out@/bin", sizeof(path));
configdata[i].setting = pstrdup(path);
i++;

@@ -105,8 +101,7 @@ get_configdata(const char *my_exec_path, size_t *configdata_len)
i++;

configdata[i].name = pstrdup("SHAREDIR");
- get_share_path(my_exec_path, path);
- cleanup_path(path);
+ strlcpy(path, "@out@/share", sizeof(path));
configdata[i].setting = pstrdup(path);
i++;

@@ -117,7 +112,7 @@ get_configdata(const char *my_exec_path, size_t *configdata_len)
i++;

configdata[i].name = pstrdup("PGXS");
- get_pkglib_path(my_exec_path, path);
+ strlcpy(path, "/dev/null", sizeof(path));
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What's the rationale in here to make this /dev/null first and then replace that with @dev@?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

IIRC pg_config.c is installed to dev output but config_info.c ends up as a part of out.

strlcat(path, "/pgxs/src/makefiles/pgxs.mk", sizeof(path));
cleanup_path(path);
configdata[i].setting = pstrdup(path);
14 changes: 0 additions & 14 deletions pkgs/servers/sql/postgresql/patches/hardcode-pgxs-path.patch

This file was deleted.

17 changes: 17 additions & 0 deletions pkgs/servers/sql/postgresql/patches/prevent-output-cycle.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
diff --git a/src/port/Makefile b/src/port/Makefile
index bfe1feb..1f9f0d3 100644
--- a/src/port/Makefile
+++ b/src/port/Makefile
@@ -152,9 +152,9 @@ pg_config_paths.h: $(top_builddir)/src/Makefile.global
echo "#define PGBINDIR \"$(bindir)\"" >$@
jtojnar marked this conversation as resolved.
Show resolved Hide resolved
echo "#define PGSHAREDIR \"$(datadir)\"" >>$@
echo "#define SYSCONFDIR \"$(sysconfdir)\"" >>$@
- echo "#define INCLUDEDIR \"$(includedir)\"" >>$@
- echo "#define PKGINCLUDEDIR \"$(pkgincludedir)\"" >>$@
- echo "#define INCLUDEDIRSERVER \"$(includedir_server)\"" >>$@
+ echo "#define INCLUDEDIR \"/dev/null/include\"" >>$@
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Doesn't this break with e.g. https://github.com/postgres/postgres/blob/aa210e0c121eb8f58c86d4fcc833a5a6fbb6f5a9/src/interfaces/ecpg/preproc/ecpg.c#L256 since get_include_path uses INCLUDEDIR generated by this makefile?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It will return the wrong value but the assumption is that nothing uses the function for anything other than printing it in some build information pages.

+ echo "#define PKGINCLUDEDIR \"/dev/null/include\"" >>$@
+ echo "#define INCLUDEDIRSERVER \"/dev/null/include/server\"" >>$@
echo "#define LIBDIR \"$(libdir)\"" >>$@
echo "#define PKGLIBDIR \"$(pkglibdir)\"" >>$@
echo "#define LOCALEDIR \"$(localedir)\"" >>$@
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We could shave 16M more by removing man and doc output references.

Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
--- a/src/Makefile.global.in
+++ b/src/Makefile.global.in
@@ -75,7 +75,7 @@ endif # not PGXS
vpathsearch = `for f in $(addsuffix /$(1),$(subst :, ,. $(VPATH))); do test -r $$f && echo $$f && break; done`

# Saved arguments from configure
-configure_args = @configure_args@
+configure_args = $(shell echo '@configure_args@' | sed -E 's~/nix/store/.{32}-~/nix/store/eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee-~g')


##########################################################################
diff --git a/src/common/Makefile b/src/common/Makefile
index 873fbb6..b6476b4 100644
--- a/src/common/Makefile
+++ b/src/common/Makefile
@@ -29,13 +29,13 @@ STD_CPPFLAGS := $(filter-out -I$(top_srcdir)/src/include -I$(top_builddir)/src/i
STD_LDFLAGS := $(filter-out -L$(top_builddir)/src/common -L$(top_builddir)/src/port,$(LDFLAGS))
override CPPFLAGS += -DVAL_CONFIGURE="\"$(configure_args)\""
override CPPFLAGS += -DVAL_CC="\"$(CC)\""
-override CPPFLAGS += -DVAL_CPPFLAGS="\"$(STD_CPPFLAGS)\""
-override CPPFLAGS += -DVAL_CFLAGS="\"$(CFLAGS)\""
-override CPPFLAGS += -DVAL_CFLAGS_SL="\"$(CFLAGS_SL)\""
-override CPPFLAGS += -DVAL_LDFLAGS="\"$(STD_LDFLAGS)\""
-override CPPFLAGS += -DVAL_LDFLAGS_EX="\"$(LDFLAGS_EX)\""
-override CPPFLAGS += -DVAL_LDFLAGS_SL="\"$(LDFLAGS_SL)\""
-override CPPFLAGS += -DVAL_LIBS="\"$(LIBS)\""
+override CPPFLAGS += -DVAL_CPPFLAGS="\"$(shell echo '$(STD_CPPFLAGS)' | sed -E 's~/nix/store/.{32}-~/nix/store/eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee-~g')\""
+override CPPFLAGS += -DVAL_CFLAGS="\"$(shell echo '$(CFLAGS)' | sed -E 's~/nix/store/.{32}-~/nix/store/eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee-~g')\""
+override CPPFLAGS += -DVAL_CFLAGS_SL="\"$(shell echo '$(CFLAGS_SL)' | sed -E 's~/nix/store/.{32}-~/nix/store/eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee-~g')\""
+override CPPFLAGS += -DVAL_LDFLAGS="\"$(shell echo '$(STD_LDFLAGS)' | sed -E 's~/nix/store/.{32}-~/nix/store/eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee-~g')\""
+override CPPFLAGS += -DVAL_LDFLAGS_EX="\"$(shell echo '$(LDFLAGS_EX)' | sed -E 's~/nix/store/.{32}-~/nix/store/eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee-~g')\""
+override CPPFLAGS += -DVAL_LDFLAGS_SL="\"$(shell echo '$(LDFLAGS_SL)' | sed -E 's~/nix/store/.{32}-~/nix/store/eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee-~g')\""
+override CPPFLAGS += -DVAL_LIBS="\"$(shell echo '$(LIBS)' | sed -E 's~/nix/store/.{32}-~/nix/store/eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee-~g')\""

override CPPFLAGS := -DFRONTEND $(CPPFLAGS)
LIBS += $(PTHREAD_LIBS)
Loading