enable libpsl by default

- add manual build for libpsl. It's shorter than calling up autotools, which is broken in curl-for-win and also has an libpsl upstream issue on Windows. It's more fragile and may break by upstream changes. - retrofit curl patch for a clean libpsl version string: curl/curl@72bd88a curl/curl#12700 - exclude libpsl from `-zero` and `-bldtst` builds. Closes #63
curl · Jan 15, 2024 · c9e2401 · c9e2401
1 parent 9601f1f
commit c9e2401
Show file tree

Hide file tree

Showing 8 changed files with 53 additions and 39 deletions.
diff --git a/README.md b/README.md
@@ -61,38 +61,38 @@ Uses [LibreSSL](https://www.libressl.org/) TLS backend.
 ```
 Windows with runtime-selectable option Schannel:
 Protocols: dict file ftp ftps gopher gophers http https imap imaps ipfs ipns ldap ldaps mqtt pop3 pop3s rtsp scp sftp smb smbs smtp smtps telnet tftp ws wss
-Features: alt-svc AsynchDNS brotli       HSTS HTTP2 HTTP3 HTTPS-proxy IDN IPv6 Kerberos Largefile libz MultiSSL NTLM     SPNEGO SSL SSPI threadsafe UnixSockets zstd
+Features: alt-svc AsynchDNS brotli HSTS HTTP2 HTTP3 HTTPS-proxy IDN IPv6 Kerberos Largefile libz MultiSSL NTLM PSL SPNEGO SSL SSPI threadsafe UnixSockets zstd
 
 macOS with runtime-selectable option SecureTransport:
 Protocols: dict file ftp ftps gopher gophers http https imap imaps ipfs ipns ldap ldaps mqtt pop3 pop3s rtsp scp sftp smb smbs smtp smtps telnet tftp ws wss
-Features: alt-svc AsynchDNS brotli       HSTS HTTP2 HTTP3 HTTPS-proxy     IPv6          Largefile libz MultiSSL NTLM            SSL      threadsafe UnixSockets zstd
+Features: alt-svc AsynchDNS brotli HSTS HTTP2 HTTP3 HTTPS-proxy     IPv6          Largefile libz MultiSSL NTLM PSL        SSL      threadsafe UnixSockets zstd
 
 Linux:
 Protocols: dict file ftp ftps gopher gophers http https imap imaps ipfs ipns            mqtt pop3 pop3s rtsp scp sftp smb smbs smtp smtps telnet tftp ws wss
-Features: alt-svc AsynchDNS brotli       HSTS HTTP2 HTTP3 HTTPS-proxy     IPv6          Largefile libz          NTLM            SSL      threadsafe UnixSockets zstd
+Features: alt-svc AsynchDNS brotli HSTS HTTP2 HTTP3 HTTPS-proxy     IPv6          Largefile libz          NTLM PSL        SSL      threadsafe UnixSockets zstd
 ```
 <details><summary>Alternate configurations:</summary><p>
 
 ```
 "noh3", HTTP/2:
 Protocols: dict file ftp ftps gopher gophers http https imap imaps ipfs ipns ldap ldaps mqtt pop3 pop3s rtsp scp sftp smb smbs smtp smtps telnet tftp ws wss
-Features: alt-svc AsynchDNS brotli       HSTS HTTP2       HTTPS-proxy IDN IPv6 Kerberos Largefile libz MultiSSL NTLM     SPNEGO SSL SSPI threadsafe UnixSockets zstd
+Features: alt-svc AsynchDNS brotli HSTS HTTP2       HTTPS-proxy IDN IPv6 Kerberos Largefile libz MultiSSL NTLM PSL SPNEGO SSL SSPI threadsafe UnixSockets zstd
 
 "mini", without brotli and zstd, with OS TLS backend (Schannel, SecureTransport) if available:
 Protocols: dict file ftp ftps gopher gophers http https imap imaps ipfs ipns ldap ldaps mqtt pop3 pop3s rtsp scp sftp smb smbs smtp smtps telnet tftp ws wss
-Features: alt-svc AsynchDNS              HSTS HTTP2       HTTPS-proxy IDN IPv6 Kerberos Largefile libz          NTLM     SPNEGO SSL SSPI threadsafe UnixSockets
+Features: alt-svc AsynchDNS        HSTS HTTP2       HTTPS-proxy IDN IPv6 Kerberos Largefile libz          NTLM PSL SPNEGO SSL SSPI threadsafe UnixSockets
 
 "micro", without libssh2:
 Protocols: dict file ftp ftps gopher gophers http https imap imaps ipfs ipns ldap ldaps mqtt pop3 pop3s rtsp          smb smbs smtp smtps telnet tftp ws wss
-Features: alt-svc AsynchDNS              HSTS HTTP2       HTTPS-proxy IDN IPv6 Kerberos Largefile libz          NTLM     SPNEGO SSL SSPI threadsafe UnixSockets
+Features: alt-svc AsynchDNS        HSTS HTTP2       HTTPS-proxy IDN IPv6 Kerberos Largefile libz          NTLM PSL SPNEGO SSL SSPI threadsafe UnixSockets
 
 "nano", HTTP/1.1:
 Protocols: dict file ftp ftps gopher gophers http https imap imaps ipfs ipns ldap ldaps mqtt pop3 pop3s rtsp          smb smbs smtp smtps telnet tftp ws wss
-Features:         AsynchDNS              HSTS             HTTPS-proxy IDN IPv6 Kerberos Largefile libz          NTLM     SPNEGO SSL SSPI threadsafe UnixSockets
+Features:         AsynchDNS        HSTS             HTTPS-proxy IDN IPv6 Kerberos Largefile libz          NTLM PSL SPNEGO SSL SSPI threadsafe UnixSockets
 
 "pico", HTTP/1.1-only:
 Protocols:                                   http https            ipfs ipns
-Features:         AsynchDNS              HSTS             HTTPS-proxy     IPv6          Largefile libz                          SSL SSPI threadsafe
+Features:         AsynchDNS        HSTS             HTTPS-proxy     IPv6          Largefile libz               PSL        SSL SSPI threadsafe
 ```
 </p></details>
 

diff --git a/_build-runner.sh b/_build-runner.sh
@@ -15,7 +15,7 @@ set -o xtrace -o errexit -o nounset; [ -n "${BASH:-}${ZSH_NAME:-}" ] && set -o p
 cd "$(dirname "$0")"
 
 # Customize these
-export CW_CONFIG='dev-x64-psl-cares'
+export CW_CONFIG='dev-x64-cares'
 #export CURL_REV_='master'
 
 # Install necessary packages

diff --git a/_build.sh b/_build.sh
@@ -48,7 +48,6 @@ set -o xtrace -o errexit -o nounset; [ -n "${BASH:-}${ZSH_NAME:-}" ] && set -o p
 #        wolfssh    build with wolfSSH (requires wolfSSL)
 #        libssh     build with libssh
 #        idn2       build with libidn2
-#        psl        build with libpsl + embedded PublicSuffixList
 #        gsasl      build with gsasl
 #        mini       build with less features, see README.md
 #        micro      build with less features, see README.md
@@ -171,8 +170,8 @@ set -o xtrace -o errexit -o nounset; [ -n "${BASH:-}${ZSH_NAME:-}" ] && set -o p
 #   zstd             cmake
 #   brotli           cmake
 #   cares            cmake
+#   libpsl           manual
 #   libidn2          autotools
-#   libpsl           autotools
 #   gsasl            autotools
 #   nghttp2          cmake
 #   nghttp3          cmake
@@ -1618,8 +1617,8 @@ build_single_target() {
   bld zstd                 "${ZSTD_VER_}"
   bld brotli             "${BROTLI_VER_}"
   bld cares               "${CARES_VER_}"
-  bld libidn2           "${LIBIDN2_VER_}"
   bld libpsl             "${LIBPSL_VER_}"
+  bld libidn2           "${LIBIDN2_VER_}"
   bld nghttp3           "${NGHTTP3_VER_}"
   bld wolfssl           "${WOLFSSL_VER_}"
   bld mbedtls           "${MBEDTLS_VER_}"

diff --git a/_ci-linux-alpine.sh b/_ci-linux-alpine.sh
@@ -36,7 +36,7 @@ fi
 # https://pkgs.alpinelinux.org/packages
 # shellcheck disable=SC2086
 apk add --no-cache curl git gpg rsync build-base cmake \
-  autoconf automake libtool \
+  autoconf automake libtool python3 \
   zip tar xz jq openssl ${extra}
 
 ./_build.sh
diff --git a/_dl.sh b/_dl.sh
@@ -681,12 +681,13 @@ if [[ "${_CONFIG}" = *'cares'* ]]; then
   _DEPS+=' cares'
 fi
 
+if [[ ! "${_CONFIG}" =~ (zero|bldtst) ]]; then
+  _DEPS+=' libpsl'
+fi
+
 if [[ "${_CONFIG}" = *'idn2'* ]]; then
   _DEPS+=' libidn2'
 fi
-if [[ "${_CONFIG}" = *'psl'* ]]; then
-  _DEPS+=' libpsl'
-fi
 if [[ "${_CONFIG}" = *'gsasl'* ]]; then
   _DEPS+=' gsasl'
 fi

diff --git a/_versions.sh b/_versions.sh
@@ -21,12 +21,12 @@ export BROTLI_VER_='1.1.0'
 export BROTLI_HASH=e720a6ca29428b803f4ad165371771f5398faba397edf6778837a18599ea13ff
 export CARES_VER_='1.24.0'
 export CARES_HASH=c517de6d5ac9cd55a9b72c1541c3e25b84588421817b5f092850ac09a8df5103
+export LIBPSL_VER_='0.21.5'
+export LIBPSL_HASH=1dcc9ceae8b128f3c0b3f654decd0e1e891afc6ff81098f227ef260449dae208
 export GSASL_VER_='2.2.1'
 export GSASL_HASH=d45b562e13bd13b9fc20b372f4b53269740cf6279f836f09ce11b9d32bcee075
 export LIBIDN2_VER_='2.3.4'
 export LIBIDN2_HASH=93caba72b4e051d1f8d4f5a076ab63c99b77faee019b72b9783b267986dbb45f
-export LIBPSL_VER_='0.21.5'
-export LIBPSL_HASH=1dcc9ceae8b128f3c0b3f654decd0e1e891afc6ff81098f227ef260449dae208
 export WOLFSSH_VER_='1.4.15'
 export WOLFSSH_HASH=7381144990fdc977c4c25e012aaa1b5cb798532a6a069634c0d6a7798c62ccec
 export LIBSSH_VER_='0.10.6'

diff --git a/curl.patch b/curl.patch
@@ -11,3 +11,22 @@ index 1f735e00144a66..737c2f7e144892 100644
    /* OpenSSL 1.1 deprecates all these cleanup functions and
       turns them into no-ops in OpenSSL 1.0 compatibility mode */
  #else
+diff --git a/lib/version.c b/lib/version.c
+index f957f085d8e23b..01c2a315e5c208 100644
+--- a/lib/version.c
++++ b/lib/version.c
+@@ -211,8 +211,12 @@ char *curl_version(void)
+ #endif
+
+ #ifdef USE_LIBPSL
+-  msnprintf(psl_version, sizeof(psl_version), "libpsl/%s", psl_get_version());
+-  src[i++] = psl_version;
++  {
++    int num = psl_check_version_number(0);
++    msnprintf(psl_version, sizeof(psl_version), "libpsl/%d.%d.%d",
++              num >> 16, (num >> 8) & 0xff, num & 0xff);
++    src[i++] = psl_version;
++  }
+ #endif
+
+ #ifdef USE_SSH
diff --git a/libpsl.sh b/libpsl.sh
@@ -6,11 +6,11 @@
 # Issues:
 # - does not support CMake (only autotools and meson). Where autotools is broken
 #   in curl-for-win.
-# - version string says "libpsl/0.21.1 (no IDNA support)".
 # - autotools build force-sets _WIN32_WINNT to 0x500, while also causing
 #   a compiler warning when overriding our value (which is a higher version).
 #   Builds are not supposed to set _WIN32_WINNT on their own, esp. not override
 #   a custom value.
+# Workaround these by building manually.
 
 # shellcheck disable=SC3040,SC2039
 set -o xtrace -o errexit -o nounset; [ -n "${BASH:-}${ZSH_NAME:-}" ] && set -o pipefail
@@ -25,32 +25,27 @@ _VER="$1"
 
   rm -r -f "${_PKGDIR:?}" "${_BLDDIR:?}"
 
-  options="${_CONFIGURE_GLOBAL}"
-  export CC="${_CC_GLOBAL}"
-  export CFLAGS="${_CFLAGS_GLOBAL} ${_CFLAGS_GLOBAL_AUTOTOOLS}"
-  export CPPFLAGS="${_CPPFLAGS_GLOBAL}"
-  export LDFLAGS="${_LDFLAGS_GLOBAL} ${_LDFLAGS_GLOBAL_AUTOTOOLS}"
-  export LIBS=''
-
-  export PKG_CONFIG_LIBDIR=''  # Avoid picking up non-cross copies
+  # Build manually
 
+  mkdir -p "${_BLDDIR}"
   (
-    mkdir "${_BLDDIR}"; cd "${_BLDDIR}"
+    cd "${_BLDDIR}"
+    python3 '../src/psl-make-dafsa' --output-format=cxx+ '../list/public_suffix_list.dat' 'suffixes_dafsa.h'
     # shellcheck disable=SC2086
-    ../configure ${options} \
-      --disable-rpath \
-      --enable-static \
-      --disable-shared \
-      --disable-runtime \
-      --enable-builtin \
-      --disable-man --silent
+    ${_CC_GLOBAL} ${_CFLAGS_GLOBAL} ${_CFLAGS_GLOBAL_AUTOTOOLS} ${_CPPFLAGS_GLOBAL} \
+      -DENABLE_BUILTIN -DHAVE_VISIBILITY -DBUILDING_PSL \
+      -DPACKAGE_VERSION="\"${LIBPSL_VER_}\"" \
+      -I. -I../include -c ../src/*.c
+    "${AR}" rcs libpsl.a ./*.o
   )
 
-  make --directory="${_BLDDIR}" --jobs="${_JOBS}" install "DESTDIR=$(pwd)/${_PKGDIR}" # >/dev/null # V=1
+  # Install manually
+
+  mkdir -p "${_PP}/include"
+  mkdir -p "${_PP}/lib"
 
-  # Delete .pc and .la files
-  rm -r -f "${_PP}"/lib/pkgconfig
-  rm -f    "${_PP}"/lib/*.la
+  cp -f -p include/libpsl.h "${_PP}/include/"
+  cp -f -p "${_BLDDIR}"/*.a "${_PP}/lib/"
 
   # Make steps for determinism