From c9e2401cb86be96800e4ed88fa967f393abc137f Mon Sep 17 00:00:00 2001 From: Viktor Szakats Date: Sun, 14 Jan 2024 15:40:55 +0000 Subject: [PATCH] enable libpsl by default - add manual build for libpsl. It's shorter than calling up autotools, which is broken in curl-for-win and also has an libpsl upstream issue on Windows. It's more fragile and may break by upstream changes. - retrofit curl patch for a clean libpsl version string: https://github.com/curl/curl/commit/72bd88adde0e8cf6e63644a7d6df1da01a399db4 https://github.com/curl/curl/pull/12700 - exclude libpsl from `-zero` and `-bldtst` builds. Closes #63 --- README.md | 16 ++++++++-------- _build-runner.sh | 2 +- _build.sh | 5 ++--- _ci-linux-alpine.sh | 2 +- _dl.sh | 7 ++++--- _versions.sh | 4 ++-- curl.patch | 19 +++++++++++++++++++ libpsl.sh | 37 ++++++++++++++++--------------------- 8 files changed, 53 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index efce7609f..16db46475 100644 --- a/README.md +++ b/README.md @@ -61,38 +61,38 @@ Uses [LibreSSL](https://www.libressl.org/) TLS backend. ``` Windows with runtime-selectable option Schannel: Protocols: dict file ftp ftps gopher gophers http https imap imaps ipfs ipns ldap ldaps mqtt pop3 pop3s rtsp scp sftp smb smbs smtp smtps telnet tftp ws wss -Features: alt-svc AsynchDNS brotli HSTS HTTP2 HTTP3 HTTPS-proxy IDN IPv6 Kerberos Largefile libz MultiSSL NTLM SPNEGO SSL SSPI threadsafe UnixSockets zstd +Features: alt-svc AsynchDNS brotli HSTS HTTP2 HTTP3 HTTPS-proxy IDN IPv6 Kerberos Largefile libz MultiSSL NTLM PSL SPNEGO SSL SSPI threadsafe UnixSockets zstd macOS with runtime-selectable option SecureTransport: Protocols: dict file ftp ftps gopher gophers http https imap imaps ipfs ipns ldap ldaps mqtt pop3 pop3s rtsp scp sftp smb smbs smtp smtps telnet tftp ws wss -Features: alt-svc AsynchDNS brotli HSTS HTTP2 HTTP3 HTTPS-proxy IPv6 Largefile libz MultiSSL NTLM SSL threadsafe UnixSockets zstd +Features: alt-svc AsynchDNS brotli HSTS HTTP2 HTTP3 HTTPS-proxy IPv6 Largefile libz MultiSSL NTLM PSL SSL threadsafe UnixSockets zstd Linux: Protocols: dict file ftp ftps gopher gophers http https imap imaps ipfs ipns mqtt pop3 pop3s rtsp scp sftp smb smbs smtp smtps telnet tftp ws wss -Features: alt-svc AsynchDNS brotli HSTS HTTP2 HTTP3 HTTPS-proxy IPv6 Largefile libz NTLM SSL threadsafe UnixSockets zstd +Features: alt-svc AsynchDNS brotli HSTS HTTP2 HTTP3 HTTPS-proxy IPv6 Largefile libz NTLM PSL SSL threadsafe UnixSockets zstd ```
Alternate configurations:

``` "noh3", HTTP/2: Protocols: dict file ftp ftps gopher gophers http https imap imaps ipfs ipns ldap ldaps mqtt pop3 pop3s rtsp scp sftp smb smbs smtp smtps telnet tftp ws wss -Features: alt-svc AsynchDNS brotli HSTS HTTP2 HTTPS-proxy IDN IPv6 Kerberos Largefile libz MultiSSL NTLM SPNEGO SSL SSPI threadsafe UnixSockets zstd +Features: alt-svc AsynchDNS brotli HSTS HTTP2 HTTPS-proxy IDN IPv6 Kerberos Largefile libz MultiSSL NTLM PSL SPNEGO SSL SSPI threadsafe UnixSockets zstd "mini", without brotli and zstd, with OS TLS backend (Schannel, SecureTransport) if available: Protocols: dict file ftp ftps gopher gophers http https imap imaps ipfs ipns ldap ldaps mqtt pop3 pop3s rtsp scp sftp smb smbs smtp smtps telnet tftp ws wss -Features: alt-svc AsynchDNS HSTS HTTP2 HTTPS-proxy IDN IPv6 Kerberos Largefile libz NTLM SPNEGO SSL SSPI threadsafe UnixSockets +Features: alt-svc AsynchDNS HSTS HTTP2 HTTPS-proxy IDN IPv6 Kerberos Largefile libz NTLM PSL SPNEGO SSL SSPI threadsafe UnixSockets "micro", without libssh2: Protocols: dict file ftp ftps gopher gophers http https imap imaps ipfs ipns ldap ldaps mqtt pop3 pop3s rtsp smb smbs smtp smtps telnet tftp ws wss -Features: alt-svc AsynchDNS HSTS HTTP2 HTTPS-proxy IDN IPv6 Kerberos Largefile libz NTLM SPNEGO SSL SSPI threadsafe UnixSockets +Features: alt-svc AsynchDNS HSTS HTTP2 HTTPS-proxy IDN IPv6 Kerberos Largefile libz NTLM PSL SPNEGO SSL SSPI threadsafe UnixSockets "nano", HTTP/1.1: Protocols: dict file ftp ftps gopher gophers http https imap imaps ipfs ipns ldap ldaps mqtt pop3 pop3s rtsp smb smbs smtp smtps telnet tftp ws wss -Features: AsynchDNS HSTS HTTPS-proxy IDN IPv6 Kerberos Largefile libz NTLM SPNEGO SSL SSPI threadsafe UnixSockets +Features: AsynchDNS HSTS HTTPS-proxy IDN IPv6 Kerberos Largefile libz NTLM PSL SPNEGO SSL SSPI threadsafe UnixSockets "pico", HTTP/1.1-only: Protocols: http https ipfs ipns -Features: AsynchDNS HSTS HTTPS-proxy IPv6 Largefile libz SSL SSPI threadsafe +Features: AsynchDNS HSTS HTTPS-proxy IPv6 Largefile libz PSL SSL SSPI threadsafe ```

diff --git a/_build-runner.sh b/_build-runner.sh index 2e0c87f92..8b0f4d41d 100755 --- a/_build-runner.sh +++ b/_build-runner.sh @@ -15,7 +15,7 @@ set -o xtrace -o errexit -o nounset; [ -n "${BASH:-}${ZSH_NAME:-}" ] && set -o p cd "$(dirname "$0")" # Customize these -export CW_CONFIG='dev-x64-psl-cares' +export CW_CONFIG='dev-x64-cares' #export CURL_REV_='master' # Install necessary packages diff --git a/_build.sh b/_build.sh index b0be15200..e4b757b92 100755 --- a/_build.sh +++ b/_build.sh @@ -48,7 +48,6 @@ set -o xtrace -o errexit -o nounset; [ -n "${BASH:-}${ZSH_NAME:-}" ] && set -o p # wolfssh build with wolfSSH (requires wolfSSL) # libssh build with libssh # idn2 build with libidn2 -# psl build with libpsl + embedded PublicSuffixList # gsasl build with gsasl # mini build with less features, see README.md # micro build with less features, see README.md @@ -171,8 +170,8 @@ set -o xtrace -o errexit -o nounset; [ -n "${BASH:-}${ZSH_NAME:-}" ] && set -o p # zstd cmake # brotli cmake # cares cmake +# libpsl manual # libidn2 autotools -# libpsl autotools # gsasl autotools # nghttp2 cmake # nghttp3 cmake @@ -1618,8 +1617,8 @@ build_single_target() { bld zstd "${ZSTD_VER_}" bld brotli "${BROTLI_VER_}" bld cares "${CARES_VER_}" - bld libidn2 "${LIBIDN2_VER_}" bld libpsl "${LIBPSL_VER_}" + bld libidn2 "${LIBIDN2_VER_}" bld nghttp3 "${NGHTTP3_VER_}" bld wolfssl "${WOLFSSL_VER_}" bld mbedtls "${MBEDTLS_VER_}" diff --git a/_ci-linux-alpine.sh b/_ci-linux-alpine.sh index 1dbbb1952..3cd377949 100755 --- a/_ci-linux-alpine.sh +++ b/_ci-linux-alpine.sh @@ -36,7 +36,7 @@ fi # https://pkgs.alpinelinux.org/packages # shellcheck disable=SC2086 apk add --no-cache curl git gpg rsync build-base cmake \ - autoconf automake libtool \ + autoconf automake libtool python3 \ zip tar xz jq openssl ${extra} ./_build.sh diff --git a/_dl.sh b/_dl.sh index 52dcb8721..a8632841d 100755 --- a/_dl.sh +++ b/_dl.sh @@ -681,12 +681,13 @@ if [[ "${_CONFIG}" = *'cares'* ]]; then _DEPS+=' cares' fi +if [[ ! "${_CONFIG}" =~ (zero|bldtst) ]]; then + _DEPS+=' libpsl' +fi + if [[ "${_CONFIG}" = *'idn2'* ]]; then _DEPS+=' libidn2' fi -if [[ "${_CONFIG}" = *'psl'* ]]; then - _DEPS+=' libpsl' -fi if [[ "${_CONFIG}" = *'gsasl'* ]]; then _DEPS+=' gsasl' fi diff --git a/_versions.sh b/_versions.sh index c8a8da0e0..2106d4862 100755 --- a/_versions.sh +++ b/_versions.sh @@ -21,12 +21,12 @@ export BROTLI_VER_='1.1.0' export BROTLI_HASH=e720a6ca29428b803f4ad165371771f5398faba397edf6778837a18599ea13ff export CARES_VER_='1.24.0' export CARES_HASH=c517de6d5ac9cd55a9b72c1541c3e25b84588421817b5f092850ac09a8df5103 +export LIBPSL_VER_='0.21.5' +export LIBPSL_HASH=1dcc9ceae8b128f3c0b3f654decd0e1e891afc6ff81098f227ef260449dae208 export GSASL_VER_='2.2.1' export GSASL_HASH=d45b562e13bd13b9fc20b372f4b53269740cf6279f836f09ce11b9d32bcee075 export LIBIDN2_VER_='2.3.4' export LIBIDN2_HASH=93caba72b4e051d1f8d4f5a076ab63c99b77faee019b72b9783b267986dbb45f -export LIBPSL_VER_='0.21.5' -export LIBPSL_HASH=1dcc9ceae8b128f3c0b3f654decd0e1e891afc6ff81098f227ef260449dae208 export WOLFSSH_VER_='1.4.15' export WOLFSSH_HASH=7381144990fdc977c4c25e012aaa1b5cb798532a6a069634c0d6a7798c62ccec export LIBSSH_VER_='0.10.6' diff --git a/curl.patch b/curl.patch index fcae3c722..5dbb2e906 100644 --- a/curl.patch +++ b/curl.patch @@ -11,3 +11,22 @@ index 1f735e00144a66..737c2f7e144892 100644 /* OpenSSL 1.1 deprecates all these cleanup functions and turns them into no-ops in OpenSSL 1.0 compatibility mode */ #else +diff --git a/lib/version.c b/lib/version.c +index f957f085d8e23b..01c2a315e5c208 100644 +--- a/lib/version.c ++++ b/lib/version.c +@@ -211,8 +211,12 @@ char *curl_version(void) + #endif + + #ifdef USE_LIBPSL +- msnprintf(psl_version, sizeof(psl_version), "libpsl/%s", psl_get_version()); +- src[i++] = psl_version; ++ { ++ int num = psl_check_version_number(0); ++ msnprintf(psl_version, sizeof(psl_version), "libpsl/%d.%d.%d", ++ num >> 16, (num >> 8) & 0xff, num & 0xff); ++ src[i++] = psl_version; ++ } + #endif + + #ifdef USE_SSH diff --git a/libpsl.sh b/libpsl.sh index 94980f289..d030cc1aa 100755 --- a/libpsl.sh +++ b/libpsl.sh @@ -6,11 +6,11 @@ # Issues: # - does not support CMake (only autotools and meson). Where autotools is broken # in curl-for-win. -# - version string says "libpsl/0.21.1 (no IDNA support)". # - autotools build force-sets _WIN32_WINNT to 0x500, while also causing # a compiler warning when overriding our value (which is a higher version). # Builds are not supposed to set _WIN32_WINNT on their own, esp. not override # a custom value. +# Workaround these by building manually. # shellcheck disable=SC3040,SC2039 set -o xtrace -o errexit -o nounset; [ -n "${BASH:-}${ZSH_NAME:-}" ] && set -o pipefail @@ -25,32 +25,27 @@ _VER="$1" rm -r -f "${_PKGDIR:?}" "${_BLDDIR:?}" - options="${_CONFIGURE_GLOBAL}" - export CC="${_CC_GLOBAL}" - export CFLAGS="${_CFLAGS_GLOBAL} ${_CFLAGS_GLOBAL_AUTOTOOLS}" - export CPPFLAGS="${_CPPFLAGS_GLOBAL}" - export LDFLAGS="${_LDFLAGS_GLOBAL} ${_LDFLAGS_GLOBAL_AUTOTOOLS}" - export LIBS='' - - export PKG_CONFIG_LIBDIR='' # Avoid picking up non-cross copies + # Build manually + mkdir -p "${_BLDDIR}" ( - mkdir "${_BLDDIR}"; cd "${_BLDDIR}" + cd "${_BLDDIR}" + python3 '../src/psl-make-dafsa' --output-format=cxx+ '../list/public_suffix_list.dat' 'suffixes_dafsa.h' # shellcheck disable=SC2086 - ../configure ${options} \ - --disable-rpath \ - --enable-static \ - --disable-shared \ - --disable-runtime \ - --enable-builtin \ - --disable-man --silent + ${_CC_GLOBAL} ${_CFLAGS_GLOBAL} ${_CFLAGS_GLOBAL_AUTOTOOLS} ${_CPPFLAGS_GLOBAL} \ + -DENABLE_BUILTIN -DHAVE_VISIBILITY -DBUILDING_PSL \ + -DPACKAGE_VERSION="\"${LIBPSL_VER_}\"" \ + -I. -I../include -c ../src/*.c + "${AR}" rcs libpsl.a ./*.o ) - make --directory="${_BLDDIR}" --jobs="${_JOBS}" install "DESTDIR=$(pwd)/${_PKGDIR}" # >/dev/null # V=1 + # Install manually + + mkdir -p "${_PP}/include" + mkdir -p "${_PP}/lib" - # Delete .pc and .la files - rm -r -f "${_PP}"/lib/pkgconfig - rm -f "${_PP}"/lib/*.la + cp -f -p include/libpsl.h "${_PP}/include/" + cp -f -p "${_BLDDIR}"/*.a "${_PP}/lib/" # Make steps for determinism