From 0ac33a93959ce1609d77fde548b961f52ccdeed0 Mon Sep 17 00:00:00 2001 From: Gregory Szorc Date: Sat, 23 Sep 2023 18:49:20 +0800 Subject: [PATCH] WIP CPython 3.12 support --- .github/workflows/apple.yml | 23 +++ .github/workflows/linux.yml | 193 ++++++++++++++++++ .github/workflows/windows.yml | 1 + cpython-unix/Makefile | 7 + cpython-unix/build-cpython-host.sh | 20 +- cpython-unix/build-cpython.sh | 45 +++- cpython-unix/build-main.py | 8 +- cpython-unix/build.py | 12 +- cpython-unix/extension-modules.yml | 80 +++++++- cpython-unix/patch-apple-cross-3.12.patch | 85 ++++++++ .../patch-checksharedmods-disable.patch | 13 ++ ...ch-configure-disable-stdlib-mod-3.12.patch | 30 +++ .../patch-write-python-for-build-3.12.patch | 17 ++ cpython-unix/targets.yml | 18 ++ cpython-windows/build.py | 8 +- pythonbuild/cpython.py | 28 ++- pythonbuild/downloads.py | 9 + src/validation.rs | 46 ++++- 18 files changed, 621 insertions(+), 22 deletions(-) create mode 100644 cpython-unix/patch-apple-cross-3.12.patch create mode 100644 cpython-unix/patch-checksharedmods-disable.patch create mode 100644 cpython-unix/patch-configure-disable-stdlib-mod-3.12.patch create mode 100644 cpython-unix/patch-write-python-for-build-3.12.patch diff --git a/.github/workflows/apple.yml b/.github/workflows/apple.yml index 2884dcab..9a92b1f1 100644 --- a/.github/workflows/apple.yml +++ b/.github/workflows/apple.yml @@ -83,6 +83,16 @@ jobs: py: 'cpython-3.11' optimizations: 'lto' + - target_triple: 'aarch64-apple-darwin' + py: 'cpython-3.12' + optimizations: 'debug' + - target_triple: 'aarch64-apple-darwin' + py: 'cpython-3.12' + optimizations: 'noopt' + - target_triple: 'aarch64-apple-darwin' + py: 'cpython-3.12' + optimizations: 'lto' + # macOS on Intel hardware. This is pretty straightforward. We exclude # noopt because it doesn't provide any compelling advantages over PGO # or LTO builds. @@ -137,6 +147,19 @@ jobs: - target_triple: 'x86_64-apple-darwin' py: 'cpython-3.11' optimizations: 'pgo+lto' + + - target_triple: 'x86_64-apple-darwin' + py: 'cpython-3.12' + optimizations: 'debug' + - target_triple: 'x86_64-apple-darwin' + py: 'cpython-3.12' + optimizations: 'lto' + - target_triple: 'x86_64-apple-darwin' + py: 'cpython-3.12' + optimizations: 'pgo' + - target_triple: 'x86_64-apple-darwin' + py: 'cpython-3.12' + optimizations: 'pgo+lto' needs: - pythonbuild runs-on: 'macos-11' diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index c28b834b..f472a1db 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -157,6 +157,16 @@ jobs: py: 'cpython-3.11' optimizations: 'lto' + - target_triple: 'aarch64-unknown-linux-gnu' + py: 'cpython-3.12' + optimizations: 'debug' + - target_triple: 'aarch64-unknown-linux-gnu' + py: 'cpython-3.12' + optimizations: 'noopt' + - target_triple: 'aarch64-unknown-linux-gnu' + py: 'cpython-3.12' + optimizations: 'lto' + # Cross-compiles can't do PGO and require Python 3.9. - target_triple: 'armv7-unknown-linux-gnueabi' py: 'cpython-3.9' @@ -188,6 +198,16 @@ jobs: py: 'cpython-3.11' optimizations: 'lto' + - target_triple: 'armv7-unknown-linux-gnueabi' + py: 'cpython-3.12' + optimizations: 'debug' + - target_triple: 'armv7-unknown-linux-gnueabi' + py: 'cpython-3.12' + optimizations: 'noopt' + - target_triple: 'armv7-unknown-linux-gnueabi' + py: 'cpython-3.12' + optimizations: 'lto' + # Cross-compiles can't do PGO and require Python 3.9. - target_triple: 'armv7-unknown-linux-gnueabihf' py: 'cpython-3.9' @@ -219,6 +239,16 @@ jobs: py: 'cpython-3.11' optimizations: 'lto' + - target_triple: 'armv7-unknown-linux-gnueabihf' + py: 'cpython-3.12' + optimizations: 'debug' + - target_triple: 'armv7-unknown-linux-gnueabihf' + py: 'cpython-3.12' + optimizations: 'noopt' + - target_triple: 'armv7-unknown-linux-gnueabihf' + py: 'cpython-3.12' + optimizations: 'lto' + # We don't publish noopt builds when PGO is available. - target_triple: 'i686-unknown-linux-gnu' py: 'cpython-3.8' @@ -272,6 +302,19 @@ jobs: py: 'cpython-3.11' optimizations: 'pgo+lto' + - target_triple: 'i686-unknown-linux-gnu' + py: 'cpython-3.12' + optimizations: 'debug' + - target_triple: 'i686-unknown-linux-gnu' + py: 'cpython-3.12' + optimizations: 'lto' + - target_triple: 'i686-unknown-linux-gnu' + py: 'cpython-3.12' + optimizations: 'pgo' + - target_triple: 'i686-unknown-linux-gnu' + py: 'cpython-3.12' + optimizations: 'pgo+lto' + # Cross-compiles can't do PGO and require Python 3.9. - target_triple: 'mips-unknown-linux-gnu' py: 'cpython-3.9' @@ -303,6 +346,16 @@ jobs: py: 'cpython-3.11' optimizations: 'lto' + - target_triple: 'mips-unknown-linux-gnu' + py: 'cpython-3.12' + optimizations: 'debug' + - target_triple: 'mips-unknown-linux-gnu' + py: 'cpython-3.12' + optimizations: 'noopt' + - target_triple: 'mips-unknown-linux-gnu' + py: 'cpython-3.12' + optimizations: 'lto' + # Cross-compiles can't do PGO and require Python 3.9. - target_triple: 'mipsel-unknown-linux-gnu' py: 'cpython-3.9' @@ -334,6 +387,16 @@ jobs: py: 'cpython-3.11' optimizations: 'lto' + - target_triple: 'mipsel-unknown-linux-gnu' + py: 'cpython-3.12' + optimizations: 'debug' + - target_triple: 'mipsel-unknown-linux-gnu' + py: 'cpython-3.12' + optimizations: 'noopt' + - target_triple: 'mipsel-unknown-linux-gnu' + py: 'cpython-3.12' + optimizations: 'lto' + # Cross-compiles can't do PGO and require Python 3.9. - target_triple: 's390x-unknown-linux-gnu' py: 'cpython-3.9' @@ -365,6 +428,16 @@ jobs: py: 'cpython-3.11' optimizations: 'lto' + - target_triple: 's390x-unknown-linux-gnu' + py: 'cpython-3.12' + optimizations: 'debug' + - target_triple: 's390x-unknown-linux-gnu' + py: 'cpython-3.12' + optimizations: 'noopt' + - target_triple: 's390x-unknown-linux-gnu' + py: 'cpython-3.12' + optimizations: 'lto' + # Cross-compiles can't do PGO and require Python 3.9. - target_triple: 'ppc64le-unknown-linux-gnu' py: 'cpython-3.9' @@ -396,6 +469,16 @@ jobs: py: 'cpython-3.11' optimizations: 'lto' + - target_triple: 'ppc64le-unknown-linux-gnu' + py: 'cpython-3.12' + optimizations: 'debug' + - target_triple: 'ppc64le-unknown-linux-gnu' + py: 'cpython-3.12' + optimizations: 'noopt' + - target_triple: 'ppc64le-unknown-linux-gnu' + py: 'cpython-3.12' + optimizations: 'lto' + # We don't publish noopt builds when PGO is available. - target_triple: 'x86_64-unknown-linux-gnu' py: 'cpython-3.8' @@ -465,6 +548,23 @@ jobs: optimizations: 'pgo+lto' run: true + - target_triple: 'x86_64-unknown-linux-gnu' + py: 'cpython-3.12' + optimizations: 'debug' + run: true + - target_triple: 'x86_64-unknown-linux-gnu' + py: 'cpython-3.12' + optimizations: 'lto' + run: true + - target_triple: 'x86_64-unknown-linux-gnu' + py: 'cpython-3.12' + optimizations: 'pgo' + run: true + - target_triple: 'x86_64-unknown-linux-gnu' + py: 'cpython-3.12' + optimizations: 'pgo+lto' + run: true + - target_triple: 'x86_64_v2-unknown-linux-gnu' py: 'cpython-3.9' optimizations: 'debug' @@ -516,6 +616,23 @@ jobs: optimizations: 'pgo+lto' run: true + - target_triple: 'x86_64_v2-unknown-linux-gnu' + py: 'cpython-3.12' + optimizations: 'debug' + run: true + - target_triple: 'x86_64_v2-unknown-linux-gnu' + py: 'cpython-3.12' + optimizations: 'lto' + run: true + - target_triple: 'x86_64_v2-unknown-linux-gnu' + py: 'cpython-3.12' + optimizations: 'pgo' + run: true + - target_triple: 'x86_64_v2-unknown-linux-gnu' + py: 'cpython-3.12' + optimizations: 'pgo+lto' + run: true + - target_triple: 'x86_64_v3-unknown-linux-gnu' py: 'cpython-3.9' optimizations: 'debug' @@ -567,6 +684,23 @@ jobs: optimizations: 'pgo+lto' run: true + - target_triple: 'x86_64_v3-unknown-linux-gnu' + py: 'cpython-3.12' + optimizations: 'debug' + run: true + - target_triple: 'x86_64_v3-unknown-linux-gnu' + py: 'cpython-3.12' + optimizations: 'lto' + run: true + - target_triple: 'x86_64_v3-unknown-linux-gnu' + py: 'cpython-3.12' + optimizations: 'pgo' + run: true + - target_triple: 'x86_64_v3-unknown-linux-gnu' + py: 'cpython-3.12' + optimizations: 'pgo+lto' + run: true + # GitHub Actions runners don't support x86-64-v4 so we can't PGO. - target_triple: 'x86_64_v4-unknown-linux-gnu' py: 'cpython-3.9' @@ -599,6 +733,16 @@ jobs: py: 'cpython-3.11' optimizations: 'lto' + - target_triple: 'x86_64_v4-unknown-linux-gnu' + py: 'cpython-3.12' + optimizations: 'debug' + - target_triple: 'x86_64_v4-unknown-linux-gnu' + py: 'cpython-3.12' + optimizations: 'noopt' + - target_triple: 'x86_64_v4-unknown-linux-gnu' + py: 'cpython-3.12' + optimizations: 'lto' + # musl doesn't support PGO. - target_triple: 'x86_64-unknown-linux-musl' py: 'cpython-3.8' @@ -652,6 +796,19 @@ jobs: optimizations: 'lto' run: true + - target_triple: 'x86_64-unknown-linux-musl' + py: 'cpython-3.12' + optimizations: 'debug' + run: true + - target_triple: 'x86_64-unknown-linux-musl' + py: 'cpython-3.12' + optimizations: 'noopt' + run: true + - target_triple: 'x86_64-unknown-linux-musl' + py: 'cpython-3.12' + optimizations: 'lto' + run: true + - target_triple: 'x86_64_v2-unknown-linux-musl' py: 'cpython-3.9' optimizations: 'debug' @@ -691,6 +848,19 @@ jobs: optimizations: 'lto' run: true + - target_triple: 'x86_64_v2-unknown-linux-musl' + py: 'cpython-3.12' + optimizations: 'debug' + run: true + - target_triple: 'x86_64_v2-unknown-linux-musl' + py: 'cpython-3.12' + optimizations: 'noopt' + run: true + - target_triple: 'x86_64_v2-unknown-linux-musl' + py: 'cpython-3.12' + optimizations: 'lto' + run: true + - target_triple: 'x86_64_v3-unknown-linux-musl' py: 'cpython-3.9' optimizations: 'debug' @@ -730,6 +900,19 @@ jobs: optimizations: 'lto' run: true + - target_triple: 'x86_64_v3-unknown-linux-musl' + py: 'cpython-3.12' + optimizations: 'debug' + run: true + - target_triple: 'x86_64_v3-unknown-linux-musl' + py: 'cpython-3.12' + optimizations: 'noopt' + run: true + - target_triple: 'x86_64_v3-unknown-linux-musl' + py: 'cpython-3.12' + optimizations: 'lto' + run: true + - target_triple: 'x86_64_v4-unknown-linux-musl' py: 'cpython-3.9' optimizations: 'debug' @@ -760,6 +943,16 @@ jobs: py: 'cpython-3.11' optimizations: 'lto' + - target_triple: 'x86_64_v4-unknown-linux-musl' + py: 'cpython-3.12' + optimizations: 'debug' + - target_triple: 'x86_64_v4-unknown-linux-musl' + py: 'cpython-3.12' + optimizations: 'noopt' + - target_triple: 'x86_64_v4-unknown-linux-musl' + py: 'cpython-3.12' + optimizations: 'lto' + needs: - pythonbuild - image diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 41b7af34..854daf1d 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -45,6 +45,7 @@ jobs: - 'cpython-3.9' - 'cpython-3.10' - 'cpython-3.11' + - 'cpython-3.12' vcvars: - 'vcvars32.bat' - 'vcvars64.bat' diff --git a/cpython-unix/Makefile b/cpython-unix/Makefile index bb82b844..e94be8a8 100644 --- a/cpython-unix/Makefile +++ b/cpython-unix/Makefile @@ -259,6 +259,10 @@ $(OUTDIR)/cpython-3.10-$(CPYTHON_3.10_VERSION)-$(HOST_PLATFORM).tar: $(PYTHON_HO $(OUTDIR)/cpython-3.11-$(CPYTHON_3.11_VERSION)-$(HOST_PLATFORM).tar: $(PYTHON_HOST_DEPENDS) $(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) cpython-3.11-host +$(OUTDIR)/cpython-3.12-$(CPYTHON_3.12_VERSION)-$(HOST_PLATFORM).tar: $(PYTHON_HOST_DEPENDS) + $(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) cpython-3.12-host + + PYTHON_DEPENDS := \ $(PYTHON_SUPPORT_FILES) \ $(OUTDIR)/versions/VERSION.pip \ @@ -302,3 +306,6 @@ $(OUTDIR)/cpython-$(CPYTHON_3.10_VERSION)-$(PACKAGE_SUFFIX).tar: $(ALL_PYTHON_DE $(OUTDIR)/cpython-$(CPYTHON_3.11_VERSION)-$(PACKAGE_SUFFIX).tar: $(ALL_PYTHON_DEPENDS) $(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) cpython-3.11 + +$(OUTDIR)/cpython-$(CPYTHON_3.12_VERSION)-$(PACKAGE_SUFFIX).tar: $(ALL_PYTHON_DEPENDS) + $(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) cpython-3.12 diff --git a/cpython-unix/build-cpython-host.sh b/cpython-unix/build-cpython-host.sh index 7011c56b..cfc1e8a7 100755 --- a/cpython-unix/build-cpython-host.sh +++ b/cpython-unix/build-cpython-host.sh @@ -69,10 +69,26 @@ case "${BUILD_TRIPLE}" in ;; esac +EXTRA_CONFIGURE_FLAGS= + +# We may not have a usable libraries to build against. Forcefully disable extensions +# that may not build. +if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_12}" ]; then + for m in _hashlib _ssl; do + EXTRA_CONFIGURE_FLAGS="${EXTRA_CONFIGURE_FLAGS} py_cv_module_${m}=n/a" + done + fi + CC="${HOST_CC}" CXX="${HOST_CXX}" CFLAGS="${EXTRA_HOST_CFLAGS}" CPPFLAGS="${EXTRA_HOST_CFLAGS}" LDFLAGS="${EXTRA_HOST_LDFLAGS}" ./configure \ --prefix /tools/host \ - --without-ensurepip + --without-ensurepip \ + ${EXTRA_CONFIGURE_FLAGS} -make -j "${NUM_CPUS}" install DESTDIR=${ROOT}/out +# Ideally we'd do `make install` here and be done with it. But there's a race +# condition in CPython's build system related to directory creation that gets +# tickled when we do this. https://github.com/python/cpython/issues/109796. +make -j "${NUM_CPUS}" +make -j sharedinstall DESTDIR=${ROOT}/out +make -j install DESTDIR=${ROOT}/out popd diff --git a/cpython-unix/build-cpython.sh b/cpython-unix/build-cpython.sh index 15821a07..d91859d0 100755 --- a/cpython-unix/build-cpython.sh +++ b/cpython-unix/build-cpython.sh @@ -72,7 +72,11 @@ cat Makefile.extra pushd Python-${PYTHON_VERSION} # configure doesn't support cross-compiling on Apple. Teach it. -patch -p1 -i ${ROOT}/patch-apple-cross.patch +if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_12}" ]; then + patch -p1 -i ${ROOT}/patch-apple-cross-3.12.patch +else + patch -p1 -i ${ROOT}/patch-apple-cross.patch +fi # This patch is slightly different on Python 3.10+. if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_10}" ]; then @@ -83,7 +87,9 @@ fi # LIBTOOL_CRUFT is unused and breaks cross-compiling on macOS. Nuke it. # Submitted upstream at https://github.com/python/cpython/pull/101048. -patch -p1 -i ${ROOT}/patch-configure-remove-libtool-cruft.patch +if [ -n "${PYTHON_MEETS_MAXIMUM_VERSION_3_11}" ]; then + patch -p1 -i ${ROOT}/patch-configure-remove-libtool-cruft.patch +fi # Configure nerfs RUNSHARED when cross-compiling, which prevents PGO from running when # we can in fact run the target binaries (e.g. x86_64 host and i686 target). Undo that. @@ -116,7 +122,11 @@ fi # Add a make target to write the PYTHON_FOR_BUILD variable so we can # invoke the host Python on our own. -patch -p1 -i ${ROOT}/patch-write-python-for-build.patch +if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_12}" ]; then + patch -p1 -i ${ROOT}/patch-write-python-for-build-3.12.patch +else + patch -p1 -i ${ROOT}/patch-write-python-for-build.patch +fi # Object files can get listed multiple times leading to duplicate symbols # when linking. Prevent this. @@ -191,7 +201,11 @@ fi # disable the functionality and require our auto-generated Setup.local to provide # everything. if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_11}" ]; then - patch -p1 -i ${ROOT}/patch-configure-disable-stdlib-mod.patch + if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_12}" ]; then + patch -p1 -i ${ROOT}/patch-configure-disable-stdlib-mod-3.12.patch + else + patch -p1 -i ${ROOT}/patch-configure-disable-stdlib-mod.patch + fi # This hack also prevents the conditional definition of the pwd module in # Setup.bootstrap.in from working. So we remove that conditional. @@ -201,7 +215,18 @@ fi # The optimization make targets are both phony and non-phony. This leads # to PGO targets getting reevaluated after a build when you use multiple # make invocations. e.g. `make install` like we do below. Fix that. -patch -p1 -i ${ROOT}/patch-pgo-make-targets.patch +if [ -n "${PYTHON_MEETS_MAXIMUM_VERSION_3_11}" ]; then + patch -p1 -i ${ROOT}/patch-pgo-make-targets.patch +fi + +# There's a post-build Python script that verifies modules were +# built correctly. Ideally we'd invoke this. But our nerfing of +# the configure-based module building and replacing it with our +# own Setup-derived version completely breaks assumptions in this +# script. So leave it off for now... at our own peril. +if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_12}" ]; then + patch -p1 -i ${ROOT}/patch-checksharedmods-disable.patch +fi # We patched configure.ac above. Reflect those changes. autoconf @@ -252,6 +277,13 @@ if [ "${PYBUILD_PLATFORM}" != "macos" ]; then fi fi +# On Python 3.12 we need to link the special hacl library provided some SHA-256 +# implementations. Since we hack up the regular extension building mechanism, we +# need to reinvent this wheel. +if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_12}" ]; then + LDFLAGS="${LDFLAGS} -LModules/_hacl" +fi + CPPFLAGS=$CFLAGS CONFIGURE_FLAGS=" @@ -405,6 +437,7 @@ CFLAGS=$CFLAGS CPPFLAGS=$CFLAGS LDFLAGS=$LDFLAGS \ cat ../Makefile.extra >> Makefile make -j ${NUM_CPUS} +make -j ${NUM_CPUS} sharedinstall DESTDIR=${ROOT}/out/python make -j ${NUM_CPUS} install DESTDIR=${ROOT}/out/python if [ -n "${CPYTHON_DEBUG}" ]; then @@ -712,7 +745,7 @@ ${BUILD_PYTHON} ${ROOT}/fix_shebangs.py ${ROOT}/out/python/install # downstream consumers. OBJECT_DIRS="Objects Parser Parser/pegen Programs Python" OBJECT_DIRS="${OBJECT_DIRS} Modules" -for ext in _blake2 cjkcodecs _ctypes _ctypes/darwin _decimal _expat _io _multiprocessing _sha3 _sqlite _sre _xxtestfuzz ; do +for ext in _blake2 cjkcodecs _ctypes _ctypes/darwin _decimal _expat _hacl _io _multiprocessing _sha3 _sqlite _sre _xxtestfuzz ; do OBJECT_DIRS="${OBJECT_DIRS} Modules/${ext}" done diff --git a/cpython-unix/build-main.py b/cpython-unix/build-main.py index e6bcc036..bc09733a 100755 --- a/cpython-unix/build-main.py +++ b/cpython-unix/build-main.py @@ -61,7 +61,13 @@ def main(): ) parser.add_argument( "--python", - choices={"cpython-3.8", "cpython-3.9", "cpython-3.10", "cpython-3.11"}, + choices={ + "cpython-3.8", + "cpython-3.9", + "cpython-3.10", + "cpython-3.11", + "cpython-3.12", + }, default="cpython-3.11", help="Python distribution to build", ) diff --git a/cpython-unix/build.py b/cpython-unix/build.py index 77f620cc..17fa733e 100755 --- a/cpython-unix/build.py +++ b/cpython-unix/build.py @@ -452,7 +452,7 @@ def build_cpython_host( # Set environment variables allowing convenient testing for Python # version ranges. - for v in ("3.8", "3.9", "3.10", "3.11"): + for v in ("3.8", "3.9", "3.10", "3.11", "3.12"): normal_version = v.replace(".", "_") if meets_python_minimum_version(python_version, v): @@ -767,7 +767,7 @@ def build_cpython( # Set environment variables allowing convenient testing for Python # version ranges. - for v in ("3.8", "3.9", "3.10", "3.11"): + for v in ("3.8", "3.9", "3.10", "3.11", "3.12"): normal_version = v.replace(".", "_") if meets_python_minimum_version(python_version, v): @@ -1159,7 +1159,13 @@ def main(): dest_archive=dest_archive, ) - elif action in ("cpython-3.8", "cpython-3.9", "cpython-3.10", "cpython-3.11"): + elif action in ( + "cpython-3.8", + "cpython-3.9", + "cpython-3.10", + "cpython-3.11", + "cpython-3.12", + ): build_cpython( settings, client, diff --git a/cpython-unix/extension-modules.yml b/cpython-unix/extension-modules.yml index 25e1d94e..37f78101 100644 --- a/cpython-unix/extension-modules.yml +++ b/cpython-unix/extension-modules.yml @@ -91,6 +91,8 @@ _ctypes: - source: _ctypes/darwin/dlfcn_simple.c targets: - .*-apple-.* + # Functionality removed in 3.12. + maximum-python-version: "3.11" - source: _ctypes/malloc_closure.c targets: - .*-apple-.* @@ -312,6 +314,16 @@ _lzma: _md5: sources: - md5module.c + includes: + - Modules/_hacl/include + sources-conditional: + - source: _hacl/Hacl_Hash_MD5.c + minimum-python-version: "3.12" + defines-conditional: + - define: _BSD_SOURCE + minimum-python-version: "3.12" + - define: _DEFAULT_SOURCE + minimum-python-version: "3.12" _multibytecodec: sources: @@ -377,18 +389,63 @@ _scproxy: _sha1: sources: - sha1module.c + sources-conditional: + - source: _hacl/Hacl_Hash_SHA1.c + minimum-python-version: "3.12" + includes: + - Modules/_hacl/include + defines-conditional: + - define: _BSD_SOURCE + minimum-python-version: "3.12" + - define: _DEFAULT_SOURCE + minimum-python-version: "3.12" + +# _sha256 refactored and renamed to _sha2 in 3.12 _sha256: + maximum-python-version: "3.11" sources: - sha256module.c -_sha3: +_sha2: + minimum-python-version: "3.12" sources: - - _sha3/sha3module.c + - sha2module.c + includes: + - Modules/_hacl/include + - Modules/_hacl/internal + defines: + - _BSD_SOURCE + - _DEFAULT_SOURCE + links: + # Use the colon syntax to prevent the library dependency from getting + # recorded in JSON metadata. This relies on setting up a linker library + # path (-L) in LDFLAGS. + - ":libHacl_Hash_SHA2.a" + +_sha3: + sources-conditional: + # _sha3/sha3module.c -> sha3module.c in 3.12. + - source: _sha3/sha3module.c + maximum-python-version: "3.11" + - source: sha3module.c + minimum-python-version: "3.12" + + - source: _hacl/Hacl_Hash_SHA3.c + minimum-python-version: "3.12" + includes: + - Modules/_hacl/include + defines-conditional: + - define: _BSD_SOURCE + minimum-python-version: "3.12" + - define: _DEFAULT_SOURCE + minimum-python-version: "3.12" _sha512: sources: - sha512module.c + # Refactored into other modules in 3.12. + maximum-python-version: "3.11" _signal: setup-enabled: true @@ -507,6 +564,11 @@ _testmultiphase: sources: - _testmultiphase.c +_testsinglephase: + minimum-python-version: '3.12' + sources: + - _testsinglephase.c + _thread: setup-enabled: true required-targets: @@ -569,6 +631,9 @@ _tracemalloc: _typing: minimum-python-version: "3.11" + setup-enabled-conditional: + - enabled: true + minimum-python-version: "3.12" sources: - _typingmodule.c @@ -592,6 +657,11 @@ _weakref: required-targets: - .* +_xxinterpchannels: + minimum-python-version: '3.12' + sources: + - _xxinterpchannelsmodule.c + _xxsubinterpreters: minimum-python-version: '3.9' sources: @@ -801,7 +871,11 @@ xxlimited_35: - .* xxsubtype: - setup-enabled: true + setup-enabled-conditional: + - enabled: true + maximum-python-version: "3.11" + sources: + - xxsubtype.c # xx is a demo extension. So disable globally. xx: diff --git a/cpython-unix/patch-apple-cross-3.12.patch b/cpython-unix/patch-apple-cross-3.12.patch new file mode 100644 index 00000000..3729fe1c --- /dev/null +++ b/cpython-unix/patch-apple-cross-3.12.patch @@ -0,0 +1,85 @@ +diff --git a/configure.ac b/configure.ac +index c62a565eb6..7e5d34632c 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -545,6 +545,15 @@ then + *-*-cygwin*) + ac_sys_system=Cygwin + ;; ++ *-apple-ios*) ++ ac_sys_system=iOS ++ ;; ++ *-apple-tvos*) ++ ac_sys_system=tvOS ++ ;; ++ *-apple-watchos*) ++ ac_sys_system=watchOS ++ ;; + *-*-vxworks*) + ac_sys_system=VxWorks + ;; +@@ -600,6 +609,19 @@ if test "$cross_compiling" = yes; then + *-*-cygwin*) + _host_cpu= + ;; ++ *-*-darwin*) ++ _host_cpu= ++ ;; ++ *-apple-*) ++ case "$host_cpu" in ++ arm*) ++ _host_cpu=arm ++ ;; ++ *) ++ _host_cpu=$host_cpu ++ ;; ++ esac ++ ;; + *-*-vxworks*) + _host_cpu=$host_cpu + ;; +@@ -614,6 +636,23 @@ if test "$cross_compiling" = yes; then + _PYTHON_HOST_PLATFORM="$MACHDEP${_host_cpu:+-$_host_cpu}" + fi + ++# The _PYTHON_HOST_PLATFORM environment variable is used to ++# override the platform name in distutils and sysconfig when ++# cross-compiling. On Apple, the platform name expansion logic ++# is non-trivial, including renaming MACHDEP=darwin to macosx ++# and including the deployment target (or current OS version if ++# not set). Here we always force an override based on the target ++# triple. We do this in all build configurations because historically ++# the automatic resolution has been brittle. ++case "$host" in ++aarch64-apple-darwin*) ++ _PYTHON_HOST_PLATFORM="macosx-${MACOSX_DEPLOYMENT_TARGET}-arm64" ++ ;; ++x86_64-apple-darwin*) ++ _PYTHON_HOST_PLATFORM="macosx-${MACOSX_DEPLOYMENT_TARGET}-x86_64" ++ ;; ++esac ++ + # Some systems cannot stand _XOPEN_SOURCE being defined at all; they + # disable features if it is defined, without any means to access these + # features as extensions. For these systems, we skip the definition of +@@ -1507,7 +1546,7 @@ if test $enable_shared = "yes"; then + BLDLIBRARY='-Wl,+b,$(LIBDIR) -L. -lpython$(LDVERSION)' + RUNSHARED=SHLIB_PATH=`pwd`${SHLIB_PATH:+:${SHLIB_PATH}} + ;; +- Darwin*) ++ Darwin*|iOS*|tvOS*|watchOS*) + LDLIBRARY='libpython$(LDVERSION).dylib' + BLDLIBRARY='-L. -lpython$(LDVERSION)' + RUNSHARED=DYLD_LIBRARY_PATH=`pwd`${DYLD_LIBRARY_PATH:+:${DYLD_LIBRARY_PATH}} +@@ -3173,6 +3203,11 @@ then + Linux*|GNU*|QNX*|VxWorks*|Haiku*) + LDSHARED='$(CC) -shared' + LDCXXSHARED='$(CXX) -shared';; ++ iOS*|tvOS*|watchOS*) ++ LDSHARED='$(CC) -bundle -undefined dynamic_lookup' ++ LDCXXSHARED='$(CXX) -bundle -undefined dynamic_lookup' ++ BLDSHARED="$LDSHARED" ++ ;; + FreeBSD*) + if [[ "`$CC -dM -E - python-for-build ++ echo "set -e" >> python-for-build ++ echo "exec env $(PYTHON_FOR_BUILD) \$$@" >> python-for-build ++ chmod +x python-for-build ++ + ########################################################################## + # Module dependencies and platform-specific files + diff --git a/cpython-unix/targets.yml b/cpython-unix/targets.yml index 998155c2..2912d5a7 100644 --- a/cpython-unix/targets.yml +++ b/cpython-unix/targets.yml @@ -60,6 +60,7 @@ aarch64-apple-darwin: - '3.9' - '3.10' - '3.11' + - '3.12' needs_toolchain: true host_cc: clang host_cxx: clang++ @@ -148,6 +149,7 @@ aarch64-unknown-linux-gnu: - '3.9' - '3.10' - '3.11' + - '3.12' docker_image_suffix: .cross host_cc: /usr/bin/x86_64-linux-gnu-gcc host_cxx: /usr/bin/x86_64-linux-gnu-g++ @@ -225,6 +227,7 @@ armv7-unknown-linux-gnueabi: - '3.9' - '3.10' - '3.11' + - '3.12' docker_image_suffix: .cross host_cc: /usr/bin/x86_64-linux-gnu-gcc host_cxx: /usr/bin/x86_64-linux-gnu-g++ @@ -262,6 +265,7 @@ armv7-unknown-linux-gnueabihf: - '3.9' - '3.10' - '3.11' + - '3.12' docker_image_suffix: .cross host_cc: /usr/bin/x86_64-linux-gnu-gcc host_cxx: /usr/bin/x86_64-linux-gnu-g++ @@ -300,6 +304,7 @@ i686-unknown-linux-gnu: - '3.9' - '3.10' - '3.11' + - '3.12' needs_toolchain: true host_cc: clang host_cxx: clang++ @@ -342,6 +347,7 @@ mips-unknown-linux-gnu: - '3.9' - '3.10' - '3.11' + - '3.12' docker_image_suffix: .cross host_cc: /usr/bin/x86_64-linux-gnu-gcc host_cxx: /usr/bin/x86_64-linux-gnu-g++ @@ -379,6 +385,7 @@ mipsel-unknown-linux-gnu: - '3.9' - '3.10' - '3.11' + - '3.12' docker_image_suffix: .cross host_cc: /usr/bin/x86_64-linux-gnu-gcc host_cxx: /usr/bin/x86_64-linux-gnu-g++ @@ -416,6 +423,7 @@ ppc64le-unknown-linux-gnu: - '3.9' - '3.10' - '3.11' + - '3.12' docker_image_suffix: .cross host_cc: /usr/bin/x86_64-linux-gnu-gcc host_cxx: /usr/bin/x86_64-linux-gnu-g++ @@ -453,6 +461,7 @@ s390x-unknown-linux-gnu: - '3.9' - '3.10' - '3.11' + - '3.12' docker_image_suffix: .cross host_cc: /usr/bin/x86_64-linux-gnu-gcc host_cxx: /usr/bin/x86_64-linux-gnu-g++ @@ -535,6 +544,7 @@ x86_64-apple-darwin: - '3.9' - '3.10' - '3.11' + - '3.12' needs_toolchain: true apple_sdk_platform: macosx host_cc: clang @@ -703,6 +713,7 @@ x86_64-unknown-linux-gnu: - '3.9' - '3.10' - '3.11' + - '3.12' needs_toolchain: true host_cc: clang host_cxx: clang++ @@ -743,6 +754,7 @@ x86_64_v2-unknown-linux-gnu: - '3.9' - '3.10' - '3.11' + - '3.12' needs_toolchain: true host_cc: clang host_cxx: clang++ @@ -784,6 +796,7 @@ x86_64_v3-unknown-linux-gnu: - '3.9' - '3.10' - '3.11' + - '3.12' needs_toolchain: true host_cc: clang host_cxx: clang++ @@ -825,6 +838,7 @@ x86_64_v4-unknown-linux-gnu: - '3.9' - '3.10' - '3.11' + - '3.12' needs_toolchain: true host_cc: clang host_cxx: clang++ @@ -866,6 +880,7 @@ x86_64-unknown-linux-musl: - '3.9' - '3.10' - '3.11' + - '3.12' needs_toolchain: true host_cc: clang host_cxx: clang++ @@ -907,6 +922,7 @@ x86_64_v2-unknown-linux-musl: - '3.9' - '3.10' - '3.11' + - '3.12' needs_toolchain: true host_cc: clang host_cxx: clang++ @@ -949,6 +965,7 @@ x86_64_v3-unknown-linux-musl: - '3.9' - '3.10' - '3.11' + - '3.12' needs_toolchain: true host_cc: clang host_cxx: clang++ @@ -991,6 +1008,7 @@ x86_64_v4-unknown-linux-musl: - '3.9' - '3.10' - '3.11' + - '3.12' needs_toolchain: true host_cc: clang host_cxx: clang++ diff --git a/cpython-windows/build.py b/cpython-windows/build.py index 0fa8e618..59f2d268 100644 --- a/cpython-windows/build.py +++ b/cpython-windows/build.py @@ -2628,7 +2628,13 @@ def main(): ) parser.add_argument( "--python", - choices={"cpython-3.8", "cpython-3.9", "cpython-3.10", "cpython-3.11"}, + choices={ + "cpython-3.8", + "cpython-3.9", + "cpython-3.10", + "cpython-3.11", + "cpython-3.12", + }, default="cpython-3.11", help="Python distribution to build", ) diff --git a/pythonbuild/cpython.py b/pythonbuild/cpython.py index 4429cc75..beb84db1 100644 --- a/pythonbuild/cpython.py +++ b/pythonbuild/cpython.py @@ -73,6 +73,19 @@ "maximum-python-version": {"type": "string"}, "required-targets": {"type": "array", "items": {"type": "string"}}, "setup-enabled": {"type": "boolean"}, + "setup-enabled-conditional": { + "type": "array", + "items": { + "type": "object", + "properties": { + "enabled": {"type": "boolean"}, + "minimum-python-version": {"type": "string"}, + "maximum-python-version": {"type": "string"}, + }, + "additionalProperties": False, + "required": ["enabled"], + }, + }, "sources": {"type": "array", "items": {"type": "string"}}, "sources-conditional": { "type": "array", @@ -248,6 +261,19 @@ def derive_setup_local( if info.get("setup-enabled", False): setup_enabled_wanted.add(name) + for entry in info.get("setup-enabled-conditional", []): + python_min_match_setup = meets_python_minimum_version( + python_version, entry.get("minimum-python-version", "1.0") + ) + python_max_match_setup = meets_python_maximum_version( + python_version, entry.get("maximum-python-version", "100.0") + ) + + if entry.get("enabled", False) and ( + python_min_match_setup and python_max_match_setup + ): + setup_enabled_wanted.add(name) + if info.get("config-c-only"): config_c_only_wanted.add(name) @@ -443,7 +469,7 @@ def derive_setup_local( # Presumably this means the extension comes from the distribution's # Setup. Lack of sources means we don't need to derive a Setup.local # line. - if "sources" not in info: + if "sources" not in info and "sources-conditional" not in info: if name not in setup_enabled_lines: raise Exception( f"found a sourceless extension ({name}) with no Setup entry" diff --git a/pythonbuild/downloads.py b/pythonbuild/downloads.py index cc267562..dc372a73 100644 --- a/pythonbuild/downloads.py +++ b/pythonbuild/downloads.py @@ -70,6 +70,15 @@ "license_file": "LICENSE.cpython.txt", "python_tag": "cp311", }, + "cpython-3.12": { + "url": "https://www.python.org/ftp/python/3.12.0/Python-3.12.0rc3.tar.xz", + "size": 20566024, + "sha256": "96397e891e98802b1d399dee3ceaeb9bcf0aa2566c8a7b1cce4d0196c277506a", + "version": "3.12.0rc3", + "licenses": ["Python-2.0", "CNRI-Python"], + "license_file": "LICENSE.cpython.txt", + "python_tag": "cp312", + }, "expat": { "url": "https://github.com/libexpat/libexpat/releases/download/R_2_5_0/expat-2.5.0.tar.xz", "size": 460560, diff --git a/src/validation.rs b/src/validation.rs index abee517d..1327dc4b 100644 --- a/src/validation.rs +++ b/src/validation.rs @@ -275,6 +275,16 @@ static DARWIN_ALLOWED_DYLIBS: Lazy> = Lazy::new(|| { max_compatibility_version: "3.11.0".try_into().unwrap(), required: false, }, + MachOAllowedDylib { + name: "@executable_path/../lib/libpython3.12.dylib".to_string(), + max_compatibility_version: "3.12.0".try_into().unwrap(), + required: false, + }, + MachOAllowedDylib { + name: "@executable_path/../lib/libpython3.12d.dylib".to_string(), + max_compatibility_version: "3.12.0".try_into().unwrap(), + required: false, + }, MachOAllowedDylib { name: "/System/Library/Frameworks/AppKit.framework/Versions/C/AppKit".to_string(), max_compatibility_version: "45.0.0".try_into().unwrap(), @@ -606,9 +616,7 @@ const GLOBAL_EXTENSIONS: &[&str] = &[ "_queue", "_random", "_sha1", - "_sha256", "_sha3", - "_sha512", "_signal", "_socket", "_sqlite3", @@ -650,23 +658,36 @@ const GLOBAL_EXTENSIONS: &[&str] = &[ // parser removed in 3.10. // _tokenize added in 3.11. // _typing added in 3.11. +// _testsinglephase added in 3.12. +// _sha256 and _sha512 merged into _sha2 in 3.12. +// _xxinterpchannels added in 3.12. // We didn't build ctypes_test until 3.9. // We didn't build some test extensions until 3.9. -const GLOBAL_EXTENSIONS_PYTHON_3_8: &[&str] = &["parser"]; +const GLOBAL_EXTENSIONS_PYTHON_3_8: &[&str] = &[ + "_sha256", + "_sha512", + "parser"]; const GLOBAL_EXTENSIONS_PYTHON_3_9: &[&str] = &[ "_peg_parser", + "_sha256", + "_sha512", "_uuid", "_xxsubinterpreters", "_zoneinfo", "parser", ]; -const GLOBAL_EXTENSIONS_PYTHON_3_10: &[&str] = &["_uuid", "_xxsubinterpreters", "_zoneinfo"]; +const GLOBAL_EXTENSIONS_PYTHON_3_10: &[&str] = &[ + "_sha256", + "_sha512", + "_uuid", "_xxsubinterpreters", "_zoneinfo"]; const GLOBAL_EXTENSIONS_PYTHON_3_11: &[&str] = &[ + "_sha256", + "_sha512", "_tokenize", "_typing", "_uuid", @@ -674,6 +695,16 @@ const GLOBAL_EXTENSIONS_PYTHON_3_11: &[&str] = &[ "_zoneinfo", ]; +const GLOBAL_EXTENSIONS_PYTHON_3_12: &[&str] = &[ + "_sha2", + "_testsinglephase", + "_tokenize", + "_typing", + "_xxinterpchannels", + "_xxsubinterpreters", + "_zoneinfo", +]; + const GLOBAL_EXTENSIONS_MACOS: &[&str] = &["_scproxy"]; const GLOBAL_EXTENSIONS_POSIX: &[&str] = &[ @@ -1404,6 +1435,9 @@ fn validate_extension_modules( "3.11" => { wanted.extend(GLOBAL_EXTENSIONS_PYTHON_3_11); } + "3.12" => { + wanted.extend(GLOBAL_EXTENSIONS_PYTHON_3_12); + } _ => { panic!("unhandled Python version: {}", python_major_minor); } @@ -1433,7 +1467,7 @@ fn validate_extension_modules( } } - if (is_linux || is_macos) && matches!(python_major_minor, "3.9" | "3.10" | "3.11") { + if (is_linux || is_macos) && matches!(python_major_minor, "3.9" | "3.10" | "3.11" | "3.12") { wanted.extend([ "_testbuffer", "_testimportmultiple", @@ -1572,6 +1606,8 @@ fn validate_distribution( "3.10" } else if dist_filename.starts_with("cpython-3.11.") { "3.11" + } else if dist_filename.starts_with("cpython-3.12.") { + "3.12" } else { return Err(anyhow!("could not parse Python version from filename")); };