unix: upgrade llvm/clang to 13.0.0

Python 3.6+ is now a build dependency. On macOS, the system Python should be sufficiently new. So the build should hopefully pick it up. On Linux, our old Debian version only ships Python 3.4, so we need to install a modern Python. We simply build it from source as part of the clang build script. A better solution would be to establish a dedicated "host Python" artifact. This way we could use this Python build for both Clang and the host Python used in cross-compiles. But the current solution in this patch is much simpler.
astral-sh · Oct 17, 2021 · e3595fa · e3595fa
1 parent b084e3f
commit e3595fa
Show file tree

Hide file tree

Showing 5 changed files with 95 additions and 42 deletions.
diff --git a/cpython-unix/build-clang-linux64.sh b/cpython-unix/build-clang-linux64.sh
@@ -47,6 +47,28 @@ mv ninja /tools/extra/bin/
 
 export PATH=/tools/extra/bin:/tools/host/bin:$PATH
 
+EXTRA_FLAGS=
+
+if [ -x "${SCCACHE}" ]; then
+  "${SCCACHE}" --start-server
+  EXTRA_FLAGS="${EXTRA_FLAGS} -DCMAKE_C_COMPILER_LAUNCHER=${SCCACHE} -DCMAKE_CXX_COMPILER_LAUNCHER=${SCCACHE}"
+fi
+
+if [ -n "${CI}" ]; then
+  NUM_JOBS=${NUM_JOBS_AGGRESSIVE}
+else
+  NUM_JOBS=${NUM_CPUS}
+fi
+
+# clang requires a modern Python to build.
+tar -xf Python-${PYTHON_VERSION}.tar.xz
+pushd "Python-${PYTHON_VERSION}"
+CC="${HOST_CC}" CFLAGS="${EXTRA_HOST_CFLAGS}" CPPFLAGS="${EXTRA_HOST_CFLAGS}" LDFLAGS="${EXTRA_HOST_LDFLAGS}" ./configure \
+  --prefix /tools/host \
+  --without-ensurepip
+make -j "${NUM_CPUS}" install
+popd
+
 mkdir llvm
 pushd llvm
 tar --strip-components=1 -xf ${ROOT}/llvm-${LLVM_VERSION}.src.tar.xz
@@ -85,19 +107,6 @@ popd
 mkdir llvm-objdir
 pushd llvm-objdir
 
-EXTRA_FLAGS=
-
-if [ -x "${SCCACHE}" ]; then
-  "${SCCACHE}" --start-server
-  EXTRA_FLAGS="${EXTRA_FLAGS} -DCMAKE_C_COMPILER_LAUNCHER=${SCCACHE} -DCMAKE_CXX_COMPILER_LAUNCHER=${SCCACHE}"
-fi
-
-if [ -n "${CI}" ]; then
-  NUM_JOBS=${NUM_JOBS_AGGRESSIVE}
-else
-  NUM_JOBS=${NUM_CPUS}
-fi
-
 # Stage 1: Build with GCC.
 mkdir stage1
 pushd stage1

diff --git a/cpython-unix/build-cpython.sh b/cpython-unix/build-cpython.sh
@@ -93,6 +93,25 @@ rm -rf setuptools-tmp
 if [ "${BUILD_TRIPLE}" != "${TARGET_TRIPLE}" ]; then
   pushd "Python-${PYTHON_VERSION}"
 
+  # Same patch as below. See comment there.
+  if [ "${CC}" = "clang" ]; then
+    patch -p1 <<"EOF"
+diff --git a/configure b/configure
+index 7cad0e2f98..50212236c4 100755
+--- a/configure
++++ b/configure
+@@ -5196,7 +5196,7 @@ $as_echo "$as_me:
+ fi
+
+
+-MULTIARCH=$($CC --print-multiarch 2>/dev/null)
++MULTIARCH=
+
+
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for the platform triplet based on compiler characteristics" >&5
+EOF
+  fi
+
   # When cross-compiling, we need to build a host Python that has working zlib
   # and ctypes extensions, otherwise various things fail. (`make install` fails
   # without zlib and setuptools / pip used by target install fail due to missing
@@ -309,6 +328,27 @@ index 1252335472..33c11fbade 100755
 EOF
 fi
 
+# Clang 13 actually prints something with --print-multiarch, confusing CPython's
+# configure. This is reported as https://bugs.python.org/issue45405. We nerf the
+# check since we know what we're doing.
+if [ "${CC}" = "clang" ]; then
+    patch -p1 <<"EOF"
+diff --git a/configure b/configure
+index 7cad0e2f98..50212236c4 100755
+--- a/configure
++++ b/configure
+@@ -5196,7 +5196,7 @@ $as_echo "$as_me:
+ fi
+ 
+ 
+-MULTIARCH=$($CC --print-multiarch 2>/dev/null)
++MULTIARCH=
+ 
+ 
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for the platform triplet based on compiler characteristics" >&5
+EOF
+fi
+
 # Add a make target to write the PYTHON_FOR_BUILD variable so we can
 # invoke the host Python on our own.
 patch -p1 << "EOF"

diff --git a/cpython-unix/build.py b/cpython-unix/build.py
@@ -322,6 +322,7 @@ def build_clang(client, image, host_platform):
     libcxx_archive = download_entry("libc++", DOWNLOADS_PATH)
     libcxxabi_archive = download_entry("libc++abi", DOWNLOADS_PATH)
     libunwind_archive = download_entry("libunwind", DOWNLOADS_PATH)
+    python_archive = download_entry("cpython-3.9", DOWNLOADS_PATH)
 
     with build_environment(client, image) as build_env:
         install_sccache(build_env)
@@ -337,6 +338,7 @@ def build_clang(client, image, host_platform):
             libcxx_archive,
             libcxxabi_archive,
             libunwind_archive,
+            python_archive,
         ):
             build_env.copy_file(a)
 
@@ -356,6 +358,7 @@ def build_clang(client, image, host_platform):
             "LIBUNWIND_VERSION": DOWNLOADS["libunwind"]["version"],
             "LLD_VERSION": DOWNLOADS["lld"]["version"],
             "LLVM_VERSION": DOWNLOADS["llvm"]["version"],
+            "PYTHON_VERSION": DOWNLOADS["cpython-3.9"]["version"],
         }
 
         add_env_common(env)

diff --git a/cpython-unix/clang.Dockerfile b/cpython-unix/clang.Dockerfile
@@ -2,8 +2,9 @@
 RUN apt-get install \
     libc6-dev \
     libc6-dev:i386 \
+    libffi-dev \
+    make \
     patch \
-    python3 \
     tar \
     xz-utils \
     unzip \

diff --git a/pythonbuild/downloads.py b/pythonbuild/downloads.py
@@ -29,16 +29,16 @@
         "license_file": "LICENSE.bzip2.txt",
     },
     "clang": {
-        "url": "https://github.com/llvm/llvm-project/releases/download/llvmorg-12.0.1/clang-12.0.1.src.tar.xz",
-        "size": 15323860,
-        "sha256": "6e912133bcf56e9cfe6a346fa7e5c52c2cde3e4e48b7a6cc6fcc7c75047da45f",
-        "version": "12.0.1",
+        "url": "https://github.com/llvm/llvm-project/releases/download/llvmorg-13.0.0/clang-13.0.0.src.tar.xz",
+        "size": 17846828,
+        "sha256": "5d611cbb06cfb6626be46eb2f23d003b2b80f40182898daa54b1c4e8b5b9e17e",
+        "version": "13.0.0",
     },
     "clang-compiler-rt": {
-        "url": "https://github.com/llvm/llvm-project/releases/download/llvmorg-12.0.1/compiler-rt-12.0.1.src.tar.xz",
-        "size": 2201284,
-        "sha256": "b4c8d5f2a802332987c1c0a95b5afb35b1a66a96fe44add4e4ed4792c4cba0a4",
-        "version": "12.0.1",
+        "url": "https://github.com/llvm/llvm-project/releases/download/llvmorg-13.0.0/compiler-rt-13.0.0.src.tar.xz",
+        "size": 2287616,
+        "sha256": "4c3602d76c7868a96b30c36165c4b7643e2a20173fced7e071b4baeb2d74db3f",
+        "version": "13.0.0",
     },
     "cmake-linux-bin": {
         "url": "https://github.com/Kitware/CMake/releases/download/v3.19.2/cmake-3.19.2-Linux-x86_64.tar.gz",
@@ -126,16 +126,16 @@
         "version": "1.0.7",
     },
     "libc++": {
-        "url": "https://github.com/llvm/llvm-project/releases/download/llvmorg-12.0.1/libcxx-12.0.1.src.tar.xz",
-        "size": 1882840,
-        "sha256": "a42089cd358f661823c490741f8be98701d983a7ee5152c8649075da681a9d15",
-        "version": "12.0.1",
+        "url": "https://github.com/llvm/llvm-project/releases/download/llvmorg-13.0.0/libcxx-13.0.0.src.tar.xz",
+        "size": 2086032,
+        "sha256": "3682f16ce33bb0a8951fc2c730af2f9b01a13b71b2b0dc1ae1e7034c7d86ca1a",
+        "version": "13.0.0",
     },
     "libc++abi": {
-        "url": "https://github.com/llvm/llvm-project/releases/download/llvmorg-12.0.1/libcxxabi-12.0.1.src.tar.xz",
-        "size": 552980,
-        "sha256": "88efe8e391767a1e8f42b509879abe766c9f44b1781ad1900975ae6b516b91d0",
-        "version": "12.0.1",
+        "url": "https://github.com/llvm/llvm-project/releases/download/llvmorg-13.0.0/libcxxabi-13.0.0.src.tar.xz",
+        "size": 554780,
+        "sha256": "becd5f1cd2c03cd6187558e9b4dc8a80b6d774ff2829fede88aa1576c5234ce3",
+        "version": "13.0.0",
     },
     "libedit": {
         "url": "https://thrysoee.dk/editline/libedit-20210910-3.1.tar.gz",
@@ -163,10 +163,10 @@
         "version": "0.1",
     },
     "libunwind": {
-        "url": "https://github.com/llvm/llvm-project/releases/download/llvmorg-12.0.1/libunwind-12.0.1.src.tar.xz",
-        "size": 98348,
-        "sha256": "0bea6089518395ca65cf58b0a450716c5c99ce1f041079d3aa42d280ace15ca4",
-        "version": "12.0.1",
+        "url": "https://github.com/llvm/llvm-project/releases/download/llvmorg-13.0.0/libunwind-13.0.0.src.tar.xz",
+        "size": 99668,
+        "sha256": "36f819091216177a61da639244eda67306ccdd904c757d70d135e273278b65e1",
+        "version": "13.0.0",
     },
     "libX11": {
         "url": "https://www.x.org/archive/individual/lib/libX11-1.6.8.tar.gz",
@@ -196,16 +196,16 @@
         "license_file": "LICENSE.libxcb.txt",
     },
     "lld": {
-        "url": "https://github.com/llvm/llvm-project/releases/download/llvmorg-12.0.1/lld-12.0.1.src.tar.xz",
-        "size": 1351580,
-        "sha256": "690b3f6a76310e13a783a142f87500ade9cafe003e088b678364487ed873e361",
-        "version": "12.0.1",
+        "url": "https://github.com/llvm/llvm-project/releases/download/llvmorg-13.0.0/lld-13.0.0.src.tar.xz",
+        "size": 1472476,
+        "sha256": "20d1900bcd64ff62047291f6edb6ba2fed34d782675ff68713bf0c2fc9e69386",
+        "version": "13.0.0",
     },
     "llvm": {
-        "url": "https://github.com/llvm/llvm-project/releases/download/llvmorg-12.0.1/llvm-12.0.1.src.tar.xz",
-        "size": 42898504,
-        "sha256": "7d9a8405f557cefc5a21bf5672af73903b64749d9bc3a50322239f56f34ffddf",
-        "version": "12.0.1",
+        "url": "https://github.com/llvm/llvm-project/releases/download/llvmorg-13.0.0/llvm-13.0.0.src.tar.xz",
+        "size": 45471992,
+        "sha256": "408d11708643ea826f519ff79761fcdfc12d641a2510229eec459e72f8163020",
+        "version": "13.0.0",
     },
     "mpc": {
         "url": "http://www.multiprecision.org/downloads/mpc-1.0.3.tar.gz",