From f786c8364992b6dacfcfdb0b601c6f3b1e1bfbf1 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sat, 20 Apr 2024 21:05:25 +0000 Subject: [PATCH 01/18] chore(deps): update dependency ziglang/zig to v0.12.0 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index d31489e..d11c155 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,7 @@ ARG TARGET="x86_64-linux-gnu" ARG VERSION="0.0.0" # renovate: datasource=github-releases depName=ziglang/zig -ARG ZIG_VERSION='0.11.0' +ARG ZIG_VERSION='0.12.0' # renovate: datasource=github-releases depName=mesonbuild/meson ARG MESON_VERSION='1.4.0' # renovate: datasource=github-releases depName=Kitware/CMake From 12f9e8cb8f69dfdbe47c031fbb297097ff1a2888 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADtor=20Vasconcellos?= Date: Mon, 22 Apr 2024 18:11:11 -0300 Subject: [PATCH 02/18] Remove Workarounf for LTO bugs when compiling C++ for windows Issue was resolved in zig 0.12 --- scripts/toolchain.sh | 9 --------- 1 file changed, 9 deletions(-) diff --git a/scripts/toolchain.sh b/scripts/toolchain.sh index b4d4e90..5f70ce3 100755 --- a/scripts/toolchain.sh +++ b/scripts/toolchain.sh @@ -188,12 +188,3 @@ EOF ln -s "gcc_eh.pc.pc" "${PREFIX}/lib/pkgconfig/libgcc_eh.pc" ;; esac - -case "$TARGET" in - *windows-gnu) - # Work around LTO bugs when compiling C++ for windows targets - # https://github.com/ziglang/zig/issues/15958#issuecomment-1764915440 - sed -i '/_free_locale))(_locale_t)/s/^/__attribute__((used)) /' "${SYSROOT}/lib/libc/mingw/misc/_free_locale.c" - sed -i '/_create_locale))(int, const char \*)/s/^/__attribute__((used)) /' "${SYSROOT}/lib/libc/mingw/misc/_create_locale.c" - ;; -esac From 4b4c5714b8b60d6847d25cb041b1772cdca0e01d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADtor=20Vasconcellos?= Date: Mon, 22 Apr 2024 18:22:00 -0300 Subject: [PATCH 03/18] Change minimum Glibc to 2.17 - Make compiled bundle compatible with RHEL 7.9 and CentOS 7, which should be the lower floor to be compatible with virtually every Linux distribution - Zig fixed issue that caused C++ code to fail compiling with Glibc versions lower than 2.18 --- scripts/cc.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/scripts/cc.sh b/scripts/cc.sh index 967b178..3e2daf1 100755 --- a/scripts/cc.sh +++ b/scripts/cc.sh @@ -5,10 +5,8 @@ set -euo pipefail case "${TARGET:?TARGET envvar is required to be defined}" in x86_64-linux-musl | aarch64-linux-musl) ;; x86_64-linux-gnu* | aarch64-linux-gnu*) - # Set the glibc minimum version - # This is the lowest we can go at the moment - # https://github.com/ziglang/zig/issues/9412 - TARGET="${TARGET%%.*}.2.18" + # Set the glibc minimum version, RHEL-7.9 and CentOS 7 + TARGET="${TARGET%%.*}.2.17" ;; x86_64-darwin-apple | x86_64-apple-darwin-macho) SDKROOT="$MACOS_SDKROOT" From e3ff15647bf7d37554087548bd04981217805063 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADtor=20Vasconcellos?= Date: Mon, 22 Apr 2024 18:44:18 -0300 Subject: [PATCH 04/18] Intall LLVM 17 Zig 0.12 uses LLVM 17 as base --- Dockerfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index d11c155..9d8609a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,12 +32,12 @@ RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selectio RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/var/lib/apt ` apt-get update && apt-get upgrade && apt-get install -y ca-certificates -# Add LLVM 16 repository +# Add LLVM 17 repository ADD https://apt.llvm.org/llvm-snapshot.gpg.key /etc/apt/trusted.gpg.d/apt.llvm.org.asc RUN chmod 644 /etc/apt/trusted.gpg.d/apt.llvm.org.asc -RUN echo "deb https://apt.llvm.org/bookworm/ llvm-toolchain-bookworm-16 main" ` +RUN echo "deb https://apt.llvm.org/bookworm/ llvm-toolchain-bookworm-17 main" ` > /etc/apt/sources.list.d/llvm.list # Install build dependencies @@ -50,9 +50,9 @@ RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/var/lib/ libtool ` python3 ` gettext ` - llvm-16 ` + llvm-17 ` autoconf ` - clang-16 ` + clang-17 ` pkg-config ` ninja-build ` libarchive-tools ` @@ -204,7 +204,7 @@ RUN --mount=type=cache,target=/root/.cache ` FROM layer-00 AS layer-10 COPY --from=layer-10-sse2neon "${PREFIX}/." "$PREFIX" -COPY --from=layer-10-compiler-rt "/usr/lib/llvm-16/lib/clang/16/." '/usr/lib/llvm-16/lib/clang/16' +COPY --from=layer-10-compiler-rt "/usr/lib/llvm-17/lib/clang/17/." '/usr/lib/llvm-17/lib/clang/17' #-- From 8f7e7ef35b86e6ad7523903c04e4887a0f07baaf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADtor=20Vasconcellos?= Date: Mon, 22 Apr 2024 18:48:12 -0300 Subject: [PATCH 05/18] Update compiler-rt darwin patch to LLVM 17 --- stages/10-compiler-rt.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/stages/10-compiler-rt.sh b/stages/10-compiler-rt.sh index ed195fd..183d158 100644 --- a/stages/10-compiler-rt.sh +++ b/stages/10-compiler-rt.sh @@ -9,7 +9,7 @@ case "$TARGET" in esac # LLVM install path -LLVM_PATH="/usr/lib/llvm-16" +LLVM_PATH="/usr/lib/llvm-17" # Remove wrapper from PATH, because we need to call the original cmake PATH="$(echo "${PATH}" | awk -v RS=: -v ORS=: '/\/wrapper^/ {next} {print}')" @@ -19,9 +19,9 @@ echo "Download llvm compiler_rt..." mkdir -p "${LLVM_PATH}/compiler_rt/build" -curl_tar 'https://github.com/llvm/llvm-project/releases/download/llvmorg-16.0.6/cmake-16.0.6.src.tar.xz' \ +curl_tar 'https://github.com/llvm/llvm-project/releases/download/llvmorg-17.0.6/cmake-17.0.6.src.tar.xz' \ "${LLVM_PATH}/cmake" 1 -curl_tar 'https://github.com/llvm/llvm-project/releases/download/llvmorg-16.0.6/compiler-rt-16.0.6.src.tar.xz' \ +curl_tar 'https://github.com/llvm/llvm-project/releases/download/llvmorg-17.0.6/compiler-rt-17.0.6.src.tar.xz' \ "${LLVM_PATH}/compiler_rt" 1 # Link cmake files to where compiler_rt expect to find them @@ -39,7 +39,7 @@ cmake \ -DLLVM_CMAKE_DIR="${LLVM_PATH}/cmake" \ -DDARWIN_osx_ARCHS="$(if [ "$_arch" == 'aarch64' ]; then echo 'arm64'; else echo "$_arch"; fi)" \ -DLLVM_MAIN_SRC_DIR="$LLVM_PATH" \ - -DCMAKE_INSTALL_PREFIX="${LLVM_PATH}/lib/clang/16" \ + -DCMAKE_INSTALL_PREFIX="${LLVM_PATH}/lib/clang/17" \ -DCMAKE_TOOLCHAIN_FILE='/srv/toolchain.cmake' \ -DDARWIN_macosx_SYSROOT="${MACOS_SDKROOT:?Missing macOS SDK path}" \ -DDARWIN_osx_BUILTIN_ARCHS="$(if [ "$_arch" == 'aarch64' ]; then echo 'arm64'; else echo "$_arch"; fi)" \ @@ -61,4 +61,4 @@ while IFS= read -r _lib; do if [ "$_arch" == 'aarch64' ]; then ln -s "${_lib_name}.a" "$(dirname "${_lib}")/${_lib_name}-arm64.a" fi -done < <(find "${LLVM_PATH}/lib/clang/16/lib/darwin/" -name 'libclang_rt.*') +done < <(find "${LLVM_PATH}/lib/clang/17/lib/darwin/" -name 'libclang_rt.*') From c54cde00634741294b081bbe32e1f3a955d79986 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADtor=20Vasconcellos?= Date: Mon, 22 Apr 2024 21:20:32 -0300 Subject: [PATCH 06/18] A lot more of LLVM 16 to 17 --- README.md | 2 +- scripts/cc.sh | 4 ++-- scripts/rc.sh | 2 +- scripts/tool-wrapper.sh | 6 +++--- stages/00-apple/01-xar.sh | 4 ++-- stages/00-apple/02-tapi.sh | 4 ++-- stages/00-apple/03-dispatch.sh | 4 ++-- stages/00-apple/04-cctools.sh | 8 ++++---- stages/50-x265.sh | 2 +- stages/99-ffmpeg.sh | 4 ++-- 10 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 034517c..672f579 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ It uses Zig 0.11 as a C/C++ toolchain to build the Windows and Linux targets: It uses LLVM/Clang 16 with some tweaks from osxcross + Apple's cctools and linker to build the Darwin (macOS) targets: -- https://github.com/llvm/llvm-project/tree/llvmorg-16.0.6 +- https://github.com/llvm/llvm-project/tree/llvmorg-17.0.6 - https://github.com/tpoechtrager/osxcross - https://github.com/tpoechtrager/cctools-port diff --git a/scripts/cc.sh b/scripts/cc.sh index 3e2daf1..60aa3d6 100755 --- a/scripts/cc.sh +++ b/scripts/cc.sh @@ -29,7 +29,7 @@ case "$(basename "$0")" in case "$TARGET" in *darwin*) # Use clang instead of zig for darwin targets - CMD='clang-16' + CMD='clang-17' ;; *) CMD='zig cc' ;; esac @@ -39,7 +39,7 @@ case "$(basename "$0")" in case "$TARGET" in *darwin*) # Use clang instead of zig for darwin targets - CMD='clang++-16' + CMD='clang++-17' ;; *) CMD='zig c++' ;; esac diff --git a/scripts/rc.sh b/scripts/rc.sh index a469a9b..2039bd0 100755 --- a/scripts/rc.sh +++ b/scripts/rc.sh @@ -31,4 +31,4 @@ case "$_name" in ;; esac -exec "/usr/bin/llvm-${_name}-16" "$@" +exec "/usr/bin/llvm-${_name}-17" "$@" diff --git a/scripts/tool-wrapper.sh b/scripts/tool-wrapper.sh index 6dc8922..cf3fce6 100755 --- a/scripts/tool-wrapper.sh +++ b/scripts/tool-wrapper.sh @@ -6,9 +6,9 @@ fallback_llvm() { if ! command -v "${_prefix}$(basename "$0")" 1>/dev/null 2>&1; then _prefix='llvm-' if [ "$0" = 'libtool' ]; then - _suffix='-darwin-16' + _suffix='-darwin-17' else - _suffix='-16' + _suffix='-17' fi fi } @@ -26,7 +26,7 @@ case "${TARGET:?TARGET envvar is required to be defined}" in ;; *) _prefix='llvm-' - _suffix='-16' + _suffix='-17' ;; esac ;; diff --git a/stages/00-apple/01-xar.sh b/stages/00-apple/01-xar.sh index 859b187..b65b72e 100755 --- a/stages/00-apple/01-xar.sh +++ b/stages/00-apple/01-xar.sh @@ -11,8 +11,8 @@ esac apt-get install libssl-dev libz-dev -export CC="clang-16" -export CXX="clang++-16" +export CC="clang-17" +export CXX="clang++-17" export CFLAGS="-I${CCTOOLS}/include" export LDFLAGS="-L${CCTOOLS}/lib" export APPLE_TARGET='__BYPASS__' diff --git a/stages/00-apple/02-tapi.sh b/stages/00-apple/02-tapi.sh index 7595de1..2998980 100755 --- a/stages/00-apple/02-tapi.sh +++ b/stages/00-apple/02-tapi.sh @@ -9,8 +9,8 @@ case "$TARGET" in ;; esac -export CC="clang-16" -export CXX="clang++-16" +export CC="clang-17" +export CXX="clang++-17" export CFLAGS="-I${CCTOOLS}/include" export LDFLAGS="-L${CCTOOLS}/lib" export APPLE_TARGET='__BYPASS__' diff --git a/stages/00-apple/03-dispatch.sh b/stages/00-apple/03-dispatch.sh index 4a9ae23..23eaf9e 100755 --- a/stages/00-apple/03-dispatch.sh +++ b/stages/00-apple/03-dispatch.sh @@ -11,8 +11,8 @@ esac apt-get install systemtap-sdt-dev libbsd-dev linux-libc-dev -export CC="clang-16" -export CXX="clang++-16" +export CC="clang-17" +export CXX="clang++-17" export CFLAGS="-I${CCTOOLS}/include" export LDFLAGS="-L${CCTOOLS}/lib" export APPLE_TARGET='__BYPASS__' diff --git a/stages/00-apple/04-cctools.sh b/stages/00-apple/04-cctools.sh index 39fdbac..4c744d9 100755 --- a/stages/00-apple/04-cctools.sh +++ b/stages/00-apple/04-cctools.sh @@ -18,8 +18,8 @@ echo "APPLE_TARGET=$_target" >>/etc/environment apt-get install uuid-dev libedit-dev -export CC="clang-16" -export CXX="clang++-16" +export CC="clang-17" +export CXX="clang++-17" export CFLAGS="-I${CCTOOLS}/include" export LDFLAGS="-L${CCTOOLS}/lib" export APPLE_TARGET='__BYPASS__' @@ -43,7 +43,7 @@ cd cctools/cctools --with-libxar="$CCTOOLS" \ --with-libtapi="$CCTOOLS" \ --with-libdispatch="$CCTOOLS" \ - --with-llvm-config=llvm-config-16 \ + --with-llvm-config=llvm-config-17 \ --with-libblocksruntime="$CCTOOLS" \ --enable-xar-support \ --enable-lto-support \ @@ -56,4 +56,4 @@ make install rm -r /srv/cctools # Create symlinks for llvm-otool because cctools use it when calling its own otool -ln -fs "$(command -v llvm-otool-16)" /opt/cctools/bin/llvm-otool +ln -fs "$(command -v llvm-otool-17)" /opt/cctools/bin/llvm-otool diff --git a/stages/50-x265.sh b/stages/50-x265.sh index 7d1f3c6..0575160 100644 --- a/stages/50-x265.sh +++ b/stages/50-x265.sh @@ -74,7 +74,7 @@ ln -s ../10bit/libx265.a libx265_main10.a mv libx265.a libx265_main.a # Must use llvm ar due to mri-script -llvm-ar-16 -M < Date: Tue, 23 Apr 2024 10:33:00 -0300 Subject: [PATCH 07/18] Remove __ENVIRONMENT_OS_VERSION_MIN_REQUIRED__ manual definition LLVM 17 does this automatically --- scripts/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build.sh b/scripts/build.sh index 0c2e8cf..54478d1 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -95,7 +95,7 @@ case "$TARGET" in FFLAGS="${FFLAGS} -fstack-check" # https://github.com/tpoechtrager/osxcross/commit/3279f86 - CFLAGS="${CFLAGS} -D__ENVIRONMENT_OS_VERSION_MIN_REQUIRED__=$(LC_ALL=C printf '%.2f' "11.0" | tr -d '.') -mmacos-version-min=${MACOSX_DEPLOYMENT_TARGET} -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" + CFLAGS="${CFLAGS} -mmacos-version-min=${MACOSX_DEPLOYMENT_TARGET} -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" LDFLAGS="-fuse-ld=$(command -v "${APPLE_TARGET:?}-ld") -L${SDKROOT}/usr/lib -L${SDKROOT}/usr/lib/system -F${SDKROOT}/System/Library/Frameworks ${LDFLAGS}" ;; *windows*) From af024f180c580b38f0c6009b3b542a20dd399608 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADtor=20Vasconcellos?= Date: Tue, 23 Apr 2024 13:49:59 -0300 Subject: [PATCH 08/18] Fix eigen deprecated-this-capture --- stages/99-onnx.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/stages/99-onnx.sh b/stages/99-onnx.sh index cda420a..af9ce19 100644 --- a/stages/99-onnx.sh +++ b/stages/99-onnx.sh @@ -112,6 +112,9 @@ case "$TARGET" in sed -i 's/foreach (s IN ITEMS ${NSYNC_COMMON_SRC} ${NSYNC_OS_CPP_SRC})/foreach (s IN ITEMS ${NSYNC_COMMON_SRC} ${NSYNC_OS_CPP_SRC})\nget_filename_component(sle ${s} NAME_WLE)/g' _deps/google_nsync-src/CMakeLists.txt sed -i 's/cpp\/${s}/cpp\/${sle}.cc/g' _deps/google_nsync-src/CMakeLists.txt + # Fix eigen deprecated-this-capture + sed '/ei_add_cxx_compiler_flag("-Wno-long-long")/a \ ei_add_cxx_compiler_flag("-Wno-deprecated-this-capture")' _deps/eigen-src/CMakeLists.txt + # Regenerate build files after cmake patches env PREFIX="$OUT" cmake "${args[@]}" ../cmake ;; From 928ae1bbe91b9d6408315b4cfff36569fe7e8321 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADtor=20Vasconcellos?= Date: Tue, 23 Apr 2024 14:18:06 -0300 Subject: [PATCH 09/18] eigen patch must run for all targets --- stages/99-onnx.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/stages/99-onnx.sh b/stages/99-onnx.sh index af9ce19..078c125 100644 --- a/stages/99-onnx.sh +++ b/stages/99-onnx.sh @@ -111,15 +111,15 @@ case "$TARGET" in # Fix google_nsync trying to compile C code as C++, which zig does not support sed -i 's/foreach (s IN ITEMS ${NSYNC_COMMON_SRC} ${NSYNC_OS_CPP_SRC})/foreach (s IN ITEMS ${NSYNC_COMMON_SRC} ${NSYNC_OS_CPP_SRC})\nget_filename_component(sle ${s} NAME_WLE)/g' _deps/google_nsync-src/CMakeLists.txt sed -i 's/cpp\/${s}/cpp\/${sle}.cc/g' _deps/google_nsync-src/CMakeLists.txt - - # Fix eigen deprecated-this-capture - sed '/ei_add_cxx_compiler_flag("-Wno-long-long")/a \ ei_add_cxx_compiler_flag("-Wno-deprecated-this-capture")' _deps/eigen-src/CMakeLists.txt - - # Regenerate build files after cmake patches - env PREFIX="$OUT" cmake "${args[@]}" ../cmake ;; esac +# Fix eigen deprecated-this-capture +sed '/ei_add_cxx_compiler_flag("-Wno-long-long")/a \ ei_add_cxx_compiler_flag("-Wno-deprecated-this-capture")' _deps/eigen-src/CMakeLists.txt + +# Regenerate build files after cmake patches +env PREFIX="$OUT" cmake "${args[@]}" ../cmake + ninja -j"$(nproc)" ninja install From fb09ce5d53bcc7ff809c0b04d71af512c5ba179d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADtor=20Vasconcellos?= Date: Tue, 23 Apr 2024 14:53:59 -0300 Subject: [PATCH 10/18] Forgot the -i for the sed cmd --- stages/99-onnx.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stages/99-onnx.sh b/stages/99-onnx.sh index 078c125..fa632a8 100644 --- a/stages/99-onnx.sh +++ b/stages/99-onnx.sh @@ -115,7 +115,7 @@ case "$TARGET" in esac # Fix eigen deprecated-this-capture -sed '/ei_add_cxx_compiler_flag("-Wno-long-long")/a \ ei_add_cxx_compiler_flag("-Wno-deprecated-this-capture")' _deps/eigen-src/CMakeLists.txt +sed -i '/ei_add_cxx_compiler_flag("-Wno-long-long")/a \ ei_add_cxx_compiler_flag("-Wno-deprecated-this-capture")' _deps/eigen-src/CMakeLists.txt # Regenerate build files after cmake patches env PREFIX="$OUT" cmake "${args[@]}" ../cmake From 7ec99ee18b51ce7469eb4005e95cff41692861d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADtor=20Vasconcellos?= Date: Tue, 23 Apr 2024 22:12:40 -0300 Subject: [PATCH 11/18] Try to fix eigen deprecated-this-capture --- stages/99-onnx.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/stages/99-onnx.sh b/stages/99-onnx.sh index fa632a8..7471ee8 100644 --- a/stages/99-onnx.sh +++ b/stages/99-onnx.sh @@ -115,7 +115,11 @@ case "$TARGET" in esac # Fix eigen deprecated-this-capture -sed -i '/ei_add_cxx_compiler_flag("-Wno-long-long")/a \ ei_add_cxx_compiler_flag("-Wno-deprecated-this-capture")' _deps/eigen-src/CMakeLists.txt +sed -i 's/pool_->Schedule([=, &handleRange]() { handleRange(midIdx, lastIdx); });/pool_->Schedule([=, this, &handleRange]() { handleRange(midIdx, lastIdx); });/' _deps/eigen-src/unsupported/Eigen/CXX11/src/Tensor/TensorDeviceThreadPool.h +sed -i 's/handleRange = [=, &handleRange, &barrier, &f](Index firstIdx,/handleRange = [=, this, &handleRange, &barrier, &f](Index firstIdx,/' _deps/eigen-src/unsupported/Eigen/CXX11/src/Tensor/TensorDeviceThreadPool.h +sed -i 's/[=]() { kernel(m, n, k, use_thread_local); });/[=, this]() { kernel(m, n, k, use_thread_local); });/' _deps/eigen-src/unsupported/Eigen/CXX11/src/Tensor/TensorContractionThreadPool.h +sed -i 's/[=]() { enqueue_packing_helper(mid, end, k, rhs); });/[=, this]() { enqueue_packing_helper(mid, end, k, rhs); });/' _deps/eigen-src/unsupported/Eigen/CXX11/src/Tensor/TensorContractionThreadPool.h +sed -i 's/[=]() { enqueue_packing_helper(start, end, k, rhs); });/[=, this]() { enqueue_packing_helper(start, end, k, rhs); });/' _deps/eigen-src/unsupported/Eigen/CXX11/src/Tensor/TensorContractionThreadPool.h # Regenerate build files after cmake patches env PREFIX="$OUT" cmake "${args[@]}" ../cmake From 92e8f2e1b3484ac12565e179732d5a4a5f290d04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADtor=20Vasconcellos?= Date: Sat, 18 May 2024 02:42:59 -0300 Subject: [PATCH 12/18] Make patch to fix eigen deprecated-this-capture --- Dockerfile | 1 + .../eigen_fix_deprecated_this_capture.patch | 50 +++++++++++++++++++ stages/99-onnx.sh | 6 +-- 3 files changed, 52 insertions(+), 5 deletions(-) create mode 100644 patches/99-onnx/eigen_fix_deprecated_this_capture.patch diff --git a/Dockerfile b/Dockerfile index 9d8609a..933fafa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -442,6 +442,7 @@ RUN --mount=type=cache,target=/root/.cache ` FROM layer-20 AS layer-99-onnx RUN --mount=type=cache,target=/root/.cache ` + --mount=type=bind,source=patches/99-onnx,target="${PREFIX}/patches" ` --mount=type=bind,source=stages/99-onnx.sh,target=/srv/stage.sh ` /srv/build.sh diff --git a/patches/99-onnx/eigen_fix_deprecated_this_capture.patch b/patches/99-onnx/eigen_fix_deprecated_this_capture.patch new file mode 100644 index 0000000..1dc62a4 --- /dev/null +++ b/patches/99-onnx/eigen_fix_deprecated_this_capture.patch @@ -0,0 +1,50 @@ +diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorContractionThreadPool.h b/unsupported/Eigen/CXX11/src/Tensor/TensorContractionThreadPool.h +index 21be6ea42..7464ee56a 100644 +--- a/unsupported/Eigen/CXX11/src/Tensor/TensorContractionThreadPool.h ++++ b/unsupported/Eigen/CXX11/src/Tensor/TensorContractionThreadPool.h +@@ -1006,7 +1006,7 @@ struct TensorEvaluator 1) { + Index mid = (start + end) / 2; + device_.enqueueNoNotification( +- [=]() { enqueue_packing_helper(mid, end, k, rhs); }); ++ [=, this]() { enqueue_packing_helper(mid, end, k, rhs); }); + end = mid; + } + +@@ -1079,7 +1079,7 @@ struct TensorEvaluator(block.count)); + std::function handleRange; +- handleRange = [=, &handleRange, &barrier, &f](Index firstIdx, ++ handleRange = [=, this, &handleRange, &barrier, &f](Index firstIdx, + Index lastIdx) { + while (lastIdx - firstIdx > block.size) { + // Split into halves and schedule the second half on a different thread. + const Index midIdx = firstIdx + divup((lastIdx - firstIdx) / 2, block.size) * block.size; +- pool_->Schedule([=, &handleRange]() { handleRange(midIdx, lastIdx); }); ++ pool_->Schedule([=, this, &handleRange]() { handleRange(midIdx, lastIdx); }); + lastIdx = midIdx; + } + // Single block or less, execute directly. diff --git a/stages/99-onnx.sh b/stages/99-onnx.sh index 7471ee8..89f9b73 100644 --- a/stages/99-onnx.sh +++ b/stages/99-onnx.sh @@ -115,11 +115,7 @@ case "$TARGET" in esac # Fix eigen deprecated-this-capture -sed -i 's/pool_->Schedule([=, &handleRange]() { handleRange(midIdx, lastIdx); });/pool_->Schedule([=, this, &handleRange]() { handleRange(midIdx, lastIdx); });/' _deps/eigen-src/unsupported/Eigen/CXX11/src/Tensor/TensorDeviceThreadPool.h -sed -i 's/handleRange = [=, &handleRange, &barrier, &f](Index firstIdx,/handleRange = [=, this, &handleRange, &barrier, &f](Index firstIdx,/' _deps/eigen-src/unsupported/Eigen/CXX11/src/Tensor/TensorDeviceThreadPool.h -sed -i 's/[=]() { kernel(m, n, k, use_thread_local); });/[=, this]() { kernel(m, n, k, use_thread_local); });/' _deps/eigen-src/unsupported/Eigen/CXX11/src/Tensor/TensorContractionThreadPool.h -sed -i 's/[=]() { enqueue_packing_helper(mid, end, k, rhs); });/[=, this]() { enqueue_packing_helper(mid, end, k, rhs); });/' _deps/eigen-src/unsupported/Eigen/CXX11/src/Tensor/TensorContractionThreadPool.h -sed -i 's/[=]() { enqueue_packing_helper(start, end, k, rhs); });/[=, this]() { enqueue_packing_helper(start, end, k, rhs); });/' _deps/eigen-src/unsupported/Eigen/CXX11/src/Tensor/TensorContractionThreadPool.h +patch -F5 -lp1 -d _deps/eigen-src -t <"$PREFIX"/patches/eigen_fix_deprecated_this_capture.patch # Regenerate build files after cmake patches env PREFIX="$OUT" cmake "${args[@]}" ../cmake From c76c2ca21a13f4b85026869c59789f493ba40869 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADtor=20Vasconcellos?= Date: Sat, 18 May 2024 03:33:12 -0300 Subject: [PATCH 13/18] Remove -Werror from eigen - Add some more ignored flags for zig in cc wrapper scripts --- .../eigen_fix_deprecated_this_capture.patch | 19 +++++ scripts/build.sh | 4 +- scripts/cc.sh | 78 ++++++++++++------- 3 files changed, 73 insertions(+), 28 deletions(-) diff --git a/patches/99-onnx/eigen_fix_deprecated_this_capture.patch b/patches/99-onnx/eigen_fix_deprecated_this_capture.patch index 1dc62a4..a8a0434 100644 --- a/patches/99-onnx/eigen_fix_deprecated_this_capture.patch +++ b/patches/99-onnx/eigen_fix_deprecated_this_capture.patch @@ -1,3 +1,22 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index f40cf7738..bd6599c9f 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -147,10 +147,10 @@ if(NOT MSVC) + + # clang outputs some warnings for unknown flags that are not caught by check_cxx_compiler_flag + # adding -Werror turns such warnings into errors +- check_cxx_compiler_flag("-Werror" COMPILER_SUPPORT_WERROR) +- if(COMPILER_SUPPORT_WERROR) +- set(CMAKE_REQUIRED_FLAGS "-Werror") +- endif() ++ #check_cxx_compiler_flag("-Werror" COMPILER_SUPPORT_WERROR) ++ #if(COMPILER_SUPPORT_WERROR) ++ # set(CMAKE_REQUIRED_FLAGS "-Werror") ++ #endif() + ei_add_cxx_compiler_flag("-pedantic") + ei_add_cxx_compiler_flag("-Wall") + ei_add_cxx_compiler_flag("-Wextra") diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorContractionThreadPool.h b/unsupported/Eigen/CXX11/src/Tensor/TensorContractionThreadPool.h index 21be6ea42..7464ee56a 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorContractionThreadPool.h diff --git a/scripts/build.sh b/scripts/build.sh index 54478d1..2077160 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -100,10 +100,10 @@ case "$TARGET" in ;; *windows*) # Zig doesn't support stack probing on Windows - # https://github.com/ziglang/zig/blob/b3462b7/src/target.zig#L326-L329 + # https://github.com/ziglang/zig/blob/0.12.0/src/target.zig#L195-L198 FFLAGS="${FFLAGS} -fno-stack-check" # https://github.com/strukturag/libheif/issues/357 - CFLAGS="${CFLAGS} -D_GLIBCXX_ASSERTIONS -D__MINGW64__" + CFLAGS="${CFLAGS} -D_GLIBCXX_ASSERTIONS=1 -D__MINGW64__" ;; esac export CFLAGS="${CFLAGS} ${FFLAGS}" diff --git a/scripts/cc.sh b/scripts/cc.sh index 60aa3d6..83cc571 100755 --- a/scripts/cc.sh +++ b/scripts/cc.sh @@ -143,29 +143,42 @@ while [ "$#" -gt 0 ]; do cpu_features+=("${_features[@]}") true else - if (case "$TARGET" in *darwin*) exit 0 ;; *) exit 1 ;; esac) then - if [ "$1" = '-arch=aarch64' ]; then - # macOS uses arm64 instead of aarch64 - argv+=('-arch=arm64') - elif [ "$1" = '-arch' ] && [ "${2:-}" = 'aarch64' ]; then - argv+=('-arch' 'arm64') - shift 2 - continue - elif (case "$1" in -DTARGET_OS_IPHONE*) exit 0 ;; *) exit 1 ;; esac) then - has_iphone=1 - else - argv+=("$1") - - # See https://github.com/apple-oss-distributions/libiconv/blob/a167071/xcodeconfig/libiconv.xcconfig - if [ "$1" = '-lcharset' ]; then - should_add_libcharset=-1 - elif [ "$1" = '-liconv' ] && [ "$should_add_libcharset" -eq 0 ]; then - should_add_libcharset=1 + case "$TARGET" in + *darwin*) + if [ "$1" = '-arch=aarch64' ]; then + # macOS uses arm64 instead of aarch64 + argv+=('-arch=arm64') + elif [ "$1" = '-arch' ] && [ "${2:-}" = 'aarch64' ]; then + argv+=('-arch' 'arm64') + shift 2 + continue + elif (case "$1" in -DTARGET_OS_IPHONE*) exit 0 ;; *) exit 1 ;; esac) then + has_iphone=1 + else + argv+=("$1") + + # See https://github.com/apple-oss-distributions/libiconv/blob/a167071/xcodeconfig/libiconv.xcconfig + if [ "$1" = '-lcharset' ]; then + should_add_libcharset=-1 + elif [ "$1" = '-liconv' ] && [ "$should_add_libcharset" -eq 0 ]; then + should_add_libcharset=1 + fi fi - fi - else - argv+=("$1") - fi + ;; + *windows*) + if (case "$1" in *.rlib | *libcompiler_builtins-*) exit 0 ;; *) exit 1 ;; esac) then + # compiler-builtins is duplicated with zig's compiler-rt + case "$CMD" in + clang*) argv+=("$1") ;; + esac + else + argv+=("$1") + fi + ;; + *) + argv+=("$1") + ;; + esac fi if [ "$1" = '-E' ]; then @@ -220,6 +233,19 @@ while [ "$#" -gt 0 ]; do argv+=(-v) ;; esac + elif [ "$1" = '-Bdynamic' ]; then + case "$CMD" in + clang*) + l_args+=(-Bdynamic) + ;; + *) + # https://github.com/ziglang/zig/pull/16058 + # zig changes the linker behavior, -Bdynamic won't search *.a for mingw, but this may be fixed in the later version + # here is a workaround to replace the linker switch with -search_paths_first, which will search for *.dll,*lib first, + # then fallback to *.a + l_args+=(-search_paths_first) + ;; + esac elif [ "$1" = '-dynamic' ]; then case "$CMD" in clang*) @@ -252,12 +278,12 @@ while [ "$#" -gt 0 ]; do ;; esac elif [ "$1" = '-dylib' ] \ + || [ "$1" = '--dynamicbase' ] \ || [ "$1" = '--large-address-aware' ] \ - || [ "$1" = '--no-undefined-version' ] \ || [ "$1" = '--disable-auto-image-base' ]; then - # zig doesn't support -dylib, -dynamic, --no-undefined-version, --large-address-aware and --disable-auto-image-base - # https://github.com/ziglang/zig/issues/16855 - # https://github.com/ziglang/zig/pull/17326 + # zig doesn't support -dylib, --dynamicbase, --large-address-aware and --disable-auto-image-base + # https://github.com/rust-cross/cargo-zigbuild/blob/841df510dff8c585690e85354998bbcb6695460f/src/zig.rs#L187-L190 + # https://github.com/ziglang/zig/issues/10336 case "$CMD" in clang*) l_args+=("$1") From eba441c756a2db5b0e62a551ce4746f50263d07d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADtor=20Vasconcellos?= Date: Thu, 23 May 2024 02:23:46 -0300 Subject: [PATCH 14/18] Revert eigen patches in onnx --- Dockerfile | 1 - .../eigen_fix_deprecated_this_capture.patch | 69 ------------------- stages/99-onnx.sh | 9 +-- 3 files changed, 3 insertions(+), 76 deletions(-) delete mode 100644 patches/99-onnx/eigen_fix_deprecated_this_capture.patch diff --git a/Dockerfile b/Dockerfile index 933fafa..9d8609a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -442,7 +442,6 @@ RUN --mount=type=cache,target=/root/.cache ` FROM layer-20 AS layer-99-onnx RUN --mount=type=cache,target=/root/.cache ` - --mount=type=bind,source=patches/99-onnx,target="${PREFIX}/patches" ` --mount=type=bind,source=stages/99-onnx.sh,target=/srv/stage.sh ` /srv/build.sh diff --git a/patches/99-onnx/eigen_fix_deprecated_this_capture.patch b/patches/99-onnx/eigen_fix_deprecated_this_capture.patch deleted file mode 100644 index a8a0434..0000000 --- a/patches/99-onnx/eigen_fix_deprecated_this_capture.patch +++ /dev/null @@ -1,69 +0,0 @@ -diff --git a/CMakeLists.txt b/CMakeLists.txt -index f40cf7738..bd6599c9f 100644 ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -147,10 +147,10 @@ if(NOT MSVC) - - # clang outputs some warnings for unknown flags that are not caught by check_cxx_compiler_flag - # adding -Werror turns such warnings into errors -- check_cxx_compiler_flag("-Werror" COMPILER_SUPPORT_WERROR) -- if(COMPILER_SUPPORT_WERROR) -- set(CMAKE_REQUIRED_FLAGS "-Werror") -- endif() -+ #check_cxx_compiler_flag("-Werror" COMPILER_SUPPORT_WERROR) -+ #if(COMPILER_SUPPORT_WERROR) -+ # set(CMAKE_REQUIRED_FLAGS "-Werror") -+ #endif() - ei_add_cxx_compiler_flag("-pedantic") - ei_add_cxx_compiler_flag("-Wall") - ei_add_cxx_compiler_flag("-Wextra") -diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorContractionThreadPool.h b/unsupported/Eigen/CXX11/src/Tensor/TensorContractionThreadPool.h -index 21be6ea42..7464ee56a 100644 ---- a/unsupported/Eigen/CXX11/src/Tensor/TensorContractionThreadPool.h -+++ b/unsupported/Eigen/CXX11/src/Tensor/TensorContractionThreadPool.h -@@ -1006,7 +1006,7 @@ struct TensorEvaluator 1) { - Index mid = (start + end) / 2; - device_.enqueueNoNotification( -- [=]() { enqueue_packing_helper(mid, end, k, rhs); }); -+ [=, this]() { enqueue_packing_helper(mid, end, k, rhs); }); - end = mid; - } - -@@ -1079,7 +1079,7 @@ struct TensorEvaluator(block.count)); - std::function handleRange; -- handleRange = [=, &handleRange, &barrier, &f](Index firstIdx, -+ handleRange = [=, this, &handleRange, &barrier, &f](Index firstIdx, - Index lastIdx) { - while (lastIdx - firstIdx > block.size) { - // Split into halves and schedule the second half on a different thread. - const Index midIdx = firstIdx + divup((lastIdx - firstIdx) / 2, block.size) * block.size; -- pool_->Schedule([=, &handleRange]() { handleRange(midIdx, lastIdx); }); -+ pool_->Schedule([=, this, &handleRange]() { handleRange(midIdx, lastIdx); }); - lastIdx = midIdx; - } - // Single block or less, execute directly. diff --git a/stages/99-onnx.sh b/stages/99-onnx.sh index 89f9b73..295362b 100644 --- a/stages/99-onnx.sh +++ b/stages/99-onnx.sh @@ -112,13 +112,10 @@ case "$TARGET" in sed -i 's/foreach (s IN ITEMS ${NSYNC_COMMON_SRC} ${NSYNC_OS_CPP_SRC})/foreach (s IN ITEMS ${NSYNC_COMMON_SRC} ${NSYNC_OS_CPP_SRC})\nget_filename_component(sle ${s} NAME_WLE)/g' _deps/google_nsync-src/CMakeLists.txt sed -i 's/cpp\/${s}/cpp\/${sle}.cc/g' _deps/google_nsync-src/CMakeLists.txt ;; -esac - -# Fix eigen deprecated-this-capture -patch -F5 -lp1 -d _deps/eigen-src -t <"$PREFIX"/patches/eigen_fix_deprecated_this_capture.patch -# Regenerate build files after cmake patches -env PREFIX="$OUT" cmake "${args[@]}" ../cmake + # Regenerate build files after cmake patches + env PREFIX="$OUT" cmake "${args[@]}" ../cmake +esac ninja -j"$(nproc)" From 6cf4c96b2db8adddda826d1d9921a8aa8aea4ed4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADtor=20Vasconcellos?= Date: Thu, 23 May 2024 23:42:23 -0300 Subject: [PATCH 15/18] Fix onnx script --- stages/99-onnx.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stages/99-onnx.sh b/stages/99-onnx.sh index 295362b..cda420a 100644 --- a/stages/99-onnx.sh +++ b/stages/99-onnx.sh @@ -111,10 +111,10 @@ case "$TARGET" in # Fix google_nsync trying to compile C code as C++, which zig does not support sed -i 's/foreach (s IN ITEMS ${NSYNC_COMMON_SRC} ${NSYNC_OS_CPP_SRC})/foreach (s IN ITEMS ${NSYNC_COMMON_SRC} ${NSYNC_OS_CPP_SRC})\nget_filename_component(sle ${s} NAME_WLE)/g' _deps/google_nsync-src/CMakeLists.txt sed -i 's/cpp\/${s}/cpp\/${sle}.cc/g' _deps/google_nsync-src/CMakeLists.txt - ;; # Regenerate build files after cmake patches env PREFIX="$OUT" cmake "${args[@]}" ../cmake + ;; esac ninja -j"$(nproc)" From 25da1d4720cb6d3ab29bcff3d7675a9eb62d06b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADtor=20Vasconcellos?= Date: Tue, 28 May 2024 03:27:51 -0300 Subject: [PATCH 16/18] Ignore deprecated usage of this capture by eigen during darwin compiling --- stages/99-onnx.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/stages/99-onnx.sh b/stages/99-onnx.sh index cda420a..cc2f54b 100644 --- a/stages/99-onnx.sh +++ b/stages/99-onnx.sh @@ -99,7 +99,8 @@ case "$TARGET" in -Donnxruntime_USE_COREML=On ) # Allow deprecated usage of ATOMIC_VAR_INIT by https://github.com/google/nsync - export CXXFLAGS="${CXXFLAGS} -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS" + # Allow deprecated usage of this capture by https://gitlab.com/libeigen/eigen + export CXXFLAGS="${CXXFLAGS} -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS -Wno-deprecated-this-capture" ;; esac From 5be3cd6236d2cdfe3109f91caddcc1e4c63968eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADtor=20Vasconcellos?= Date: Tue, 28 May 2024 04:16:45 -0300 Subject: [PATCH 17/18] Disable i8mm ARM extension in FFmpeg for Linux and Windows --- stages/99-ffmpeg.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/stages/99-ffmpeg.sh b/stages/99-ffmpeg.sh index 678ce17..65b85af 100755 --- a/stages/99-ffmpeg.sh +++ b/stages/99-ffmpeg.sh @@ -141,6 +141,15 @@ case "$TARGET" in ;; esac +case "$TARGET" in + aarch64-linux* | aarch64-windows*) + # Our baseline ARM platform for Linux and Windows is raspberry pi 3 which doesn't support i8mm + env_specific_arg+=( + --disable-i8mm + ) + ;; +esac + _arch="${TARGET%%-*}" case "$TARGET" in aarch64-darwin*) From bf8466ccbc5c4ec4b851d7c76c20866ebe6c7f04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADtor=20Vasconcellos?= Date: Tue, 28 May 2024 04:53:37 -0300 Subject: [PATCH 18/18] Disable i8mm ARM extension in FFmpeg for macOS also --- stages/99-ffmpeg.sh | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/stages/99-ffmpeg.sh b/stages/99-ffmpeg.sh index 65b85af..ffd37b5 100755 --- a/stages/99-ffmpeg.sh +++ b/stages/99-ffmpeg.sh @@ -72,6 +72,8 @@ case "$TARGET" in --x86asmexe=false --enable-vfp --enable-neon + # Our baseline ARM platform for Linux and Windows is raspberry pi 3 which doesn't support i8mm + --disable-i8mm ) ;; esac @@ -141,15 +143,6 @@ case "$TARGET" in ;; esac -case "$TARGET" in - aarch64-linux* | aarch64-windows*) - # Our baseline ARM platform for Linux and Windows is raspberry pi 3 which doesn't support i8mm - env_specific_arg+=( - --disable-i8mm - ) - ;; -esac - _arch="${TARGET%%-*}" case "$TARGET" in aarch64-darwin*)