diff --git a/Dockerfile b/Dockerfile index 8e4eaf6..5ec85b7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ ARG OUT="/opt/out" ARG TARGET="x86_64-linux-gnu" ARG ZIG_VERSION="0.11.0" -ARG MESON_VERSION="1.2.3" +ARG MESON_VERSION="1.3.0" ARG CMAKE_VERSION="3.27.0" ARG PATCHELF_VERSION="0.18.0" ARG MACOS_SDK_VERSION="14.0" @@ -50,7 +50,8 @@ RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/var/lib/ clang-16 ` pkg-config ` ninja-build ` - libarchive-tools + libarchive-tools ` + protobuf-compiler # Configure sysroot and prefix ARG OUT @@ -58,6 +59,7 @@ ENV OUT="${OUT:?}" ENV PREFIX="/opt/prefix" ENV SYSROOT="/opt/sysroot" ENV CCTOOLS="/opt/cctools" +ENV CIPHERSUITES="TLS_AES_128_GCM_SHA256:TLS_CHACHA20_POLY1305_SHA256:TLS_AES_256_GCM_SHA384:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384" # Ensure sysroot and cctools are present on PATH ENV PATH="${CCTOOLS}/bin:${SYSROOT}/bin:$PATH" @@ -154,8 +156,12 @@ RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/var/lib/ /srv/02-tapi.sh RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/var/lib/apt --mount=type=cache,target=/root/.cache ` - --mount=type=bind,source=stages/00-apple/03-cctools.sh,target=/srv/03-cctools.sh ` - /srv/03-cctools.sh + --mount=type=bind,source=stages/00-apple/03-dispatch.sh,target=/srv/03-dispatch.sh ` + /srv/03-dispatch.sh + +RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/var/lib/apt --mount=type=cache,target=/root/.cache ` + --mount=type=bind,source=stages/00-apple/04-cctools.sh,target=/srv/04-cctools.sh ` + /srv/04-cctools.sh # Ensure no one tries to call the native system linker RUN ln -s '/usr/bin/false' "${SYSROOT}/bin/ld" @@ -409,12 +415,6 @@ COPY --from=layer-50-zimg "${PREFIX}/." "$PREFIX" #-- -FROM layer-45 AS layer-99-heif - -RUN --mount=type=cache,target=/root/.cache ` - --mount=type=bind,source=stages/99-heif.sh,target=/srv/stage.sh ` - /srv/build.sh - FROM layer-00 AS layer-99-protoc ADD https://raw.githubusercontent.com/protocolbuffers/protobuf/v25.0/LICENSE '/srv/protoc/LICENSE' @@ -429,12 +429,32 @@ RUN --mount=type=cache,target=/root/.cache ` --mount=type=bind,source=stages/99-pdfium.sh,target=/srv/stage.sh ` /srv/build.sh +FROM layer-00 AS layer-99-yolo + +RUN --mount=type=cache,target=/root/.cache ` + --mount=type=bind,source=stages/99-yolov8.sh,target=/srv/stage.sh ` + /srv/build.sh + +FROM layer-20 AS layer-99-onnx + +RUN --mount=type=cache,target=/root/.cache ` + --mount=type=bind,source=stages/99-onnx.sh,target=/srv/stage.sh ` + /srv/build.sh + FROM layer-50 AS layer-99-ffmpeg RUN --mount=type=cache,target=/root/.cache ` --mount=type=bind,source=stages/99-ffmpeg.sh,target=/srv/stage.sh ` /srv/build.sh +FROM layer-45 AS layer-99-heif + +COPY --from=layer-99-ffmpeg "${OUT}/." "$PREFIX" + +RUN --mount=type=cache,target=/root/.cache ` + --mount=type=bind,source=stages/99-heif.sh,target=/srv/stage.sh ` + /srv/build.sh + FROM layer-00 AS layer-99 COPY --from=layer-99-heif "${OUT}/." "$OUT" @@ -447,6 +467,12 @@ COPY --from=layer-99-protoc "${PREFIX}/licenses/." "${OUT}/licenses" COPY --from=layer-99-pdfium "${OUT}/." "$OUT" COPY --from=layer-99-pdfium "${PREFIX}/licenses/." "${OUT}/licenses" +COPY --from=layer-99-yolo "${OUT}/." "$OUT" + +COPY --from=layer-99-onnx "${OUT}/." "$OUT" +COPY --from=layer-99-onnx "${PREFIX}/srv/." "${OUT}/srv" +COPY --from=layer-99-onnx "${PREFIX}/licenses/." "${OUT}/licenses" + COPY --from=layer-99-ffmpeg "${OUT}/." "$OUT" COPY --from=layer-99-ffmpeg "${PREFIX}/srv/." "${OUT}/srv" COPY --from=layer-99-ffmpeg "${PREFIX}/licenses/." "${OUT}/licenses" @@ -456,7 +482,7 @@ RUN rm -rf "${OUT}/share" "${OUT}/lib/pkgconfig" "${OUT}/lib/cmake" RUN find "${OUT}" \( -name '*.def' -o -name '*.dll.a' \) -delete # Move .lib files to the lib folder (Windows target only) -RUN find "${OUT}/bin" -name '*.lib' -exec install -Dt ../lib/ -m a-rwx,u+rw,g+r,o+r {} + +RUN if [ -d "${OUT}/bin" ]; then find "${OUT}/bin" -name '*.lib' -exec install -Dt ../lib/ -m a-rwx,u+rw,g+r,o+r {} + ; fi # Copy .lib to .dll.a (Windows target only) RUN find "$OUT/lib" -name '*.lib' -exec ` @@ -473,7 +499,7 @@ RUN --mount=type=cache,target=/root/.cache ` RUN find "$OUT" -type f \( -name '*.so' -o -name '*.so.*' \) -exec patchelf --set-rpath '$ORIGIN' {} \; # Remove non executable files from bin folder -RUN find "${OUT}/bin" -type f -not -executable -delete +RUN if [ -d "${OUT}/bin" ]; then find "${OUT}/bin" -type f -not -executable -delete; fi # Remove empty directories RUN find "$OUT" -type d -delete 2>/dev/null || true diff --git a/scripts/build.sh b/scripts/build.sh index 4a57491..cc2c4b3 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -47,7 +47,6 @@ LDFLAGS="-L${PREFIX}/lib -pipe" case "$TARGET" in *linux*) FFLAGS="-fno-semantic-interposition" - CFLAGS="${CFLAGS} -D_GLIBCXX_ASSERTIONS" LDFLAGS="${LDFLAGS} -Wl,-z,relro,-z,now,-z,defs" case "$TARGET" in @@ -59,11 +58,21 @@ case "$TARGET" in FFLAGS="${FFLAGS} -fno-stack-protector -fno-stack-check" ;; esac + + case "$TARGET" in + *gnu) + CFLAGS="${CFLAGS} -D_GLIBCXX_ASSERTIONS=1" + ;; + *musl) + CFLAGS="${CFLAGS} -D_LARGEFILE64_SOURCE=1" + ;; + esac ;; *darwin*) # Apple tools and linker fails to LTO static libraries # https://github.com/tpoechtrager/osxcross/issues/366 export LTO=0 + export LD_LIBRARY_PATH="${CCTOOLS}/lib:/usr/local/lib:${LD_LIBRARY_PATH:-}" # Ugly workaround for apple linker not finding the macOS SDK's Framework directory ln -fs "${MACOS_SDKROOT}/System" '/System' @@ -72,13 +81,11 @@ case "$TARGET" in case "$TARGET" in x86_64*) - export CMAKE_OSX_ARCHITECTURES='x86_64' export MACOSX_DEPLOYMENT_TARGET="10.15" export CMAKE_APPLE_SILICON_PROCESSOR='x86_64' LDFLAGS="${LDFLAGS} -Wl,-arch,x86_64" ;; aarch64*) - export CMAKE_OSX_ARCHITECTURES='aarch64' export MACOSX_DEPLOYMENT_TARGET="11.0" export CMAKE_APPLE_SILICON_PROCESSOR='aarch64' LDFLAGS="${LDFLAGS} -Wl,-arch,arm64" @@ -103,6 +110,10 @@ export CFLAGS="${CFLAGS} ${FFLAGS}" export LDFLAGS="${LDFLAGS} ${FFLAGS}" export CXXFLAGS="${CFLAGS}" +curl () { + env curl --proto '=https' --tlsv1.2 --ciphers "${CIPHERSUITES:?Missing curl ciphersuite}" --silent --show-error --fail --location "$@" +} + bak_src() { if ! { [ "$#" -eq 1 ] && [ -d "$1" ]; }; then echo "bak_src: " >&2 diff --git a/scripts/cc.sh b/scripts/cc.sh index 27a769e..3cef402 100755 --- a/scripts/cc.sh +++ b/scripts/cc.sh @@ -8,7 +8,7 @@ case "${TARGET:?TARGET envvar is required to be defined}" in # Set the glibc minimum version # This is the lowest we can go at the moment # https://github.com/ziglang/zig/issues/9412 - TARGET="${TARGET}.2.18" + TARGET="${TARGET%%.*}.2.18" ;; x86_64-darwin-apple | x86_64-apple-darwin-macho) SDKROOT="$MACOS_SDKROOT" @@ -64,6 +64,7 @@ sysroot='' assembler=0 has_iphone=0 preprocessor=0 +cpu_features=() assembler_file=0 should_add_libcharset=0 has_undefined_dynamic_lookup=0 @@ -77,7 +78,15 @@ while [ "$#" -gt 0 ]; do continue fi - if [ "$1" = '-o-' ] || [ "$1" = '-o=-' ]; then + if [ "$1" = '-Xlinker' ] || [ "$1" = '--for-linker' ]; then + l_args+=("${2:?Linker argument not passed}") + shift 2 + continue + elif (case "$1" in --for-linker=*) exit 0 ;; *) exit 1 ;; esac) then + l_args+=("${1#*=}") + shift + continue + elif [ "$1" = '-o-' ] || [ "$1" = '-o=-' ]; then # -E redirect to stdout by default, -o - breaks it so we ignore it stdout=1 elif [ "$1" = '-o' ] && [ "${2:-}" = '-' ]; then @@ -120,7 +129,7 @@ while [ "$#" -gt 0 ]; do # https://github.com/llvm/llvm-project/issues/47432 # https://github.com/llvm/llvm-project/issues/66912 true - elif [ "$1" = '-xassembler' ] || [ "$1" = '--language=assembler' ]; then + elif [ "$1" = '-xassembler' ] || [ "$1" = '-Xassembler' ] || [ "$1" = '--language=assembler' ]; then # Zig behaves very oddly when passed the explicit assembler language option # https://github.com/ziglang/zig/issues/10915 # https://github.com/ziglang/zig/pull/13544 @@ -130,7 +139,10 @@ while [ "$#" -gt 0 ]; do shift 2 continue elif (case "$1" in -mcpu=* | -march=*) exit 0 ;; *) exit 1 ;; esac) then - # Ignore -mcpu and -march flags, we set them ourselves + # Save each feature to the cpu_features array + IFS='+' read -ra _features <<<"$1" + unset "_features[0]" + cpu_features+=("${_features[@]}") true else if (case "$TARGET" in *darwin*) exit 0 ;; *) exit 1 ;; esac) then @@ -269,6 +281,9 @@ while [ "$#" -gt 0 ]; do l_args+=("$1") ;; esac + elif (case "$1" in --version-script=*) exit 0 ;; *) exit 1 ;; esac) then + # Zig only support --version-script with a separate argument + l_args+=(--version-script "${1#*=}") else l_args+=("$1") fi @@ -282,7 +297,9 @@ if [ $stdout -eq 1 ] && ! [ $preprocessor -eq 1 ]; then fi # Work-around Zig not respecting -fno-lto when -flto is set -if [ -n "$lto" ]; then +if [ "${LTO:-1}" -eq 0 ]; then + argv+=('-fno-lto') +elif [ -n "$lto" ]; then argv+=("$lto") fi @@ -292,32 +309,56 @@ if [ $should_add_libcharset -eq 1 ]; then fi # Compiler specific flags +features="" +for feature in "${cpu_features[@]}"; do + case "$CMD" in + clang*) ;; + *) + if [ "$assembler" -eq 0 ] || [ "$preprocessor" -eq 1 ]; then + # Zig specific changes + case "$feature" in + fp16) + feature="fullfp16" + ;; + esac + fi + ;; + esac + + features="${features}+${feature}" +done + case "${TARGET:-}" in x86_64*) case "${TARGET:-}" in *darwin*) # macOS 10.15 (Catalina) only supports Macs made with Ivy Bridge or later - c_argv+=(-march=ivybridge) + c_argv+=("-march=ivybridge${features}") ;; *) - c_argv+=(-march=x86_64_v2) + c_argv+=("-march=x86_64_v2${features}") ;; esac ;; arm64* | aarch64*) - case "${TARGET:-}" in - *darwin*) - c_argv+=(-mcpu=apple-m1) - ;; - *) - # Raspberry Pi 3 - c_argv+=(-mcpu=cortex_a53) - ;; - esac + if [ "$assembler" -eq 1 ] && [ "$preprocessor" -eq 0 ]; then + # This is an workaround for zig not supporting some features when compiling arm assembler code + c_argv+=(-Xassembler "-march=armv8.2-a${features}") + else + case "${TARGET:-}" in + *darwin*) + c_argv+=("-mcpu=apple-m1${features}") + ;; + *) + # Raspberry Pi 3 + c_argv+=("-mcpu=cortex_a53${features}") + ;; + esac + fi ;; esac -# Like -O2 with extra optimizations to reduce code size +# Like -O2, but with extra optimizations to reduce code size c_argv+=(-Os) # Resolve sysroot arguments per target diff --git a/scripts/convert_yolo.ipynb b/scripts/convert_yolo.ipynb new file mode 100644 index 0000000..0d83274 --- /dev/null +++ b/scripts/convert_yolo.ipynb @@ -0,0 +1,254 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "provenance": [], + "gpuType": "T4" + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + }, + "language_info": { + "name": "python" + }, + "accelerator": "GPU" + }, + "cells": [ + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "aDGVZFDDcvZE", + "outputId": "48876f9b-84c3-4f59-a16c-764378735b9d" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Collecting ultralytics\n", + " Downloading ultralytics-8.0.222-py3-none-any.whl (653 kB)\n", + "\u001b[?25l \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/654.0 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K \u001b[91m━━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[90m╺\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m348.2/654.0 kB\u001b[0m \u001b[31m10.9 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m654.0/654.0 kB\u001b[0m \u001b[31m13.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: matplotlib>=3.3.0 in /usr/local/lib/python3.10/dist-packages (from ultralytics) (3.7.1)\n", + "Requirement already satisfied: numpy>=1.22.2 in /usr/local/lib/python3.10/dist-packages (from ultralytics) (1.23.5)\n", + "Requirement already satisfied: opencv-python>=4.6.0 in /usr/local/lib/python3.10/dist-packages (from ultralytics) (4.8.0.76)\n", + "Requirement already satisfied: pillow>=7.1.2 in /usr/local/lib/python3.10/dist-packages (from ultralytics) (9.4.0)\n", + "Requirement already satisfied: pyyaml>=5.3.1 in /usr/local/lib/python3.10/dist-packages (from ultralytics) (6.0.1)\n", + "Requirement already satisfied: requests>=2.23.0 in /usr/local/lib/python3.10/dist-packages (from ultralytics) (2.31.0)\n", + "Requirement already satisfied: scipy>=1.4.1 in /usr/local/lib/python3.10/dist-packages (from ultralytics) (1.11.4)\n", + "Requirement already satisfied: torch>=1.8.0 in /usr/local/lib/python3.10/dist-packages (from ultralytics) (2.1.0+cu118)\n", + "Requirement already satisfied: torchvision>=0.9.0 in /usr/local/lib/python3.10/dist-packages (from ultralytics) (0.16.0+cu118)\n", + "Requirement already satisfied: tqdm>=4.64.0 in /usr/local/lib/python3.10/dist-packages (from ultralytics) (4.66.1)\n", + "Requirement already satisfied: pandas>=1.1.4 in /usr/local/lib/python3.10/dist-packages (from ultralytics) (1.5.3)\n", + "Requirement already satisfied: seaborn>=0.11.0 in /usr/local/lib/python3.10/dist-packages (from ultralytics) (0.12.2)\n", + "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from ultralytics) (5.9.5)\n", + "Requirement already satisfied: py-cpuinfo in /usr/local/lib/python3.10/dist-packages (from ultralytics) (9.0.0)\n", + "Collecting thop>=0.1.1 (from ultralytics)\n", + " Downloading thop-0.1.1.post2209072238-py3-none-any.whl (15 kB)\n", + "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=3.3.0->ultralytics) (1.2.0)\n", + "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=3.3.0->ultralytics) (0.12.1)\n", + "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=3.3.0->ultralytics) (4.45.1)\n", + "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=3.3.0->ultralytics) (1.4.5)\n", + "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=3.3.0->ultralytics) (23.2)\n", + "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=3.3.0->ultralytics) (3.1.1)\n", + "Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=3.3.0->ultralytics) (2.8.2)\n", + "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas>=1.1.4->ultralytics) (2023.3.post1)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.23.0->ultralytics) (3.3.2)\n", + "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.23.0->ultralytics) (3.6)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.23.0->ultralytics) (2.0.7)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.23.0->ultralytics) (2023.11.17)\n", + "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch>=1.8.0->ultralytics) (3.13.1)\n", + "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (from torch>=1.8.0->ultralytics) (4.5.0)\n", + "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.8.0->ultralytics) (1.12)\n", + "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.8.0->ultralytics) (3.2.1)\n", + "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.8.0->ultralytics) (3.1.2)\n", + "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch>=1.8.0->ultralytics) (2023.6.0)\n", + "Requirement already satisfied: triton==2.1.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.8.0->ultralytics) (2.1.0)\n", + "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.7->matplotlib>=3.3.0->ultralytics) (1.16.0)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.8.0->ultralytics) (2.1.3)\n", + "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.8.0->ultralytics) (1.3.0)\n", + "Installing collected packages: thop, ultralytics\n", + "Successfully installed thop-0.1.1.post2209072238 ultralytics-8.0.222\n" + ] + } + ], + "source": [ + "!pip install ultralytics" + ] + }, + { + "cell_type": "code", + "source": [ + "!yolo export model=yolov8s.pt device=0 format=onnx imgsz=640 half=True simplify=True dynamic=False opset=12" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "eJpL1wTPdCUh", + "outputId": "3ea3e9bf-09ea-4441-d07c-dff3dc4e8ecc" + }, + "execution_count": 3, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Downloading https://github.com/ultralytics/assets/releases/download/v0.0.0/yolov8s.pt to 'yolov8s.pt'...\n", + "100% 21.5M/21.5M [00:00<00:00, 254MB/s]\n", + "Ultralytics YOLOv8.0.222 🚀 Python-3.10.12 torch-2.1.0+cu118 CUDA:0 (Tesla T4, 15102MiB)\n", + "YOLOv8s summary (fused): 168 layers, 11156544 parameters, 0 gradients, 28.6 GFLOPs\n", + "\n", + "\u001b[34m\u001b[1mPyTorch:\u001b[0m starting from 'yolov8s.pt' with input shape (1, 3, 640, 640) BCHW and output shape(s) (1, 84, 8400) (21.5 MB)\n", + "\u001b[31m\u001b[1mrequirements:\u001b[0m Ultralytics requirements ['onnx>=1.12.0', 'onnxsim>=0.4.33', 'onnxruntime-gpu'] not found, attempting AutoUpdate...\n", + "Collecting onnx>=1.12.0\n", + " Downloading onnx-1.15.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (15.7 MB)\n", + " ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 15.7/15.7 MB 156.4 MB/s eta 0:00:00\n", + "Collecting onnxsim>=0.4.33\n", + " Downloading onnxsim-0.4.35-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.2 MB)\n", + " ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 2.2/2.2 MB 124.8 MB/s eta 0:00:00\n", + "Collecting onnxruntime-gpu\n", + " Downloading onnxruntime_gpu-1.16.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (157.1 MB)\n", + " ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 157.1/157.1 MB 231.9 MB/s eta 0:00:00\n", + "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from onnx>=1.12.0) (1.23.5)\n", + "Requirement already satisfied: protobuf>=3.20.2 in /usr/local/lib/python3.10/dist-packages (from onnx>=1.12.0) (3.20.3)\n", + "Requirement already satisfied: rich in /usr/local/lib/python3.10/dist-packages (from onnxsim>=0.4.33) (13.7.0)\n", + "Collecting coloredlogs (from onnxruntime-gpu)\n", + " Downloading coloredlogs-15.0.1-py2.py3-none-any.whl (46 kB)\n", + " ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 46.0/46.0 kB 15.7 MB/s eta 0:00:00\n", + "Requirement already satisfied: flatbuffers in /usr/local/lib/python3.10/dist-packages (from onnxruntime-gpu) (23.5.26)\n", + "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from onnxruntime-gpu) (23.2)\n", + "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from onnxruntime-gpu) (1.12)\n", + "Collecting humanfriendly>=9.1 (from coloredlogs->onnxruntime-gpu)\n", + " Downloading humanfriendly-10.0-py2.py3-none-any.whl (86 kB)\n", + " ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 86.8/86.8 kB 198.3 MB/s eta 0:00:00\n", + "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.10/dist-packages (from rich->onnxsim>=0.4.33) (3.0.0)\n", + "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.10/dist-packages (from rich->onnxsim>=0.4.33) (2.16.1)\n", + "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->onnxruntime-gpu) (1.3.0)\n", + "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.10/dist-packages (from markdown-it-py>=2.2.0->rich->onnxsim>=0.4.33) (0.1.2)\n", + "Installing collected packages: onnx, humanfriendly, coloredlogs, onnxsim, onnxruntime-gpu\n", + "Successfully installed coloredlogs-15.0.1 humanfriendly-10.0 onnx-1.15.0 onnxruntime-gpu-1.16.3 onnxsim-0.4.35\n", + "\n", + "\u001b[31m\u001b[1mrequirements:\u001b[0m AutoUpdate success ✅ 19.5s, installed 3 packages: ['onnx>=1.12.0', 'onnxsim>=0.4.33', 'onnxruntime-gpu']\n", + "\u001b[31m\u001b[1mrequirements:\u001b[0m ⚠️ \u001b[1mRestart runtime or rerun command for updates to take effect\u001b[0m\n", + "\n", + "\n", + "\u001b[34m\u001b[1mONNX:\u001b[0m starting export with onnx 1.15.0 opset 12...\n", + "\u001b[34m\u001b[1mONNX:\u001b[0m simplifying with onnxsim 0.4.35...\n", + "\u001b[34m\u001b[1mONNX:\u001b[0m export success ✅ 21.4s, saved as 'yolov8s.onnx' (21.4 MB)\n", + "\n", + "Export complete (35.5s)\n", + "Results saved to \u001b[1m/content\u001b[0m\n", + "Predict: yolo predict task=detect model=yolov8s.onnx imgsz=640 half \n", + "Validate: yolo val task=detect model=yolov8s.onnx imgsz=640 data=coco.yaml half \n", + "Visualize: https://netron.app\n", + "💡 Learn more at https://docs.ultralytics.com/modes/export\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "from google.colab import files\n", + "files.download('yolov8s.onnx')" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 17 + }, + "id": "PyGDfJZVda5P", + "outputId": "395c1c7a-2b9f-471c-9ecb-5df8a166d282" + }, + "execution_count": 4, + "outputs": [ + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "application/javascript": [ + "\n", + " async function download(id, filename, size) {\n", + " if (!google.colab.kernel.accessAllowed) {\n", + " return;\n", + " }\n", + " const div = document.createElement('div');\n", + " const label = document.createElement('label');\n", + " label.textContent = `Downloading \"${filename}\": `;\n", + " div.appendChild(label);\n", + " const progress = document.createElement('progress');\n", + " progress.max = size;\n", + " div.appendChild(progress);\n", + " document.body.appendChild(div);\n", + "\n", + " const buffers = [];\n", + " let downloaded = 0;\n", + "\n", + " const channel = await google.colab.kernel.comms.open(id);\n", + " // Send a message to notify the kernel that we're ready.\n", + " channel.send({})\n", + "\n", + " for await (const message of channel.messages) {\n", + " // Send a message to notify the kernel that we're ready.\n", + " channel.send({})\n", + " if (message.buffers) {\n", + " for (const buffer of message.buffers) {\n", + " buffers.push(buffer);\n", + " downloaded += buffer.byteLength;\n", + " progress.value = downloaded;\n", + " }\n", + " }\n", + " }\n", + " const blob = new Blob(buffers, {type: 'application/binary'});\n", + " const a = document.createElement('a');\n", + " a.href = window.URL.createObjectURL(blob);\n", + " a.download = filename;\n", + " div.appendChild(a);\n", + " a.click();\n", + " div.remove();\n", + " }\n", + " " + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "application/javascript": [ + "download(\"download_77f6dfda-b7f1-4f03-9d91-e30e5369513f\", \"yolov8s.onnx\", 22424156)" + ] + }, + "metadata": {} + } + ] + }, + { + "cell_type": "code", + "source": [], + "metadata": { + "id": "PO1wCeNMePDq" + }, + "execution_count": 4, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [], + "metadata": { + "id": "o6dEdltJebUT" + } + } + ] +} \ No newline at end of file diff --git a/scripts/create-framework.sh b/scripts/create-framework.sh index f2b3442..c508358 100755 --- a/scripts/create-framework.sh +++ b/scripts/create-framework.sh @@ -54,6 +54,9 @@ mv "${OUT}/include" "${OUT}/${_framework}/Versions/A/Headers" # Move licenses to Framework mv "${OUT}/licenses" "${OUT}/${_framework}/Versions/A/Resources/Licenses" +# Move models to Framework +mv "${OUT}/models" "${OUT}/${_framework}/Versions/A/Resources/Models" + # Create required framework symlinks ln -s A "${OUT}/${_framework}/Versions/Current" ln -s Versions/Current/Headers "${OUT}/${_framework}/Headers" diff --git a/scripts/curl_tar.sh b/scripts/curl_tar.sh index e306a8b..cea4c5c 100755 --- a/scripts/curl_tar.sh +++ b/scripts/curl_tar.sh @@ -31,12 +31,11 @@ fi _url="$1" _cache="/root/.cache/_curl_tar/$(md5sum - <<<"$_url" | awk '{ print $1 }')" -_ciphersuites="TLS_AES_128_GCM_SHA256:TLS_CHACHA20_POLY1305_SHA256:TLS_AES_256_GCM_SHA384:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384" mkdir -p "$(dirname "$_cache")" if ! [ -s "$_cache" ]; then - curl --proto '=https' --tlsv1.2 --ciphers "$_ciphersuites" --silent --show-error --fail --location "$_url" >"$_cache" + curl --proto '=https' --tlsv1.2 --ciphers "${CIPHERSUITES:?Missing curl ciphersuite}" --silent --show-error --fail --location "$_url" >"$_cache" fi trap 'rm -rf "$_cache"' ERR diff --git a/scripts/toolchain.sh b/scripts/toolchain.sh index b944316..b4d4e90 100755 --- a/scripts/toolchain.sh +++ b/scripts/toolchain.sh @@ -90,6 +90,17 @@ set(CMAKE_SYSTEM_NAME ${SYSTEM_NAME^}) set(CMAKE_SYSTEM_VERSION ${SYSTEM_VERSION}) set(CMAKE_SYSTEM_PROCESSOR ${SYSTEM_PROCESSOR}) +$( + case "$TARGET" in + x86_64-darwin*) + echo 'set(CMAKE_OSX_ARCHITECTURES "x86_64" CACHE STRING "")' + ;; + aarch64-darwin*) + echo 'set(CMAKE_OSX_ARCHITECTURES "arm64" CACHE STRING "")' + ;; + esac +) + $(if [ -n "${SDKROOT:-}" ]; then echo "set(CMAKE_SYSROOT ${SDKROOT})"; fi) set(CMAKE_CROSSCOMPILING TRUE) diff --git a/stages/00-apple/02-tapi.sh b/stages/00-apple/02-tapi.sh index ce481b1..978dc49 100755 --- a/stages/00-apple/02-tapi.sh +++ b/stages/00-apple/02-tapi.sh @@ -24,10 +24,12 @@ echo "Download tapi ..." mkdir -p "tapi" -curl_tar 'https://github.com/tpoechtrager/apple-libtapi/archive/43a0c04.tar.gz' 'tapi' 1 +curl_tar 'https://github.com/tpoechtrager/apple-libtapi/archive/b8c5ac4.tar.gz' 'tapi' 1 cd tapi +export NINJA=1 + ./build.sh ./install.sh diff --git a/stages/00-apple/03-dispatch.sh b/stages/00-apple/03-dispatch.sh new file mode 100755 index 0000000..2896391 --- /dev/null +++ b/stages/00-apple/03-dispatch.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +set -euo pipefail + +case "$TARGET" in + *darwin*) ;; + *) + exit 0 + ;; +esac + +apt-get install systemtap-sdt-dev libbsd-dev linux-libc-dev + +export CC="clang-16" +export CXX="clang++-16" +export CFLAGS="-I${CCTOOLS}/include" +export LDFLAGS="-L${CCTOOLS}/lib" +export APPLE_TARGET='__BYPASS__' + +cd /srv + +echo "Download dispatch ..." + +mkdir -p 'dispatch/build' + +curl_tar 'https://github.com/tpoechtrager/apple-libdispatch/archive/ee39300.tar.gz' 'dispatch' 1 + +cd dispatch/build + +cmake -G Ninja -DCMAKE_C_COMPILER="$CC" -DCMAKE_CXX_COMPILER="$CXX" .. + +ninja -j"$(nproc)" + +ninja install + +rm -r /srv/dispatch diff --git a/stages/00-apple/03-cctools.sh b/stages/00-apple/04-cctools.sh similarity index 67% rename from stages/00-apple/03-cctools.sh rename to stages/00-apple/04-cctools.sh index 5d2d5f3..abfde2b 100755 --- a/stages/00-apple/03-cctools.sh +++ b/stages/00-apple/04-cctools.sh @@ -30,12 +30,7 @@ echo "Download cctools ..." mkdir -p "cctools" -curl_tar 'https://github.com/tpoechtrager/cctools-port/archive/437ced3.tar.gz' 'cctools' 1 - -sed -i "/^if readelf -p .comment \$LIBDIR\/libLTO.so | grep clang &>\/dev\/null; then/,/^fi/d;" cctools/tools/fix_liblto.sh -sed -ie 's/wget/curl -LSsOJ/' cctools/tools/fix_liblto.sh - -env LLVM_CONFIG=llvm-config-16 cctools/tools/fix_liblto.sh +curl_tar 'https://github.com/tpoechtrager/cctools-port/archive/59f5fb8.tar.gz' 'cctools' 1 cd cctools/cctools @@ -44,12 +39,18 @@ cd cctools/cctools --target="$_target" \ --with-libxar="$CCTOOLS" \ --with-libtapi="$CCTOOLS" \ + --with-libdispatch="$CCTOOLS" \ --with-llvm-config=llvm-config-16 \ + --with-libblocksruntime="$CCTOOLS" \ --enable-xar-support \ - --enable-lto-support + --enable-lto-support \ + --enable-tapi-support make -j"$(nproc)" make install rm -r /srv/cctools + +# Create symlinks for llvm-otool because cctools use it when calling its own otool +ln -fs "$(command -v llvm-otool-16)" /opt/cctools/bin/llvm-otool diff --git a/stages/10-sse2neon.sh b/stages/10-sse2neon.sh index 8fe2b4e..16486ec 100644 --- a/stages/10-sse2neon.sh +++ b/stages/10-sse2neon.sh @@ -14,7 +14,7 @@ mkdir -p sse2neon curl_tar 'https://github.com/DLTcollab/sse2neon/archive/refs/tags/v1.6.0.tar.gz' 'sse2neon' 1 -curl -LSs 'https://raw.githubusercontent.com/HandBrake/HandBrake/172cd5d/contrib/sse2neon/A01-types-fix.patch' \ +curl 'https://raw.githubusercontent.com/HandBrake/HandBrake/172cd5d/contrib/sse2neon/A01-types-fix.patch' \ | patch -F5 -lp1 -d "sse2neon" -t # Remove unused components diff --git a/stages/20-zlib.sh b/stages/20-zlib.sh index fe911f7..6139129 100755 --- a/stages/20-zlib.sh +++ b/stages/20-zlib.sh @@ -11,7 +11,7 @@ esac echo "Download zlib..." mkdir -p zlib -curl_tar 'https://github.com/zlib-ng/zlib-ng/archive/refs/tags/2.1.4.tar.gz' zlib 1 +curl_tar 'https://github.com/zlib-ng/zlib-ng/archive/refs/tags/2.1.5.tar.gz' zlib 1 # Patch cmake to disable armv6 assembly, it thinks we support it, but we don't sed -i 's/HAVE_ARMV6_INLINE_ASM OR HAVE_ARMV6_INTRIN/HAVE_ARMV6_INLINE_ASM AND HAVE_ARMV6_INTRIN/' zlib/CMakeLists.txt diff --git a/stages/25-lcms.sh b/stages/25-lcms.sh index 4b1866f..ad4f37a 100755 --- a/stages/25-lcms.sh +++ b/stages/25-lcms.sh @@ -14,12 +14,12 @@ esac # Some required patches for fixing meson and windows cross-compile issues for patch in \ - https://github.com/mm2/Little-CMS/commit/4e55c55.patch \ - https://github.com/mm2/Little-CMS/commit/8ddc268.patch \ - https://github.com/mm2/Little-CMS/commit/8769c0e.patch \ - https://github.com/mm2/Little-CMS/commit/7984408.patch \ - https://github.com/mm2/Little-CMS/commit/b35e271.patch; do - curl -LSs "$patch" | patch -F5 -lp1 -d lcms -t + 'https://github.com/mm2/Little-CMS/commit/4e55c55.patch' \ + 'https://github.com/mm2/Little-CMS/commit/8ddc268.patch' \ + 'https://github.com/mm2/Little-CMS/commit/8769c0e.patch' \ + 'https://github.com/mm2/Little-CMS/commit/7984408.patch' \ + 'https://github.com/mm2/Little-CMS/commit/b35e271.patch'; do + curl "$patch" | patch -F5 -lp1 -d lcms -t done sed -i "/subdir('utils')/d" lcms/meson.build diff --git a/stages/45-de265.sh b/stages/45-de265.sh index a45f9c6..71a877b 100755 --- a/stages/45-de265.sh +++ b/stages/45-de265.sh @@ -3,7 +3,7 @@ echo "Download de265..." mkdir -p de265 -curl_tar 'https://github.com/strukturag/libde265/archive/refs/tags/v1.0.12.tar.gz' de265 1 +curl_tar 'https://github.com/strukturag/libde265/archive/refs/tags/v1.0.14.tar.gz' de265 1 case "$TARGET" in aarch64*) diff --git a/stages/50-lame.sh b/stages/50-lame.sh index 95ef7ef..6ba67d4 100644 --- a/stages/50-lame.sh +++ b/stages/50-lame.sh @@ -12,7 +12,7 @@ curl_tar 'https://deb.debian.org/debian/pool/main/l/lame/lame_3.100.orig.tar.gz' curl_tar 'https://github.com/mesonbuild/wrapdb/releases/download/lame_3.100-9/lame_3.100-9_patch.zip' lame 1 # Fix warning on 64 bit machines. explicitly set variables as unsigned ints. -curl -LSs 'https://sources.debian.org/data/main/l/lame/3.100-6/debian/patches/07-field-width-fix.patch' \ +curl 'https://sources.debian.org/data/main/l/lame/3.100-6/debian/patches/07-field-width-fix.patch' \ | patch -F5 -lp1 -d lame -t # Remove some superfluous files diff --git a/stages/50-opus.sh b/stages/50-opus.sh index a1bea94..e0b60a1 100644 --- a/stages/50-opus.sh +++ b/stages/50-opus.sh @@ -6,7 +6,7 @@ mkdir -p opus curl_tar 'https://github.com/xiph/opus/releases/download/v1.4/opus-1.4.tar.gz' opus 1 # Required patch to fix meson for arm builds -curl -LSs 'https://github.com/xiph/opus/commit/20c032d.patch' \ +curl 'https://github.com/xiph/opus/commit/20c032d.patch' \ | patch -F5 -lp1 -d opus -t # Remove unused components diff --git a/stages/50-svt-av1.sh b/stages/50-svt-av1.sh index 4c37883..d0b5c3d 100644 --- a/stages/50-svt-av1.sh +++ b/stages/50-svt-av1.sh @@ -11,8 +11,8 @@ case "$TARGET" in ;; aarch64*) ENABLE_NASM=Off - # Patch to enable SSE in aarch64 - curl -LSs 'https://gitlab.com/AOMediaCodec/SVT-AV1/-/merge_requests/2135.patch' | patch -F5 -lp1 -d svt-av1 -t + # Patch to enable NEON in aarch64 + curl 'https://gitlab.com/AOMediaCodec/SVT-AV1/-/merge_requests/2135.patch' | patch -F5 -lp1 -d svt-av1 -t ;; esac diff --git a/stages/50-vulkan/45-vulkan.sh b/stages/50-vulkan/45-vulkan.sh index dec1c3e..b52d029 100755 --- a/stages/50-vulkan/45-vulkan.sh +++ b/stages/50-vulkan/45-vulkan.sh @@ -10,7 +10,7 @@ esac echo "Download vulkan..." mkdir -p vulkan-headers -curl_tar 'https://github.com/KhronosGroup/Vulkan-Headers/archive/refs/tags/v1.3.269.tar.gz' vulkan-headers 1 +curl_tar 'https://github.com/KhronosGroup/Vulkan-Headers/archive/refs/tags/v1.3.268.tar.gz' vulkan-headers 1 VERSION="$( sed -nr \ diff --git a/stages/50-vulkan/55-spirv-cross.sh b/stages/50-vulkan/55-spirv-cross.sh index aed03d8..198004e 100755 --- a/stages/50-vulkan/55-spirv-cross.sh +++ b/stages/50-vulkan/55-spirv-cross.sh @@ -10,7 +10,7 @@ esac echo "Download spirv..." mkdir -p spirv -curl_tar 'https://github.com/KhronosGroup/SPIRV-Cross/archive/refs/tags/sdk-1.3.261.1.tar.gz' spirv 1 +curl_tar 'https://github.com/KhronosGroup/SPIRV-Cross/archive/refs/tags/vulkan-sdk-1.3.268.0.tar.gz' spirv 1 VERSION="$( grep -Po 'set\(spirv-cross-abi-major\s+\K\d+' spirv/CMakeLists.txt diff --git a/stages/50-vulkan/60-placebo.sh b/stages/50-vulkan/60-placebo.sh index 885bea6..8b4a25b 100755 --- a/stages/50-vulkan/60-placebo.sh +++ b/stages/50-vulkan/60-placebo.sh @@ -10,29 +10,20 @@ esac echo "Download placebo..." mkdir -p placebo -curl_tar 'https://github.com/haasn/libplacebo/archive/refs/tags/v5.264.1.tar.gz' placebo 1 - -# Some required patches for fixing log for windows -for patch in \ - 'https://github.com/haasn/libplacebo/commit/c02a8a2.patch' \ - 'https://github.com/haasn/libplacebo/commit/7f9eb40.patch' \ - 'https://github.com/haasn/libplacebo/commit/cd7a371.patch' \ - 'https://github.com/haasn/libplacebo/commit/dc6e5a5.patch' \ - 'https://github.com/haasn/libplacebo/commit/97d008b.patch'; do - curl -LSs "$patch" | patch -F5 -lp1 -d placebo -t -done - -# Fix spirv import -sed -ie 's@spirv_cross_c.h@spirv_cross/spirv_cross_c.h@' placebo/src/d3d11/gpu.h +curl_tar 'https://github.com/haasn/libplacebo/archive/refs/tags/v6.338.1.tar.gz' placebo 1 # Thrid party deps curl_tar 'https://github.com/pallets/jinja/archive/refs/tags/3.1.2.tar.gz' placebo/3rdparty/jinja 1 curl_tar 'https://github.com/pallets/markupsafe/archive/refs/tags/2.1.3.tar.gz' placebo/3rdparty/markupsafe 1 +curl_tar 'https://github.com/fastfloat/fast_float/archive/refs/tags/v5.3.0.tar.gz' placebo/3rdparty/fast_float 1 + +sed -i "s|windows.compile_resources(libplacebo_rc, depends: version_h,|windows.compile_resources(libplacebo_rc, depends: version_h, args: '/c65001',|" placebo/src/meson.build # Remove some superfluous files -rm -rf placebo/{.github,docs,demos} -rm -rf placebo/3rdparty/jinja/{.github,artwork,docs,examples,requirements,scripts,tests} -rm -rf placebo/3rdparty/markupsafe/{.github,bench,docs,requirements,tests} +rm -rf placebo/{.*,docs,demos} +rm -rf placebo/3rdparty/jinja/{.*,artwork,docs,examples,requirements,scripts,tests} +rm -rf placebo/3rdparty/markupsafe/{.*,bench,docs,requirements,tests} +rm -rf placebo/3rdparty/fast_float/{.*,ci,fuzz,script,tests} # Backup source bak_src 'placebo' diff --git a/stages/50-x264.sh b/stages/50-x264.sh index c56c482..05db9f3 100644 --- a/stages/50-x264.sh +++ b/stages/50-x264.sh @@ -4,13 +4,13 @@ echo "Download x264..." mkdir -p x264 # Using master due to aarch64 improvements -curl_tar 'https://code.videolan.org/videolan/x264/-/archive/a354f11/x264.tar.bz2' x264 1 +curl_tar 'https://code.videolan.org/videolan/x264/-/archive/c1962404/x264.tar.bz2' x264 1 # Some minor fixes to x264's pkg-config for patch in \ - https://github.com/msys2/MINGW-packages/raw/f4bd368/mingw-w64-x264/0001-beautify-pc.all.patch \ - https://github.com/msys2/MINGW-packages/raw/f4bd368/mingw-w64-x264/0003-pkgconfig-add-Cflags-private.patch; do - curl -LSs "$patch" | patch -F5 -lp1 -d x264 -t + 'https://github.com/msys2/MINGW-packages/raw/f4bd368/mingw-w64-x264/0001-beautify-pc.all.patch' \ + 'https://github.com/msys2/MINGW-packages/raw/f4bd368/mingw-w64-x264/0003-pkgconfig-add-Cflags-private.patch'; do + curl "$patch" | patch -F5 -lp1 -d x264 -t done case "$TARGET" in diff --git a/stages/50-x265.sh b/stages/50-x265.sh index 4f0cde4..b5b41c0 100644 --- a/stages/50-x265.sh +++ b/stages/50-x265.sh @@ -9,6 +9,13 @@ curl_tar 'https://bitbucket.org/multicoreware/x265_git/get/8ee01d4.tar.bz2' x265 # Remove some superfluous files rm -rf x265/{doc,build} +# Handbreak patches +for patch in \ + 'https://github.com/HandBrake/HandBrake/raw/9c9cf41/contrib/x265/A03-sei-length-crash-fix.patch' \ + 'https://github.com/HandBrake/HandBrake/raw/9c9cf41/contrib/x265/A05-memory-leaks.patch'; do + curl "$patch" | patch -F5 -lp1 -d x265 -t +done + # Backup source bak_src 'x265' diff --git a/stages/99-ffmpeg.sh b/stages/99-ffmpeg.sh index c7968ba..bd50031 100755 --- a/stages/99-ffmpeg.sh +++ b/stages/99-ffmpeg.sh @@ -3,7 +3,31 @@ echo "Download ffmpeg..." mkdir -p ffmpeg -curl_tar 'https://github.com/FFmpeg/FFmpeg/archive/refs/tags/n6.0.tar.gz' ffmpeg 1 +curl_tar 'https://github.com/FFmpeg/FFmpeg/archive/refs/tags/n6.1.tar.gz' ffmpeg 1 + +# Handbreak patches +for patch in \ + 'https://github.com/HandBrake/HandBrake/raw/9c9cf41/contrib/ffmpeg/A01-mov-read-name-track-tag-written-by-movenc.patch' \ + 'https://github.com/HandBrake/HandBrake/raw/9c9cf41/contrib/ffmpeg/A02-movenc-write-3gpp-track-titl-tag.patch' \ + 'https://github.com/HandBrake/HandBrake/raw/9c9cf41/contrib/ffmpeg/A03-mov-read-3gpp-udta-tags.patch' \ + 'https://github.com/HandBrake/HandBrake/raw/9c9cf41/contrib/ffmpeg/A04-movenc-write-3gpp-track-names-tags-for-all-available.patch' \ + 'https://github.com/HandBrake/HandBrake/raw/9c9cf41/contrib/ffmpeg/A05-dvdsubdec-fix-processing-of-partial-packets.patch' \ + 'https://github.com/HandBrake/HandBrake/raw/9c9cf41/contrib/ffmpeg/A06-dvdsubdec-return-number-of-bytes-used.patch' \ + 'https://github.com/HandBrake/HandBrake/raw/9c9cf41/contrib/ffmpeg/A07-dvdsubdec-use-pts-of-initial-packet.patch' \ + 'https://github.com/HandBrake/HandBrake/raw/9c9cf41/contrib/ffmpeg/A08-ccaption_dec-fix-pts-in-real_time-mode.patch' \ + 'https://github.com/HandBrake/HandBrake/raw/9c9cf41/contrib/ffmpeg/A09-matroskaenc-aac-extradata-updated.patch' \ + 'https://github.com/HandBrake/HandBrake/raw/9c9cf41/contrib/ffmpeg/A10-amfenc-Add-support-for-pict_type-field.patch' \ + 'https://github.com/HandBrake/HandBrake/raw/9c9cf41/contrib/ffmpeg/A11-amfenc-Fixes-the-color-information-in-the-ou.patch' \ + 'https://github.com/HandBrake/HandBrake/raw/9c9cf41/contrib/ffmpeg/A12-amfenc-HDR-metadata.patch' \ + 'https://github.com/HandBrake/HandBrake/raw/9c9cf41/contrib/ffmpeg/A13-libavcodec-amfenc-Fix-issue-with-missing-headers-in-.patch' \ + 'https://github.com/HandBrake/HandBrake/raw/9c9cf41/contrib/ffmpeg/A14-avcodec-add-ambient-viewing-environment-packet-side-.patch' \ + 'https://github.com/HandBrake/HandBrake/raw/9c9cf41/contrib/ffmpeg/A15-avformat-mov-add-support-for-amve-ambient-viewing-en.patch' \ + 'https://github.com/HandBrake/HandBrake/raw/9c9cf41/contrib/ffmpeg/A16-videotoolbox-dec-h264-10bit.patch' \ + 'https://github.com/HandBrake/HandBrake/raw/9c9cf41/contrib/ffmpeg/A17-libswscale-fix-yuv420p-to-p01xle-color-conversion-bu.patch' \ + 'https://github.com/HandBrake/HandBrake/raw/9c9cf41/contrib/ffmpeg/A18-qsv-fix-decode-10bit-hdr.patch' \ + 'https://github.com/HandBrake/HandBrake/raw/9c9cf41/contrib/ffmpeg/A19-ffbuild-common-use-gzip-n-flag-for-cuda.patch'; do + curl "$patch" | patch -F5 -lp1 -d ffmpeg -t +done # Backup source bak_src 'ffmpeg' @@ -57,7 +81,6 @@ case "$TARGET" in # TODO: Maybe try macOS own metal compiler under darling? https://github.com/darlinghq/darling/issues/326 # TODO: Add support for vulkan (+ libplacebo) on macOS with MoltenVK --sysroot="${MACOS_SDKROOT:?Missing macOS SDK path}" - --disable-lto --disable-metal --disable-vulkan --disable-w32threads @@ -90,7 +113,6 @@ case "$TARGET" in # TODO: Add support for mediafoundation on Windows (zig doesn't seem to have the necessary bindings to it yet) # FIX-ME: LTO isn't working on Windows rn env_specific_arg+=( - --disable-lto --disable-pthreads --disable-coreimage --disable-videotoolbox diff --git a/stages/99-heif.sh b/stages/99-heif.sh index 9c15bb9..672d1fe 100755 --- a/stages/99-heif.sh +++ b/stages/99-heif.sh @@ -3,7 +3,7 @@ echo "Download heif..." mkdir -p heif -curl_tar 'https://github.com/strukturag/libheif/releases/download/v1.17.3/libheif-1.17.3.tar.gz' heif 1 +curl_tar 'https://github.com/strukturag/libheif/releases/download/v1.17.5/libheif-1.17.5.tar.gz' heif 1 case "$TARGET" in *windows*) @@ -11,6 +11,8 @@ case "$TARGET" in ;; esac +sed -i 's/find_package(FFMPEG COMPONENTS avcodec)/find_package(FFMPEG COMPONENTS avcodec avutil)/' heif/CMakeLists.txt + # Remove unused components rm -r heif/{go,fuzzing,tests,examples} @@ -44,7 +46,7 @@ env SHARED=On PREFIX="$OUT" cmake \ -DWITH_AOM_ENCODER=Off \ -DWITH_JPEG_DECODER=Off \ -DWITH_JPEG_ENCODER=Off \ - -DWITH_FFMPEG_DECODER=Off \ + -DWITH_FFMPEG_DECODER=On \ -DWITH_OpenJPEG_DECODER=Off \ -DWITH_OpenJPEG_ENCODER=Off \ -DENABLE_PLUGIN_LOADING=Off \ diff --git a/stages/99-onnx.sh b/stages/99-onnx.sh new file mode 100644 index 0000000..96bf7e6 --- /dev/null +++ b/stages/99-onnx.sh @@ -0,0 +1,119 @@ +#!/usr/bin/env -S bash -euo pipefail + +echo "Download onnx..." +mkdir -p onnx + +_version='1.16.3' + +case "$TARGET" in + *windows*) + # We just download the MS pre-compiled binaries which include the DirectML backend and are most likely better optimized than what we can build + curl_tar "https://www.nuget.org/api/v2/package/Microsoft.ML.OnnxRuntime.DirectML/${_version}" onnx 0 + + mkdir -p "$OUT"/{bin,lib,include} + + mv onnx/build/native/include "${OUT}/include/onnxruntime" + + case "${TARGET%%-*}" in + x86_64) + cd onnx/runtimes/win-x64/native + ;; + aarch64) + cd onnx/runtimes/win-arm64/native + ;; + esac + + mv onnxruntime.dll "${OUT}/bin/" + mv onnxruntime.lib "${OUT}/lib/" + + exit 0 + ;; +esac + +curl_tar "https://github.com/microsoft/onnxruntime/archive/refs/tags/v${_version}.tar.gz" onnx 1 + +# Patch to only include execinfo.h on supported environments +sed -i 's/defined(__ANDROID__)/defined(__ANDROID__) \&\& (defined(__APPLE__) || defined(__GLIBC__) || defined(PLATFORM_IS_BSD))/g' onnx/onnxruntime/core/platform/posix/stacktrace.cc + +# Remove unused components +rm -r onnx/{.*,requirements*.txt,*swift,*.py,cgmanifests,dockerfiles,objectivec,ORT_icon_for_light_bg.png,rust,samples,java,docs,orttraining,js,csharp,winml,onnxruntime/{wasm,test,tool,python,core/flatbuffers/ort_flatbuffers_py,contrib_ops/{js,rocm,cuda}}} + +# Backup source +bak_src 'onnx' + +mkdir -p onnx/build +cd onnx/build + +echo "Build onnx..." + +# Enable caching cmake downloaded deps +mkdir -p "/root/.cache/onnx_deps/${TARGET}" +ln -sf "/root/.cache/onnx_deps/${TARGET}" _deps + +args=( + -DCMAKE_TLS_VERIFY=On + -DBUILD_PKGCONFIG_FILES=Off + -Donnxruntime_USE_XNNPACK=On + -Donnxruntime_BUILD_SHARED_LIB=On + -Donnxruntime_CROSS_COMPILING=On + -Donnxruntime_ENABLE_LTO="$([ "${LTO:-1}" -eq 1 ] && echo On || echo Off)" + -DONNX_CUSTOM_PROTOC_EXECUTABLE=/usr/bin/protoc + -DPython_EXECUTABLE=/usr/bin/python3 + -DPYTHON_EXECUTABLE=/usr/bin/python3 + -Donnxruntime_RUN_ONNX_TESTS=Off + -Donnxruntime_USE_MPI=Off + -Donnxruntime_USE_TELEMETRY=Off + -DOnnxruntime_GCOV_COVERAGE=Off + -Donnxruntime_BUILD_MS_EXPERIMENTAL_OPS=Off + -Donnxruntime_BUILD_JAVA=Off + -Donnxruntime_BUILD_OBJC=Off + -Donnxruntime_BUILD_UNIT_TESTS=Off + -Donnxruntime_BUILD_NODEJS=Off + -Donnxruntime_BUILD_CSHARP=Off + -Donnxruntime_BUILD_BENCHMARKS=Off + -Donnxruntime_BUILD_APPLE_FRAMEWORK=Off + -Donnxruntime_ENABLE_PYTHON=Off + -Donnxruntime_ENABLE_MEMORY_PROFILE=Off + -Donnxruntime_ENABLE_TRAINING=Off + -Donnxruntime_GENERATE_TEST_REPORTS=Off + -DFETCHCONTENT_QUIET=Off + -DNSYNC_ENABLE_TESTS=Off + -Dprotobuf_BUILD_TESTS=Off + -Dprotobuf_BUILD_PROTOBUF_BINARIES=Off + -Dprotobuf_BUILD_PROTOC_BINARIES=Off + -DFLATBUFFERS_BUILD_TESTS=Off + -DFLATBUFFERS_INSTALL=Off + -DFLATBUFFERS_BUILD_FLATC=Off + -DCPUINFO_BUILD_TOOLS=Off + -DCPUINFO_BUILD_UNIT_TESTS=Off + -DCPUINFO_BUILD_MOCK_TESTS=Off + -DCPUINFO_BUILD_BENCHMARKS=Off + -DCPUINFO_BUILD_PKG_CONFIG=Off + -DEIGEN_BUILD_PKGCONFIG=Off +) + +case "$TARGET" in + *darwin*) + args+=( + -Donnxruntime_USE_COREML=On + ) + ;; +esac + +# WARNING: Must not set Shared Library to On, or else it will fail to build. This is already handled above by onnx custom argument. +env PREFIX="$OUT" cmake "${args[@]}" ../cmake + +case "$TARGET" in + *linux*) + # Fix google_nsync trying to compile C code as C++, which zig does not support + sed -i 's/foreach (s IN ITEMS ${NSYNC_COMMON_SRC} ${NSYNC_OS_CPP_SRC})/foreach (s IN ITEMS ${NSYNC_COMMON_SRC} ${NSYNC_OS_CPP_SRC})\nget_filename_component(sle ${s} NAME_WLE)/g' _deps/google_nsync-src/CMakeLists.txt + sed -i 's/cpp\/${s}/cpp\/${sle}.cc/g' _deps/google_nsync-src/CMakeLists.txt + + # Regenerate build files after cmake patches + env PREFIX="$OUT" cmake "${args[@]}" ../cmake + ;; +esac + +ninja -j"$(nproc)" + +ninja install diff --git a/stages/99-pdfium.sh b/stages/99-pdfium.sh index b97e100..872e401 100644 --- a/stages/99-pdfium.sh +++ b/stages/99-pdfium.sh @@ -3,7 +3,7 @@ echo "Download pdfium..." mkdir -p pdfium -_tag='chromium/6110' +_tag='chromium/6150' case "$TARGET" in x86_64-windows*) _name='win-x64' @@ -38,12 +38,12 @@ curl_tar "https://github.com/bblanchon/pdfium-binaries/releases/download/${_tag} cd pdfium # Install -mkdir -p "$OUT/include" +mkdir -p "$OUT"/{bin,lib,include} case "$TARGET" in *windows*) - mv bin "$OUT/bin" + mv bin/* "${OUT}/bin" mv lib/pdfium.dll.lib lib/pdfium.lib ;; esac -mv lib "$OUT/lib" -mv include "$OUT/include/libpdfium" +mv lib/* "${OUT}/lib/" +mv include "${OUT}/include/libpdfium" diff --git a/stages/99-yolov8.sh b/stages/99-yolov8.sh new file mode 100644 index 0000000..86ca6cf --- /dev/null +++ b/stages/99-yolov8.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env -S bash -euo pipefail + +echo "Download yolov8 model..." + +mkdir -p "$OUT"/{models,licenses} +curl -o "${OUT}/models/yolov8s.onnx" "https://github.com/spacedriveapp/native-deps/releases/download/yolo-2023-12-05/yolov8s.onnx" +curl -o "${OUT}/licenses/yolov8s.LICENSE" 'https://raw.githubusercontent.com/ultralytics/assets/main/LICENSE'