Merge pull request #1 from spacedriveapp/add-onnx-update-ffmpeg

spacedriveapp · Dec 6, 2023 · df2bca9 · df2bca9
2 parents 3c83efe + 6b63f10
commit df2bca9
Show file tree

Hide file tree

Showing 27 changed files with 620 additions and 88 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -3,7 +3,7 @@
 ARG OUT="/opt/out"
 ARG TARGET="x86_64-linux-gnu"
 ARG ZIG_VERSION="0.11.0"
-ARG MESON_VERSION="1.2.3"
+ARG MESON_VERSION="1.3.0"
 ARG CMAKE_VERSION="3.27.0"
 ARG PATCHELF_VERSION="0.18.0"
 ARG MACOS_SDK_VERSION="14.0"
@@ -50,14 +50,16 @@ RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/var/lib/
 	clang-16 `
 	pkg-config `
 	ninja-build `
-	libarchive-tools
+	libarchive-tools `
+	protobuf-compiler
 
 # Configure sysroot and prefix
 ARG OUT
 ENV OUT="${OUT:?}"
 ENV PREFIX="/opt/prefix"
 ENV SYSROOT="/opt/sysroot"
 ENV CCTOOLS="/opt/cctools"
+ENV CIPHERSUITES="TLS_AES_128_GCM_SHA256:TLS_CHACHA20_POLY1305_SHA256:TLS_AES_256_GCM_SHA384:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384"
 
 # Ensure sysroot and cctools are present on PATH
 ENV PATH="${CCTOOLS}/bin:${SYSROOT}/bin:$PATH"
@@ -154,8 +156,12 @@ RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/var/lib/
 	/srv/02-tapi.sh
 
 RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/var/lib/apt --mount=type=cache,target=/root/.cache `
-	--mount=type=bind,source=stages/00-apple/03-cctools.sh,target=/srv/03-cctools.sh `
-	/srv/03-cctools.sh
+	--mount=type=bind,source=stages/00-apple/03-dispatch.sh,target=/srv/03-dispatch.sh `
+	/srv/03-dispatch.sh
+
+RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/var/lib/apt --mount=type=cache,target=/root/.cache `
+	--mount=type=bind,source=stages/00-apple/04-cctools.sh,target=/srv/04-cctools.sh `
+	/srv/04-cctools.sh
 
 # Ensure no one tries to call the native system linker
 RUN ln -s '/usr/bin/false' "${SYSROOT}/bin/ld"
@@ -409,12 +415,6 @@ COPY --from=layer-50-zimg "${PREFIX}/." "$PREFIX"
 
 #--
 
-FROM layer-45 AS layer-99-heif
-
-RUN --mount=type=cache,target=/root/.cache `
-	--mount=type=bind,source=stages/99-heif.sh,target=/srv/stage.sh `
-	/srv/build.sh
-
 FROM layer-00 AS layer-99-protoc
 
 ADD https://raw.githubusercontent.com/protocolbuffers/protobuf/v25.0/LICENSE '/srv/protoc/LICENSE'
@@ -429,12 +429,32 @@ RUN --mount=type=cache,target=/root/.cache `
 	--mount=type=bind,source=stages/99-pdfium.sh,target=/srv/stage.sh `
 	/srv/build.sh
 
+FROM layer-00 AS layer-99-yolo
+
+RUN --mount=type=cache,target=/root/.cache `
+	--mount=type=bind,source=stages/99-yolov8.sh,target=/srv/stage.sh `
+	/srv/build.sh
+
+FROM layer-20 AS layer-99-onnx
+
+RUN --mount=type=cache,target=/root/.cache `
+	--mount=type=bind,source=stages/99-onnx.sh,target=/srv/stage.sh `
+	/srv/build.sh
+
 FROM layer-50 AS layer-99-ffmpeg
 
 RUN --mount=type=cache,target=/root/.cache `
 	--mount=type=bind,source=stages/99-ffmpeg.sh,target=/srv/stage.sh `
 	/srv/build.sh
 
+FROM layer-45 AS layer-99-heif
+
+COPY --from=layer-99-ffmpeg "${OUT}/." "$PREFIX"
+
+RUN --mount=type=cache,target=/root/.cache `
+	--mount=type=bind,source=stages/99-heif.sh,target=/srv/stage.sh `
+	/srv/build.sh
+
 FROM layer-00 AS layer-99
 
 COPY --from=layer-99-heif "${OUT}/." "$OUT"
@@ -447,6 +467,12 @@ COPY --from=layer-99-protoc "${PREFIX}/licenses/." "${OUT}/licenses"
 COPY --from=layer-99-pdfium "${OUT}/." "$OUT"
 COPY --from=layer-99-pdfium "${PREFIX}/licenses/." "${OUT}/licenses"
 
+COPY --from=layer-99-yolo "${OUT}/." "$OUT"
+
+COPY --from=layer-99-onnx "${OUT}/." "$OUT"
+COPY --from=layer-99-onnx "${PREFIX}/srv/." "${OUT}/srv"
+COPY --from=layer-99-onnx "${PREFIX}/licenses/." "${OUT}/licenses"
+
 COPY --from=layer-99-ffmpeg "${OUT}/." "$OUT"
 COPY --from=layer-99-ffmpeg "${PREFIX}/srv/." "${OUT}/srv"
 COPY --from=layer-99-ffmpeg "${PREFIX}/licenses/." "${OUT}/licenses"
@@ -456,7 +482,7 @@ RUN rm -rf "${OUT}/share" "${OUT}/lib/pkgconfig" "${OUT}/lib/cmake"
 RUN find "${OUT}"  \( -name '*.def' -o -name '*.dll.a' \) -delete
 
 # Move .lib files to the lib folder (Windows target only)
-RUN find "${OUT}/bin" -name '*.lib' -exec install -Dt ../lib/ -m a-rwx,u+rw,g+r,o+r {} +
+RUN if [ -d "${OUT}/bin" ]; then find "${OUT}/bin" -name '*.lib' -exec install -Dt ../lib/ -m a-rwx,u+rw,g+r,o+r {} + ; fi
 
 # Copy .lib to .dll.a (Windows target only)
 RUN find "$OUT/lib" -name '*.lib' -exec `
@@ -473,7 +499,7 @@ RUN --mount=type=cache,target=/root/.cache `
 RUN find "$OUT" -type f \( -name '*.so' -o -name '*.so.*' \) -exec patchelf --set-rpath '$ORIGIN' {} \;
 
 # Remove non executable files from bin folder
-RUN find "${OUT}/bin" -type f -not -executable -delete
+RUN if [ -d "${OUT}/bin" ]; then find "${OUT}/bin" -type f -not -executable -delete; fi
 
 # Remove empty directories
 RUN find "$OUT" -type d -delete 2>/dev/null || true

diff --git a/scripts/build.sh b/scripts/build.sh
@@ -47,7 +47,6 @@ LDFLAGS="-L${PREFIX}/lib -pipe"
 case "$TARGET" in
   *linux*)
     FFLAGS="-fno-semantic-interposition"
-    CFLAGS="${CFLAGS} -D_GLIBCXX_ASSERTIONS"
     LDFLAGS="${LDFLAGS} -Wl,-z,relro,-z,now,-z,defs"
 
     case "$TARGET" in
@@ -59,11 +58,21 @@ case "$TARGET" in
         FFLAGS="${FFLAGS} -fno-stack-protector -fno-stack-check"
         ;;
     esac
+
+    case "$TARGET" in
+      *gnu)
+        CFLAGS="${CFLAGS} -D_GLIBCXX_ASSERTIONS=1"
+        ;;
+      *musl)
+        CFLAGS="${CFLAGS} -D_LARGEFILE64_SOURCE=1"
+        ;;
+    esac
     ;;
   *darwin*)
     # Apple tools and linker fails to LTO static libraries
     # https://github.com/tpoechtrager/osxcross/issues/366
     export LTO=0
+    export LD_LIBRARY_PATH="${CCTOOLS}/lib:/usr/local/lib:${LD_LIBRARY_PATH:-}"
 
     # Ugly workaround for apple linker not finding the macOS SDK's Framework directory
     ln -fs "${MACOS_SDKROOT}/System" '/System'
@@ -72,13 +81,11 @@ case "$TARGET" in
 
     case "$TARGET" in
       x86_64*)
-        export CMAKE_OSX_ARCHITECTURES='x86_64'
         export MACOSX_DEPLOYMENT_TARGET="10.15"
         export CMAKE_APPLE_SILICON_PROCESSOR='x86_64'
         LDFLAGS="${LDFLAGS} -Wl,-arch,x86_64"
         ;;
       aarch64*)
-        export CMAKE_OSX_ARCHITECTURES='aarch64'
         export MACOSX_DEPLOYMENT_TARGET="11.0"
         export CMAKE_APPLE_SILICON_PROCESSOR='aarch64'
         LDFLAGS="${LDFLAGS} -Wl,-arch,arm64"
@@ -103,6 +110,10 @@ export CFLAGS="${CFLAGS} ${FFLAGS}"
 export LDFLAGS="${LDFLAGS} ${FFLAGS}"
 export CXXFLAGS="${CFLAGS}"
 
+curl () {
+  env curl --proto '=https' --tlsv1.2 --ciphers "${CIPHERSUITES:?Missing curl ciphersuite}" --silent --show-error --fail --location "$@"
+}
+
 bak_src() {
   if ! { [ "$#" -eq 1 ] && [ -d "$1" ]; }; then
     echo "bak_src: <SRC_DIR>" >&2

diff --git a/scripts/cc.sh b/scripts/cc.sh
@@ -8,7 +8,7 @@ case "${TARGET:?TARGET envvar is required to be defined}" in
     # Set the glibc minimum version
     # This is the lowest we can go at the moment
     # https://github.com/ziglang/zig/issues/9412
-    TARGET="${TARGET}.2.18"
+    TARGET="${TARGET%%.*}.2.18"
     ;;
   x86_64-darwin-apple | x86_64-apple-darwin-macho)
     SDKROOT="$MACOS_SDKROOT"
@@ -64,6 +64,7 @@ sysroot=''
 assembler=0
 has_iphone=0
 preprocessor=0
+cpu_features=()
 assembler_file=0
 should_add_libcharset=0
 has_undefined_dynamic_lookup=0
@@ -77,7 +78,15 @@ while [ "$#" -gt 0 ]; do
     continue
   fi
 
-  if [ "$1" = '-o-' ] || [ "$1" = '-o=-' ]; then
+  if [ "$1" = '-Xlinker' ] || [ "$1" = '--for-linker' ]; then
+    l_args+=("${2:?Linker argument not passed}")
+    shift 2
+    continue
+  elif (case "$1" in --for-linker=*) exit 0 ;; *) exit 1 ;; esac) then
+    l_args+=("${1#*=}")
+    shift
+    continue
+  elif [ "$1" = '-o-' ] || [ "$1" = '-o=-' ]; then
     # -E redirect to stdout by default, -o - breaks it so we ignore it
     stdout=1
   elif [ "$1" = '-o' ] && [ "${2:-}" = '-' ]; then
@@ -120,7 +129,7 @@ while [ "$#" -gt 0 ]; do
     # https://github.com/llvm/llvm-project/issues/47432
     # https://github.com/llvm/llvm-project/issues/66912
     true
-  elif [ "$1" = '-xassembler' ] || [ "$1" = '--language=assembler' ]; then
+  elif [ "$1" = '-xassembler' ] || [ "$1" = '-Xassembler' ] || [ "$1" = '--language=assembler' ]; then
     # Zig behaves very oddly when passed the explicit assembler language option
     # https://github.com/ziglang/zig/issues/10915
     # https://github.com/ziglang/zig/pull/13544
@@ -130,7 +139,10 @@ while [ "$#" -gt 0 ]; do
     shift 2
     continue
   elif (case "$1" in -mcpu=* | -march=*) exit 0 ;; *) exit 1 ;; esac) then
-    # Ignore -mcpu and -march flags, we set them ourselves
+    # Save each feature to the cpu_features array
+    IFS='+' read -ra _features <<<"$1"
+    unset "_features[0]"
+    cpu_features+=("${_features[@]}")
     true
   else
     if (case "$TARGET" in *darwin*) exit 0 ;; *) exit 1 ;; esac) then
@@ -269,6 +281,9 @@ while [ "$#" -gt 0 ]; do
         l_args+=("$1")
         ;;
     esac
+  elif (case "$1" in --version-script=*) exit 0 ;; *) exit 1 ;; esac) then
+    # Zig only support --version-script with a separate argument
+    l_args+=(--version-script "${1#*=}")
   else
     l_args+=("$1")
   fi
@@ -282,7 +297,9 @@ if [ $stdout -eq 1 ] && ! [ $preprocessor -eq 1 ]; then
 fi
 
 # Work-around Zig not respecting -fno-lto when -flto is set
-if [ -n "$lto" ]; then
+if [ "${LTO:-1}" -eq 0 ]; then
+  argv+=('-fno-lto')
+elif [ -n "$lto" ]; then
   argv+=("$lto")
 fi
 
@@ -292,32 +309,56 @@ if [ $should_add_libcharset -eq 1 ]; then
 fi
 
 # Compiler specific flags
+features=""
+for feature in "${cpu_features[@]}"; do
+  case "$CMD" in
+    clang*) ;;
+    *)
+      if [ "$assembler" -eq 0 ] || [ "$preprocessor" -eq 1 ]; then
+        # Zig specific changes
+        case "$feature" in
+          fp16)
+            feature="fullfp16"
+            ;;
+        esac
+      fi
+      ;;
+  esac
+
+  features="${features}+${feature}"
+done
+
 case "${TARGET:-}" in
   x86_64*)
     case "${TARGET:-}" in
       *darwin*)
         # macOS 10.15 (Catalina) only supports Macs made with Ivy Bridge or later
-        c_argv+=(-march=ivybridge)
+        c_argv+=("-march=ivybridge${features}")
         ;;
       *)
-        c_argv+=(-march=x86_64_v2)
+        c_argv+=("-march=x86_64_v2${features}")
         ;;
     esac
     ;;
   arm64* | aarch64*)
-    case "${TARGET:-}" in
-      *darwin*)
-        c_argv+=(-mcpu=apple-m1)
-        ;;
-      *)
-        # Raspberry Pi 3
-        c_argv+=(-mcpu=cortex_a53)
-        ;;
-    esac
+    if [ "$assembler" -eq 1 ] && [ "$preprocessor" -eq 0 ]; then
+      # This is an workaround for zig not supporting some features when compiling arm assembler code
+      c_argv+=(-Xassembler "-march=armv8.2-a${features}")
+    else
+      case "${TARGET:-}" in
+        *darwin*)
+          c_argv+=("-mcpu=apple-m1${features}")
+          ;;
+        *)
+          # Raspberry Pi 3
+          c_argv+=("-mcpu=cortex_a53${features}")
+          ;;
+      esac
+    fi
     ;;
 esac
 
-# Like -O2 with extra optimizations to reduce code size
+# Like -O2, but with extra optimizations to reduce code size
 c_argv+=(-Os)
 
 # Resolve sysroot arguments per target