From b4d46ffb1bc1be370ba00e432cd998c73727a8e8 Mon Sep 17 00:00:00 2001 From: Ryo Yamashita Date: Sat, 10 Feb 2024 19:31:41 +0900 Subject: [PATCH] Add `decrypt_vv_model` --- .github/workflows/build.yml | 102 ++++++++++++++++++++++++++++-------- 1 file changed, 79 insertions(+), 23 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 841356e..8d03215 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -36,7 +36,7 @@ jobs: include: - artifact_name: onnxruntime-win-x64 os: windows-2022 - build_opts: --cmake_extra_defines CMAKE_SYSTEM_NAME=Windows CMAKE_SYSTEM_PROCESSOR=x86_64 --config Release --parallel --compile_no_warning_as_error --update --build --build_shared_lib + build_opts: --cmake_extra_defines CMAKE_SYSTEM_NAME=Windows CMAKE_SYSTEM_PROCESSOR=x86_64 Rust_CARGO_TARGET=x86_64-pc-windows-msvc --config Release --parallel --compile_no_warning_as_error --update --build --build_shared_lib result_dir: build/Release release_config: Release - artifact_name: onnxruntime-win-x64-gpu @@ -45,17 +45,17 @@ jobs: # Windowsの場合デフォルトのパッケージ群では不十分であるため、必要そうなパッケージを指定する。ただしいくつかは不要かもしれない cuda_sub_packages: '["cudart", "cuobjdump", "nvcc", "nvdisasm", "thrust", "cublas_dev", "cufft_dev", "curand_dev", "cusolver_dev", "cusparse_dev", "visual_studio_integration"]' cudnn_url: https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/windows-x86_64/cudnn-windows-x86_64-8.9.2.26_cuda12-archive.zip - build_opts: --cmake_extra_defines CMAKE_SYSTEM_NAME=Windows CMAKE_SYSTEM_PROCESSOR=x86_64 --config Release --parallel --compile_no_warning_as_error --update --build --build_shared_lib --use_dml --use_cuda --cuda_version 12.2 + build_opts: --cmake_extra_defines CMAKE_SYSTEM_NAME=Windows CMAKE_SYSTEM_PROCESSOR=x86_64 Rust_CARGO_TARGET=x86_64-pc-windows-msvc --config Release --parallel --compile_no_warning_as_error --update --build --build_shared_lib --use_dml --use_cuda --cuda_version 12.2 result_dir: build/Release release_config: Release - artifact_name: onnxruntime-win-x86 os: windows-2022 - build_opts: --x86 --cmake_extra_defines CMAKE_SYSTEM_NAME=Windows CMAKE_SYSTEM_PROCESSOR=x86 --config Release --parallel --compile_no_warning_as_error --update --build --build_shared_lib + build_opts: --x86 --cmake_extra_defines CMAKE_SYSTEM_NAME=Windows CMAKE_SYSTEM_PROCESSOR=x86 Rust_CARGO_TARGET=i686-pc-windows-msvc --config Release --parallel --compile_no_warning_as_error --update --build --build_shared_lib result_dir: build/Release release_config: Release - artifact_name: onnxruntime-linux-x64 os: ubuntu-20.04 - build_opts: --cmake_extra_defines CMAKE_SYSTEM_NAME=Linux CMAKE_SYSTEM_PROCESSOR=x86_64 --config Release --parallel --compile_no_warning_as_error --update --build --build_shared_lib + build_opts: --cmake_extra_defines CMAKE_SYSTEM_NAME=Linux CMAKE_SYSTEM_PROCESSOR=x86_64 Rust_CARGO_TARGET=x86_64-unknown-linux-gnu --config Release --parallel --compile_no_warning_as_error --update --build --build_shared_lib result_dir: build release_config: Release - artifact_name: onnxruntime-linux-x64-gpu @@ -63,7 +63,7 @@ jobs: cuda_version: 12.2.2 cuda_sub_packages: "[]" # デフォルト cudnn_url: https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-x86_64/cudnn-linux-x86_64-8.9.2.26_cuda12-archive.tar.xz - build_opts: --cmake_extra_defines CMAKE_SYSTEM_NAME=Linux CMAKE_SYSTEM_PROCESSOR=x86_64 --config Release --parallel --compile_no_warning_as_error --update --build --build_shared_lib --use_cuda + build_opts: --cmake_extra_defines CMAKE_SYSTEM_NAME=Linux CMAKE_SYSTEM_PROCESSOR=x86_64 Rust_CARGO_TARGET=x86_64-unknown-linux-gnu --config Release --parallel --compile_no_warning_as_error --update --build --build_shared_lib --use_cuda result_dir: build release_config: Release - artifact_name: onnxruntime-linux-armhf @@ -72,7 +72,7 @@ jobs: cxx_version: "10" linux_cross_arch: arm-linux-gnueabihf symlink_workaround: true - build_opts: --arm --cmake_extra_defines CMAKE_SYSTEM_NAME=Linux CMAKE_SYSTEM_PROCESSOR=armv7l --config Release --parallel --compile_no_warning_as_error --update --build --build_shared_lib + build_opts: --arm --cmake_extra_defines CMAKE_SYSTEM_NAME=Linux CMAKE_SYSTEM_PROCESSOR=armv7l Rust_CARGO_TARGET=armv7-unknown-linux-gnueabihf --config Release --parallel --compile_no_warning_as_error --update --build --build_shared_lib result_dir: build release_config: Release - artifact_name: onnxruntime-linux-arm64 @@ -81,45 +81,48 @@ jobs: cxx_version: "10" linux_cross_arch: aarch64-linux-gnu symlink_workaround: true - build_opts: --arm64 --cmake_extra_defines CMAKE_SYSTEM_NAME=Linux CMAKE_SYSTEM_PROCESSOR=aarch64 --config Release --parallel --compile_no_warning_as_error --update --build --build_shared_lib + build_opts: --arm64 --cmake_extra_defines CMAKE_SYSTEM_NAME=Linux CMAKE_SYSTEM_PROCESSOR=aarch64 Rust_CARGO_TARGET=aarch64-unknown-linux-gnu --config Release --parallel --compile_no_warning_as_error --update --build --build_shared_lib result_dir: build release_config: Release - artifact_name: onnxruntime-osx-arm64 - os: macos-11 - build_opts: --cmake_extra_defines CMAKE_SYSTEM_NAME=Darwin CMAKE_OSX_ARCHITECTURES=arm64 --config Release --parallel --compile_no_warning_as_error --update --build --build_shared_lib + os: macos-12 + build_opts: --cmake_extra_defines CMAKE_SYSTEM_NAME=Darwin CMAKE_OSX_ARCHITECTURES=arm64 Rust_CARGO_TARGET=aarch64-apple-darwin --config Release --parallel --compile_no_warning_as_error --update --build --build_shared_lib result_dir: build release_config: Release - artifact_name: onnxruntime-osx-x86_64 - os: macos-11 - build_opts: --cmake_extra_defines CMAKE_SYSTEM_NAME=Darwin CMAKE_OSX_ARCHITECTURES=x86_64 --config Release --parallel --compile_no_warning_as_error --update --build --build_shared_lib + os: macos-12 + build_opts: --cmake_extra_defines CMAKE_SYSTEM_NAME=Darwin CMAKE_OSX_ARCHITECTURES=x86_64 Rust_CARGO_TARGET=x86_64-apple-darwin --config Release --parallel --compile_no_warning_as_error --update --build --build_shared_lib result_dir: build release_config: Release - artifact_name: onnxruntime-android-x64 - os: ubuntu-20.04 - build_opts: --android_abi x86_64 --cmake_extra_defines CMAKE_SYSTEM_NAME=Android CMAKE_SYSTEM_PROCESSOR=x86_64 --config Release --parallel --compile_no_warning_as_error --update --build --build_shared_lib --android + os: ubuntu-22.04 + build_opts: --android_abi x86_64 --cmake_generator Ninja --cmake_extra_defines CMAKE_SYSTEM_NAME=Android CMAKE_SYSTEM_PROCESSOR=x86_64 "CMAKE_ANDROID_NDK=$ANDROID_NDK_HOME" Rust_CARGO_TARGET=x86_64-linux-android --config Release --parallel --compile_no_warning_as_error --update --build --build_shared_lib --android result_dir: build release_config: Release - artifact_name: onnxruntime-android-arm64 - os: ubuntu-20.04 - build_opts: --android_abi arm64-v8a --cmake_extra_defines CMAKE_SYSTEM_NAME=Android CMAKE_SYSTEM_PROCESSOR=aarch64 --config Release --parallel --compile_no_warning_as_error --update --build --build_shared_lib --android + os: ubuntu-22.04 + build_opts: --android_abi arm64-v8a --cmake_generator Ninja --cmake_extra_defines CMAKE_SYSTEM_NAME=Android CMAKE_SYSTEM_PROCESSOR=aarch64 "CMAKE_ANDROID_NDK=$ANDROID_NDK_HOME" Rust_CARGO_TARGET=aarch64-linux-android --config Release --parallel --compile_no_warning_as_error --update --build --build_shared_lib --android result_dir: build release_config: Release - artifact_name: onnxruntime-ios-arm64 - os: macos-12 - build_opts: --config Release --parallel --compile_no_warning_as_error --update --build --build_shared_lib --skip_tests --use_xcode --ios --apple_sysroot iphoneos --osx_arch arm64 --apple_deploy_target 16.0 + os: macos-13 + build_opts: --cmake_extra_defines CMAKE_SYSTEM_NAME=iOS Rust_CARGO_TARGET=aarch64-apple-ios --config Release --parallel --compile_no_warning_as_error --update --build --build_shared_lib --skip_tests --use_xcode --ios --apple_sysroot iphoneos --osx_arch arm64 --apple_deploy_target 16.0 build_opts_workaround_protoc: --path_to_protoc_exe /usr/local/opt/protobuf@21/bin/protoc + #build_opts_workaround_protoc: --path_to_protoc_exe /opt/homebrew/opt/protobuf@21/bin/protoc result_dir: build/Release release_config: Release-iphoneos - artifact_name: onnxruntime-ios-sim-arm64 - os: macos-12 - build_opts: --config Release --parallel --compile_no_warning_as_error --update --build --build_shared_lib --skip_tests --use_xcode --ios --apple_sysroot iphonesimulator --osx_arch arm64 --apple_deploy_target 16.0 + os: macos-13 + build_opts: --cmake_extra_defines CMAKE_SYSTEM_NAME=iOS Rust_CARGO_TARGET=aarch64-apple-ios-sim --config Release --parallel --compile_no_warning_as_error --update --build --build_shared_lib --skip_tests --use_xcode --ios --apple_sysroot iphonesimulator --osx_arch arm64 --apple_deploy_target 16.0 build_opts_workaround_protoc: --path_to_protoc_exe /usr/local/opt/protobuf@21/bin/protoc + #build_opts_workaround_protoc: --path_to_protoc_exe /opt/homebrew/opt/protobuf@21/bin/protoc result_dir: build/Release release_config: Release-iphonesimulator - artifact_name: onnxruntime-ios-sim-x86_64 - os: macos-12 - build_opts: --config Release --parallel --compile_no_warning_as_error --update --build --build_shared_lib --skip_tests --use_xcode --ios --apple_sysroot iphonesimulator --osx_arch x86_64 --apple_deploy_target 16.0 + os: macos-13 + build_opts: --cmake_extra_defines CMAKE_SYSTEM_NAME=iOS Rust_CARGO_TARGET=x86_64-apple-ios --config Release --parallel --compile_no_warning_as_error --update --build --build_shared_lib --skip_tests --use_xcode --ios --apple_sysroot iphonesimulator --osx_arch x86_64 --apple_deploy_target 16.0 build_opts_workaround_protoc: --path_to_protoc_exe /usr/local/opt/protobuf@21/bin/protoc + #build_opts_workaround_protoc: --path_to_protoc_exe /opt/homebrew/opt/protobuf@21/bin/protoc result_dir: build/Release release_config: Release-iphonesimulator @@ -149,9 +152,9 @@ jobs: - name: Checkout uses: actions/checkout@v4 with: - repository: microsoft/onnxruntime + repository: qryxip/onnxruntime submodules: true - ref: v${{ env.ONNXRUNTIME_VERSION }} + ref: 82b6fd565b54932504f558c5995f704792093922 - name: Checkout builder uses: actions/checkout@v4 with: @@ -174,6 +177,14 @@ jobs: key: ${{ matrix.artifact_name }}-v${{ env.ONNXRUNTIME_VERSION }}-cache-${{ hashFiles('matrix.json') }} + - name: Free Disk Space + uses: jlumbroso/free-disk-space@main + if: steps.cache-build-result.outputs.cache-hit != 'true' && runner.os == 'Linux' && contains(matrix.build_opts, '--use_cuda') + with: + tool-cache: false + large-packages: false + swap-storage: false + - name: Install tree if: steps.cache-build-result.outputs.cache-hit != 'true' && runner.os == 'Windows' uses: msys2/setup-msys2@v2 @@ -202,6 +213,13 @@ jobs: # Workaround for protoc https://github.com/microsoft/onnxruntime/issues/16238#issuecomment-1590398821 brew install protobuf@21 + - name: Install build dependencies on android + if: steps.cache-build-result.outputs.cache-hit != 'true' && startsWith(matrix.artifact_name, 'onnxruntime-android') + run: | + sudo apt-get update + sudo apt-get install -y \ + ninja-build + # ONNX Runtime v1.16.1 requires CMake 3.27 or higher. - name: Install CMake if: steps.cache-build-result.outputs.cache-hit != 'true' && startsWith(matrix.os, 'ubuntu') @@ -247,6 +265,30 @@ jobs: echo "CUDNN_HOME=$cudnn_path" >> "$GITHUB_ENV" + - name: Extract Rust toolchain and target triple + id: rust-toolchain-and-target-triple + if: steps.cache-build-result.outputs.cache-hit != 'true' + run: | + echo "toolchain=$(cat ./rust-toolchain)" >> "$GITHUB_OUTPUT" + # shellcheck disable=SC2001 + echo "target=$(sed -E 's/.*Rust_CARGO_TARGET=([a-z0-9_-]+).*/\1/' <<< '${{ matrix.build_opts }}')" >> "$GITHUB_OUTPUT" + + - name: Set up Rust + if: steps.cache-build-result.outputs.cache-hit != 'true' + uses: dtolnay/rust-toolchain@master + with: + toolchain: ${{ steps.rust-toolchain-and-target-triple.outputs.toolchain }} + targets: ${{ steps.rust-toolchain-and-target-triple.outputs.target }} + + - name: Install cargo-binstall + uses: taiki-e/install-action@cargo-binstall + + - name: Install cxxbridge-cmd + run: | + md=$(cargo metadata --manifest-path ./decrypt_vv_model/Cargo.toml --format-version 1) + version=$(jq -r '.packages[] | select(.name == "cxx").version' <<< "$md") + cargo binstall "cxxbridge-cmd@$version" --no-confirm --log-level debug + - name: Configure build environment for non-x86_64 Linux if: steps.cache-build-result.outputs.cache-hit != 'true' && startsWith(matrix.os, 'ubuntu') && matrix.linux_cross_arch run: | @@ -271,6 +313,20 @@ jobs: # https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2004-Readme.md#environment-variables-2 echo "ANDROID_NDK_HOME=$ANDROID_NDK_LATEST_HOME" >> "$GITHUB_ENV" + - name: ls + if: steps.cache-build-result.outputs.cache-hit != 'true' && runner.os == 'macOS' + run: ls -l /Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/lib || true + + - name: cat + if: steps.cache-build-result.outputs.cache-hit != 'true' && runner.os == 'macOS' + run: cat /Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/lib/libSystem.tbd || true + + - name: CMake version + if: steps.cache-build-result.outputs.cache-hit != 'true' + run: | + type cmake + cmake --version + - name: Build ONNX Runtime if: steps.cache-build-result.outputs.cache-hit != 'true' run: |