From 39b19361db8616ff97b708ab152258808eaa04ec Mon Sep 17 00:00:00 2001 From: Roy Shilkrot Date: Sat, 4 Mar 2023 21:53:26 -0500 Subject: [PATCH] More CI & Packaging work (#151) * trial * cmake * ox build onx ocv static * fix build script * s3 download , cmake * don't use realpath * fix * protobuf * early exit? * fixup * make surework dir present * quiet dl, include dir * lint cmake --- .github/actions/build-plugin/action.yml | 5 ++ .github/scripts/.Aptfile | 3 + .github/scripts/.Brewfile | 3 +- .github/scripts/.Brewfile.lock.json | 2 +- .github/workflows/main.yml | 4 +- .gitignore | 2 +- CMakeLists.txt | 25 ++++---- cmake/FindOnnxruntime.cmake | 24 ++++---- scripts/makeOnnxruntime_osx.sh | 77 ++++++++++++++++++++++++- scripts/makeOpenCV_osx.sh | 77 ++++++++++++++++++++++++- 10 files changed, 182 insertions(+), 40 deletions(-) diff --git a/.github/actions/build-plugin/action.yml b/.github/actions/build-plugin/action.yml index 1e91d155..3cc678d1 100644 --- a/.github/actions/build-plugin/action.yml +++ b/.github/actions/build-plugin/action.yml @@ -33,6 +33,11 @@ runs: env: CODESIGN_IDENT: ${{ inputs.codesignIdent }} run: | + set -e + + ${{ inputs.workingDirectory }}/scripts/makeOnnxruntime_osx.sh + ${{ inputs.workingDirectory }}/scripts/makeOpenCV_osx.sh + build_args=( -c ${{ inputs.config }} -t macos-${{ inputs.target }} diff --git a/.github/scripts/.Aptfile b/.github/scripts/.Aptfile index 59196f02..a15828d7 100644 --- a/.github/scripts/.Aptfile +++ b/.github/scripts/.Aptfile @@ -7,3 +7,6 @@ package 'ninja-build', bin: 'ninja' package 'pkg-config' package 'clang' package 'clang-format-13' +package 'libopencv-dev' +package 'libopencv-imgproc-dev' +package 'libopencv-core-dev' diff --git a/.github/scripts/.Brewfile b/.github/scripts/.Brewfile index 4f99d577..bca4173c 100644 --- a/.github/scripts/.Brewfile +++ b/.github/scripts/.Brewfile @@ -4,5 +4,4 @@ brew "cmake" brew "git" brew "jq" brew "ninja" -brew "onnxruntime" -brew "opencv" +brew "protobuf" diff --git a/.github/scripts/.Brewfile.lock.json b/.github/scripts/.Brewfile.lock.json index 2e26d423..bad7e4fa 100644 --- a/.github/scripts/.Brewfile.lock.json +++ b/.github/scripts/.Brewfile.lock.json @@ -393,7 +393,7 @@ "system": { "macos": { "ventura": { - "HOMEBREW_VERSION": "4.0.3-142-g0172ad1", + "HOMEBREW_VERSION": "4.0.3-149-g66303bd", "HOMEBREW_PREFIX": "/usr/local", "Homebrew/homebrew-core": "api", "CLT": "14.2.0.0.1.1668646533", diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index bccbb63c..33345fc2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -17,7 +17,7 @@ on: - main env: - PLUGIN_NAME: 'obs-plugintemplate' + PLUGIN_NAME: 'obs-backgroundremoval' jobs: clang_check: @@ -178,7 +178,7 @@ jobs: fail-fast: true matrix: arch: [x86_64] - if: false # always() + if: false # always() needs: [clang_check] outputs: commitHash: ${{ steps.setup.outputs.commitHash }} diff --git a/.gitignore b/.gitignore index 9016c904..a47b78b2 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,7 @@ /build64/ /release/ /installer/Output/ -/build_x86_x64/ +build_x86_64 .vscode .idea diff --git a/CMakeLists.txt b/CMakeLists.txt index 0747ea65..42e37cf7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -50,20 +50,6 @@ elseif(APPLE) endif() endif() -if(UNIX) - set(Onnxruntime_INCLUDE_HINT /usr/local/include/) -endif() - -if(APPLE) - set(OpenCV_DIR ${CMAKE_BINARY_DIR}/opencv/lib/cmake/opencv4) -endif() - -set(RELEASE_DIR - "${PROJECT_SOURCE_DIR}/release" - CACHE STRING "Directory for building release packages") - -set(CMAKE_PREFIX_PATH "${QTDIR}") -set(CMAKE_INCLUDE_CURRENT_DIR ON) set(CMAKE_MODULE_PATH "${CMAKE_MODULE_PATH};${CMAKE_SOURCE_DIR}/cmake") # In case you need C++ @@ -79,8 +65,19 @@ configure_file(src/plugin-macros.h.in ${CMAKE_SOURCE_DIR}/src/plugin-macros.gene target_sources(${CMAKE_PROJECT_NAME} PRIVATE src/plugin-macros.generated.h) +if(UNIX) + set(CMAKE_MODULE_PATH + ${CMAKE_MODULE_PATH} + "${CMAKE_SOURCE_DIR}/obs-build-dependencies/plugin-deps-2023-01-06-qt6-x86_64/lib/cmake/libobs/" + ) + set(OpenCV_DIR "${CMAKE_SOURCE_DIR}/build/opencv/lib/cmake/opencv4") + # add -framework Foundation to the target + set_target_properties(${CMAKE_PROJECT_NAME} PROPERTIES LINK_FLAGS "-framework Foundation") +endif() + # Import libobs as main plugin dependency find_package(libobs REQUIRED) + include(cmake/ObsPluginHelpers.cmake) find_package(Onnxruntime REQUIRED) diff --git a/cmake/FindOnnxruntime.cmake b/cmake/FindOnnxruntime.cmake index 1fcfb7e8..7ecd1c7c 100644 --- a/cmake/FindOnnxruntime.cmake +++ b/cmake/FindOnnxruntime.cmake @@ -4,21 +4,17 @@ # directory of onnxruntime libraries if(APPLE) - # use pkg-config to find the onnxruntime library - find_package(PkgConfig REQUIRED) + set(Onnxruntime_DIR_BUILD ${CMAKE_SOURCE_DIR}/build/onnxruntime) + file(GLOB Onnxruntime_LIBRARIES_EX ${Onnxruntime_DIR_BUILD}/lib/lib*.a) - pkg_check_modules(Onnxruntime REQUIRED IMPORTED_TARGET libonnxruntime) - if(NOT Onnxruntime_FOUND) - message(FATAL_ERROR "Onnxruntime not found by pkg-config (as expected from homebrew install)") - endif() - message(STATUS "Onnxruntime found by pkg-config") + find_package(Protobuf REQUIRED) + + set(Onnxruntime_LIBRARIES + ${Onnxruntime_LIBRARIES_EX} ${Protobuf_LIBRARIES} + CACHE STRING "Onnxruntime libraries") set(Onnxruntime_INCLUDE_DIR - ${Onnxruntime_INCLUDE_DIRS} + ${Onnxruntime_DIR_BUILD}/include CACHE STRING "Onnxruntime include directory") - set(Onnxruntime_LIBRARY_DIR - ${Onnxruntime_LIBRARY_DIRS} - CACHE STRING "Onnxruntime library directory") - set(Onnxruntime_LIBRARIES "${pkgcfg_lib_Onnxruntime_onnxruntime}") message(STATUS "Onnxruntime_INCLUDE_DIRS: ${Onnxruntime_INCLUDE_DIRS}") message(STATUS "Onnxruntime_LIBRARY_DIRS: ${Onnxruntime_LIBRARY_DIRS}") message(STATUS "Onnxruntime_LIBRARIES: ${Onnxruntime_LIBRARIES}") @@ -53,8 +49,8 @@ if(Onnxruntime_FOUND) ${Onnxruntime_INCLUDE_DIR} CACHE STRING "Onnxruntime include directories") if(APPLE) - set(Onnxruntime_INCLUDE_DIRS ${Onnxruntime_INCLUDE_DIRS} - ${Onnxruntime_INCLUDE_DIR}/core/session) + set(Onnxruntime_INCLUDE_DIRS ${Onnxruntime_INCLUDE_DIRS} ${Onnxruntime_INCLUDE_DIR}/onnxruntime + ${Onnxruntime_INCLUDE_DIR}/onnxruntime/core/session) endif() list(GET Onnxruntime_LIBRARIES 0 Onnxruntime_LIBRARY) get_filename_component(Onnxruntime_LIBRARY_DIR_EX ${Onnxruntime_LIBRARY} DIRECTORY) diff --git a/scripts/makeOnnxruntime_osx.sh b/scripts/makeOnnxruntime_osx.sh index 0845f124..65cf11cf 100755 --- a/scripts/makeOnnxruntime_osx.sh +++ b/scripts/makeOnnxruntime_osx.sh @@ -1,17 +1,88 @@ #!/bin/bash -WORK_DIR=$(git rev-parse --show-toplevel)/build +set -eax + +# get script directory +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +WORK_DIR="${SCRIPT_DIR}/../build" CURRENT_DIR=$(pwd) OUTPUT_DIR=$WORK_DIR/onnxruntime -if [[ -d $OUTPUT_DIR ]]; then - rm -rf $OUTPUT_DIR +function verify_onnxruntime_exists() { + # check that onnxruntime libraries and include files exist + if [[ ! -d $OUTPUT_DIR ]]; then + echo "onnxruntime not found in $OUTPUT_DIR" + return 1 + fi + if [[ ! -d $OUTPUT_DIR/include ]]; then + echo "onnxruntime include files not found in $OUTPUT_DIR/include" + return 1 + fi + if [[ ! -d $OUTPUT_DIR/lib ]]; then + echo "onnxruntime libraries not found in $OUTPUT_DIR/lib" + return 1 + fi + if [[ ! -f $OUTPUT_DIR/lib/libonnx.a ]]; then + echo "onnxruntime library not found in $OUTPUT_DIR/lib/libonnx.a" + return 1 + fi + if [[ ! -f $OUTPUT_DIR/include/onnxruntime/core/session/onnxruntime_c_api.h ]]; then + echo "onnxruntime includes not found in $OUTPUT_DIR/include/onnxruntime/core/session/onnxruntime_c_api.h" + return 1 + fi + echo "onnxruntime found in $OUTPUT_DIR" + return 0 +} + +# check if onnxruntime is already built using verify_onnxruntime_exists +if verify_onnxruntime_exists; then + echo "onnxruntime already built" + exit 0 +fi + +# try to download onnxruntime from s3 if not already downloaded, use wget if available +# https://obs-backgroundremoval-build.s3.amazonaws.com/onnxruntime-1.14.0-x86_64.tar.gz +if [[ ! $( which wget ) ]]; then + echo "wget is not available, please install it e.g. `$ brew install wget`" + exit 1 fi if [[ ! -d $WORK_DIR ]]; then + echo "creating work directory $WORK_DIR" mkdir -p $WORK_DIR fi +ONNXRT_TAR_FILENAME="onnxruntime-1.14.0-x86_64.tar.gz" +ONNXRT_TAR_LOCATION="${WORK_DIR}/${ONNXRT_TAR_FILENAME}" +if [[ ! -f $ONNXRT_TAR_LOCATION ]]; then + echo "onnxruntime tar file not found in $ONNXRT_TAR_LOCATION, downloading from s3" + wget -q -O $ONNXRT_TAR_LOCATION https://obs-backgroundremoval-build.s3.amazonaws.com/$ONNXRT_TAR_FILENAME +fi +if [[ ! -f $ONNXRT_TAR_LOCATION ]]; then + echo "onnxruntime tar file not found in $ONNXRT_TAR_LOCATION" + exit 1 +fi +echo "onnxruntime tar file found in $ONNXRT_TAR_LOCATION" + +cd "${WORK_DIR}/.." +echo "extracting onnxruntime tar file" +tar -xzf $ONNXRT_TAR_LOCATION +rm $ONNXRT_TAR_LOCATION +cd $CURRENT_DIR + +# verify onnxruntime exists +if verify_onnxruntime_exists; then + echo "onnxruntime successfully downloaded and extracted" + exit 0 +fi + +# build onnxruntime from source +echo "building onnxruntime from source" + +if [[ -d $OUTPUT_DIR ]]; then + rm -rf $OUTPUT_DIR +fi + cd $WORK_DIR ONNXRT_VER=1.14.0 diff --git a/scripts/makeOpenCV_osx.sh b/scripts/makeOpenCV_osx.sh index 40686fc3..495c7afa 100755 --- a/scripts/makeOpenCV_osx.sh +++ b/scripts/makeOpenCV_osx.sh @@ -1,17 +1,88 @@ #!/bin/bash -WORK_DIR=$(git rev-parse --show-toplevel)/build +set -e + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +WORK_DIR="${SCRIPT_DIR}/../build" CURRENT_DIR=$(pwd) OUTPUT_DIR=$WORK_DIR/opencv -if [[ -d $OUTPUT_DIR ]]; then - rm -rf $OUTPUT_DIR +function verify_opencv_exists() { + # check that opencv libraries and include files exist + if [[ ! -d $OUTPUT_DIR ]]; then + echo "opencv not found in $OUTPUT_DIR" + return 1 + fi + if [[ ! -d $OUTPUT_DIR/include ]]; then + echo "opencv include files not found in $OUTPUT_DIR/include" + return 1 + fi + if [[ ! -d $OUTPUT_DIR/lib ]]; then + echo "opencv libraries not found in $OUTPUT_DIR/lib" + return 1 + fi + if [[ ! -f $OUTPUT_DIR/lib/libopencv_core.a ]]; then + echo "opencv core library not found in $OUTPUT_DIR/lib/libopencv_core.a" + return 1 + fi + if [[ ! -f $OUTPUT_DIR/lib/libopencv_imgproc.a ]]; then + echo "opencv imgproc library not found in $OUTPUT_DIR/lib/libopencv_imgproc.a" + return 1 + fi + if [[ ! -f $OUTPUT_DIR/include/opencv4/opencv2/opencv.hpp ]]; then + echo "opencv includes not found in $OUTPUT_DIR/include/opencv4/opencv2/opencv.hpp" + return 1 + fi + echo "opencv found in $OUTPUT_DIR" + return 0 +} + +# check if opencv is already built using verify_opencv_exists +if verify_opencv_exists; then + echo "opencv already built" + exit 0 +fi + +# try to download opencv from s3 if not already downloaded, use wget if available +echo "downloading opencv from s3" +if [[ ! $( which wget ) ]]; then + echo "wget is not available, please install it e.g. `$ brew install wget`" + exit 1 fi if [[ ! -d $WORK_DIR ]]; then + echo "creating work directory $WORK_DIR" mkdir -p $WORK_DIR fi +OPENCV_TAR_FILENAME="opencv-4.5.2-x86_64.tar.gz" +OPENCV_TAR_LOCATION="${WORK_DIR}/${OPENCV_TAR_FILENAME}" +wget -q https://obs-backgroundremoval-build.s3.amazonaws.com/opencv-4.5.2-x86_64.tar.gz -O ${OPENCV_TAR_LOCATION} + +if [[ ! -f $OPENCV_TAR_LOCATION ]]; then + echo "failed to download opencv from s3" + exit 1 +fi + +# extract opencv if downloaded +echo "extracting opencv" +cd "${SCRIPT_DIR}/../" +tar xzf ${OPENCV_TAR_LOCATION} +rm ${OPENCV_TAR_LOCATION} +cd "${CURRENT_DIR}" + +if verify_opencv_exists; then + echo "opencv successfully downloaded from s3" + exit 0 +fi + +# build opencv from source +echo "building opencv from source" + +if [[ -d $OUTPUT_DIR ]]; then + rm -rf $OUTPUT_DIR +fi + cd $WORK_DIR # download opencv if not already downloaded