From 6bfe48468d0e73005f4a4e2d97787761a8a1a68b Mon Sep 17 00:00:00 2001 From: Paul-Edouard Sarlin Date: Fri, 1 Sep 2023 00:08:29 +0200 Subject: [PATCH] Fix the CI (#155) - Deprecate Python 3.7 - Add MacOS 12 - Fix COLMAP commit - Replace with newer FindGlog - Clean linux boost install - Use pip wheel instead of setuptools --- .github/workflows/build-new.yml | 8 ++--- package/build-wheels-linux.sh | 49 ++++++++++++------------- package/build-wheels-macos.sh | 64 ++++++++++++++------------------- 3 files changed, 52 insertions(+), 69 deletions(-) diff --git a/.github/workflows/build-new.yml b/.github/workflows/build-new.yml index e7a327a9..9dd8e59d 100644 --- a/.github/workflows/build-new.yml +++ b/.github/workflows/build-new.yml @@ -16,7 +16,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - pyversion: [ "cp37-cp37m", "cp38-cp38", "cp39-cp39", "cp310-cp310" ] + pyversion: [ "cp38-cp38", "cp39-cp39", "cp310-cp310"] steps: - name: Checkout uses: actions/checkout@v3 @@ -37,15 +37,15 @@ jobs: with: # we strip the version number from the artifact name name: pycolmap-${{ matrix.pyversion }}-manylinux2014_x86_64 - path: wheelhouse/pycolmap-*-${{ matrix.pyversion }}-manylinux2014_x86_64.whl + path: wheelhouse/pycolmap-*-${{ matrix.pyversion }}-*.whl mac-build: name: Wrapper macOS Build runs-on: ${{ matrix.os }} strategy: matrix: - os: [ macos-11 ] - pyversion: [ "python@3.7", "python@3.8", "python@3.9", "python@3.10" ] + os: [ macos-11, macos-12 ] + pyversion: [ "python@3.8", "python@3.9", "python@3.10"] steps: - name: Checkout uses: actions/checkout@v3 diff --git a/package/build-wheels-linux.sh b/package/build-wheels-linux.sh index 33e7db98..3d45ef2e 100755 --- a/package/build-wheels-linux.sh +++ b/package/build-wheels-linux.sh @@ -38,34 +38,30 @@ CURRDIR=$(pwd) echo "Num. processes to use for building: ${nproc}" -# ------ Install boost (build it staticly) ------ +# ------ Install boost ------ cd $CURRDIR -yum install -y libicu libicu-devel centos-release-scl-rh devtoolset-7-gcc-c++ +yum install -y centos-release-scl-rh devtoolset-7-gcc-c++ -# Download and install Boost-1.65.1 -# colmap needs only program_options filesystem graph system unit_test_framework -mkdir -p boost && \ - cd boost && \ - wget -nv https://boostorg.jfrog.io/artifactory/main/release/1.65.1/source/boost_1_65_1.tar.gz && \ - tar xzf boost_1_65_1.tar.gz && \ - cd boost_1_65_1 && \ - ./bootstrap.sh --with-libraries=serialization,filesystem,thread,system,atomic,date_time,timer,chrono,program_options,regex,graph,test && \ - ./b2 -j$(nproc) cxxflags="-fPIC" runtime-link=static variant=release link=static install - -# Boost should now be visible under /usr/local -ls -ltrh /usr/local +mkdir -p boost +cd boost +export BOOST_FILENAME=boost_1_71_0 +wget -nv https://boostorg.jfrog.io/artifactory/main/release/1.71.0/source/${BOOST_FILENAME}.tar.gz +tar xzf ${BOOST_FILENAME}.tar.gz +cd ${BOOST_FILENAME} +./bootstrap.sh --with-libraries=filesystem,system,program_options,graph,test --without-icu +./b2 -j$(nproc) cxxflags="-fPIC" variant=release link=shared --disable-icu install # ------ Install dependencies from the default repositories ------ cd $CURRDIR yum install -y \ git \ - cmake \ gcc gcc-c++ make \ freeimage-devel \ metis-devel \ glog-devel \ gflags-devel \ glew-devel +cmake --version yum install -y suitesparse-devel atlas-devel lapack-devel blas-devel flann flann-devel lz4 lz4-devel @@ -87,8 +83,6 @@ mkdir build cd build cmake .. -ls -ltrh "$EIGEN_DIR/cmake/" - # ------ Install CERES solver ------ cd $CURRDIR git clone https://ceres-solver.googlesource.com/ceres-solver @@ -118,11 +112,13 @@ echo "PYTHON_LIBRARY:${PYTHON_LIBRARY}" cd $CURRDIR git clone https://github.com/colmap/colmap.git cd colmap -git checkout dev +git checkout 567d29ea7ddd96e1882e90d469e6b188ce16d297 +rm -f cmake/FindGlog.cmake +wget https://raw.githubusercontent.com/colmap/colmap/main/cmake/FindGlog.cmake -P cmake/ mkdir build/ cd build/ CXXFLAGS="-fPIC" CFLAGS="-fPIC" cmake .. -DCMAKE_BUILD_TYPE=Release \ - -DBoost_USE_STATIC_LIBS=ON \ + -DBoost_USE_STATIC_LIBS=OFF \ -DBOOST_ROOT=/usr/local \ -DGUI_ENABLED=OFF \ -DEIGEN3_INCLUDE_DIRS=$EIGEN_DIR @@ -137,14 +133,13 @@ make -j$(nproc) install # ------ Build pycolmap wheel ------ cd /io/ -cat setup.py - -PLAT=manylinux2014_x86_64 -EIGEN3_INCLUDE_DIRS="$EIGEN_DIR" "${PYBIN}/python" setup.py bdist_wheel --plat-name=$PLAT +WHEEL_DIR="wheels/" +EIGEN3_INCLUDE_DIRS="$EIGEN_DIR" "${PYBIN}/pip" wheel --no-deps -w ${WHEEL_DIR} . # Bundle external shared libraries into the wheels -mkdir -p /io/wheelhouse -for whl in ./dist/*.whl; do - auditwheel repair "$whl" -w /io/wheelhouse/ --no-update-tags +OUT_DIR="/io/wheelhouse" +mkdir -p ${OUT_DIR} +for whl in ${WHEEL_DIR}/*.whl; do + auditwheel repair "$whl" -w ${OUT_DIR} --plat manylinux2014_x86_64 done -ls -ltrh /io/wheelhouse/ +ls -ltrh ${OUT_DIR} diff --git a/package/build-wheels-macos.sh b/package/build-wheels-macos.sh index 033feff7..dee3d406 100755 --- a/package/build-wheels-macos.sh +++ b/package/build-wheels-macos.sh @@ -30,6 +30,9 @@ find /usr/local/bin -lname '*/Library/Frameworks/Python.framework/*' -delete rm /usr/local/bin/go || true rm /usr/local/bin/gofmt || true +# Updating requires Xcode 14.0, which cannot be installed on macOS 11. +brew remove swiftlint + brew update brew upgrade brew install wget cmake @@ -51,6 +54,7 @@ brew install \ sqlite3 \ libomp \ llvm \ + boost \ lz4 brew info gcc @@ -69,14 +73,15 @@ echo "Python bin path: $PYBIN" CURRDIR=$(pwd) ls -ltrh $CURRDIR -# Build Boost staticly -mkdir -p boost_build -cd boost_build -retry 3 wget https://boostorg.jfrog.io/artifactory/main/release/1.81.0/source/boost_1_81_0.tar.gz -tar xzf boost_1_81_0.tar.gz -cd boost_1_81_0 -./bootstrap.sh --prefix=$CURRDIR/boost_install --with-libraries=serialization,filesystem,thread,system,atomic,date_time,timer,chrono,program_options,regex clang-darwin -./b2 -j$(sysctl -n hw.logicalcpu) cxxflags="-fPIC" runtime-link=static variant=release link=static install +# Install Boost +#mkdir -p boost +#cd boost +#retry 3 wget https://boostorg.jfrog.io/artifactory/main/release/1.81.0/source/boost_1_81_0.tar.gz +#tar xzf boost_1_81_0.tar.gz +#cd boost_1_81_0 +#BOOST_DIR=$CURRDIR/boost_install +#./bootstrap.sh --prefix=${BOOST_DIR} --with-libraries=filesystem,system,program_options,graph,test --without-icu clang-darwin +#./b2 -j$(sysctl -n hw.logicalcpu) cxxflags="-fPIC" link=static runtime-link=static variant=release --disable-icu --prefix=${BOOST_DIR} install echo "CURRDIR is: ${CURRDIR}" @@ -89,53 +94,36 @@ touch ${PYTHON_LIBRARY} git clone https://github.com/colmap/colmap.git -for compiler in cc c++ gcc g++ clang clang++ -do - which $compiler - $compiler --version -done - -# Install `delocate` -- OSX equivalent of `auditwheel` -# see https://pypi.org/project/delocate/ for more details cd $CURRDIR -$INTERPRETER -m pip install -U delocate -$INTERPRETER -m pip install -U pip setuptools wheel cffi - -ls -ltrh /usr/local -ls -ltrh /usr/local/opt cd $CURRDIR cd colmap -git checkout dev +git checkout 567d29ea7ddd96e1882e90d469e6b188ce16d297 mkdir build cd build -cmake .. -DGUI_ENABLED=OFF - -# examine exit code of last command -ec=$? -if [ $ec -ne 0 ]; then - echo "Error:" - cat ./CMakeCache.txt - exit $ec -fi -set -e -x +cmake .. -DGUI_ENABLED=OFF #-DBoost_USE_STATIC_LIBS=ON -DBOOSTROOT=${BOOST_DIR} -DBoost_NO_SYSTEM_PATHS=ON NUM_LOGICAL_CPUS=$(sysctl -n hw.logicalcpu) echo "Number of logical CPUs is: ${NUM_LOGICAL_CPUS}" make -j $NUM_LOGICAL_CPUS install sudo make install +# Install `delocate` -- OSX equivalent of `auditwheel` +# see https://pypi.org/project/delocate/ for more details +$INTERPRETER -m pip install -U delocate +$INTERPRETER -m pip install -U pip setuptools wheel cffi + cd $CURRDIR -cat setup.py # flags must be passed, to avoid the issue: `Unsupported compiler -- pybind11 requires C++11 support!` # see https://github.com/quantumlib/qsim/issues/242 for more details -CC=/usr/local/opt/llvm/bin/clang CXX=/usr/local/opt/llvm/bin/clang++ LDFLAGS=-L/usr/local/opt/libomp/lib $INTERPRETER setup.py bdist_wheel -cp ./dist/*.whl $CURRDIR/wheelhouse_unrepaired +WHEEL_DIR="${CURRDIR}/wheelhouse_unrepaired/" +CC=/usr/local/opt/llvm/bin/clang CXX=/usr/local/opt/llvm/bin/clang++ LDFLAGS=-L/usr/local/opt/libomp/lib $INTERPRETER -m pip wheel --no-deps -w ${WHEEL_DIR} . # Bundle external shared libraries into the wheels -ls -ltrh $CURRDIR/wheelhouse_unrepaired/ -for whl in $CURRDIR/wheelhouse_unrepaired/*.whl; do +OUT_DIR="${CURRDIR}/wheelhouse" +for whl in ${WHEEL_DIR}/*.whl; do delocate-listdeps --all "$whl" - delocate-wheel -w "$CURRDIR/wheelhouse" -v "$whl" + delocate-wheel -w "${OUT_DIR}" -v "$whl" rm $whl done +ls -ltrh ${OUT_DIR}