Merge branch 'master' into sarlinpe/repr

colmap · Jan 10, 2024 · 10425d7 · 10425d7
2 parents 53cbbf6 + ab04fc8
commit 10425d7
Show file tree

Hide file tree

Showing 5 changed files with 145 additions and 85 deletions.
diff --git a/.github/workflows/build-new.yml b/.github/workflows/build-new.yml
@@ -22,10 +22,21 @@ jobs:
               {os: macos-13, arch: arm64},
               {os: windows-latest},
         ]
+    env:
+      COMPILER_CACHE_VERSION: 1
+      COMPILER_CACHE_DIR: ${{ github.workspace }}/compiler-cache
+      CCACHE_DIR: ${{ github.workspace }}/compiler-cache/ccache
+      CCACHE_BASEDIR: ${{ github.workspace }}
     steps:
       - uses: actions/checkout@v4
-      - name: Set macOS env
-        if: ${{ startsWith(matrix.config.os, 'macos') }}
+      - uses: actions/cache@v3
+        id: cache-builds
+        with:
+          key: v${{ env.COMPILER_CACHE_VERSION }}-${{ matrix.config.os }}-${{ matrix.config.arch }}-${{ github.run_id }}-${{ github.run_number }}
+          restore-keys: v${{ env.COMPILER_CACHE_VERSION }}-${{ matrix.config.os }}-${{ matrix.config.arch }}
+          path: ${{ env.COMPILER_CACHE_DIR }}
+      - name: Set env (macOS)
+        if: runner.os == 'macOS'
         run: |
           if [[ ${{ matrix.config.arch }} == "x86_64" ]]; then
             VCPKG_TARGET_TRIPLET="x64-osx"
@@ -48,19 +59,61 @@ jobs:
           CONFIG_SETTINGS="${CONFIG_SETTINGS} cmake.define.VCPKG_TARGET_TRIPLET=${VCPKG_TARGET_TRIPLET}"
           CONFIG_SETTINGS="${CONFIG_SETTINGS} cmake.define.CMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES}"
           echo "CIBW_CONFIG_SETTINGS_MACOS=${CONFIG_SETTINGS}" >> "$GITHUB_ENV"
-      - name: Set Windows env
-        if: ${{ startsWith(matrix.config.os, 'windows') }}
+
+          # vcpkg binary caching
+          VCPKG_CACHE_DIR="${COMPILER_CACHE_DIR}/vcpkg"
+          VCPKG_BINARY_SOURCES="clear;files,${VCPKG_CACHE_DIR},readwrite"
+          echo "VCPKG_BINARY_SOURCES=${VCPKG_BINARY_SOURCES}" >> "$GITHUB_ENV"
+      - name: Set env (Windows)
+        if: runner.os == 'Windows'
         shell: pwsh
         run: |
           $CMAKE_TOOLCHAIN_FILE = "${env:VCPKG_INSTALLATION_ROOT}/scripts/buildsystems/vcpkg.cmake"
           echo "CMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}" >> "${env:GITHUB_ENV}"
           $VCPKG_TARGET_TRIPLET = "x64-windows"
           echo "VCPKG_TARGET_TRIPLET=${VCPKG_TARGET_TRIPLET}" >> "${env:GITHUB_ENV}"
+
           # Fix: cibuildhweel cannot interpolate env variables.
           $CMAKE_TOOLCHAIN_FILE = $CMAKE_TOOLCHAIN_FILE.replace('\', '/')
           $CONFIG_SETTINGS = "cmake.define.CMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}"
           $CONFIG_SETTINGS = "${CONFIG_SETTINGS} cmake.define.VCPKG_TARGET_TRIPLET=${VCPKG_TARGET_TRIPLET}"
           echo "CIBW_CONFIG_SETTINGS_WINDOWS=${CONFIG_SETTINGS}" >> "${env:GITHUB_ENV}"
+
+          # vcpkg binary caching
+          $VCPKG_CACHE_DIR = "${env:COMPILER_CACHE_DIR}/vcpkg"
+          $VCPKG_BINARY_SOURCES = "clear;files,${VCPKG_CACHE_DIR},readwrite"
+          echo "VCPKG_BINARY_SOURCES=${VCPKG_BINARY_SOURCES}" >> "${env:GITHUB_ENV}"
+      - name: Set env (Ubuntu)
+        if: runner.os == 'Linux'
+        run: |
+          VCPKG_TARGET_TRIPLET="x64-linux-release"
+          echo "VCPKG_TARGET_TRIPLET=${VCPKG_TARGET_TRIPLET}" >> "$GITHUB_ENV"
+
+          VCPKG_INSTALLATION_ROOT="${{ github.workspace }}/vcpkg"
+          CMAKE_TOOLCHAIN_FILE="${VCPKG_INSTALLATION_ROOT}/scripts/buildsystems/vcpkg.cmake"
+          echo "VCPKG_INSTALLATION_ROOT=${VCPKG_INSTALLATION_ROOT}" >> "$GITHUB_ENV"
+          echo "CMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}" >> "$GITHUB_ENV"
+
+          # Fix: cibuildhweel cannot interpolate env variables.
+          CONFIG_SETTINGS="cmake.define.CMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}"
+          CONFIG_SETTINGS="${CONFIG_SETTINGS} cmake.define.VCPKG_TARGET_TRIPLET=${VCPKG_TARGET_TRIPLET}"
+          echo "CIBW_CONFIG_SETTINGS_LINUX=${CONFIG_SETTINGS}" >> "$GITHUB_ENV"
+
+          # Remap caching paths to the container
+          CONTAINER_COMPILER_CACHE_DIR="/compiler-cache"
+          CIBW_CONTAINER_ENGINE="docker; create_args: -v ${COMPILER_CACHE_DIR}:${CONTAINER_COMPILER_CACHE_DIR}"
+          echo "CIBW_CONTAINER_ENGINE=${CIBW_CONTAINER_ENGINE}" >> "$GITHUB_ENV"
+          echo "COMPILER_CACHE_DIR=${CONTAINER_COMPILER_CACHE_DIR}" >> "$GITHUB_ENV"
+          echo "CCACHE_DIR=${CONTAINER_COMPILER_CACHE_DIR}/ccache" >> "$GITHUB_ENV"
+          echo "CCACHE_BASEDIR=/project" >> "$GITHUB_ENV"
+
+          # vcpkg binary caching
+          VCPKG_CACHE_DIR="${CONTAINER_COMPILER_CACHE_DIR}/vcpkg"
+          VCPKG_BINARY_SOURCES="clear;files,${VCPKG_CACHE_DIR},readwrite"
+          echo "VCPKG_BINARY_SOURCES=${VCPKG_BINARY_SOURCES}" >> "$GITHUB_ENV"
+
+          CIBW_ENVIRONMENT_PASS_LINUX="VCPKG_TARGET_TRIPLET VCPKG_INSTALLATION_ROOT CMAKE_TOOLCHAIN_FILE VCPKG_BINARY_SOURCES COMPILER_CACHE_DIR CCACHE_DIR CCACHE_BASEDIR"
+          echo "CIBW_ENVIRONMENT_PASS_LINUX=${CIBW_ENVIRONMENT_PASS_LINUX}" >> "$GITHUB_ENV"
       - name: Build wheels
         uses: pypa/[email protected]
         env:

diff --git a/package/install-colmap-centos.sh b/package/install-colmap-centos.sh
@@ -1,86 +1,64 @@
 #!/bin/bash
 set -e -x
 uname -a
-
 CURRDIR=$(pwd)
-echo "${CURRDIR}"
 
-# ------ Install dependencies from the default repositories ------
-cd ${CURRDIR}
-yum install -y \
-    wget \
-    git \
-    gcc gcc-c++ make \
-    freeimage-devel \
-    metis-devel \
-    glew-devel \
-    suitesparse-devel \
-    atlas-devel \
-    lapack-devel \
-    blas-devel \
-    flann \
-    flann-devel \
-    lz4 \
-    lz4-devel
+yum install -y gcc gcc-c++ ninja-build curl zip unzip tar
 
-# ------ Install boost ------
-cd ${CURRDIR}
-export BOOST_FILENAME=boost_1_71_0
-wget -nv https://boostorg.jfrog.io/artifactory/main/release/1.71.0/source/${BOOST_FILENAME}.tar.gz
-tar xzf ${BOOST_FILENAME}.tar.gz
-cd ${BOOST_FILENAME}
-./bootstrap.sh --with-libraries=filesystem,system,program_options,graph,test --without-icu
-./b2 -j$(nproc) cxxflags="-fPIC" variant=release link=shared --disable-icu install
+# ccache shipped by CentOS is too old so we download and cache it.
+COMPILER_TOOLS_DIR="${COMPILER_CACHE_DIR}/bin"
+mkdir -p ${COMPILER_TOOLS_DIR}
+if [ ! -f "${COMPILER_TOOLS_DIR}/ccache" ]; then
+    FILE="ccache-4.9-linux-x86_64"
+    curl -sSLO https://github.com/ccache/ccache/releases/download/v4.9/${FILE}.tar.xz
+    tar -xf ${FILE}.tar.xz
+    cp ${FILE}/ccache ${COMPILER_TOOLS_DIR}
+fi
+export PATH="${COMPILER_TOOLS_DIR}:${PATH}"
+ccache --version
+ccache --help
 
-# ------ Install gflags ------
-cd ${CURRDIR}
-git clone --branch v2.2.2 --depth 1 https://github.com/gflags/gflags.git
-cd gflags
-mkdir build && cd build
-cmake .. -DBUILD_SHARED_LIBS=ON
-make -j$(nproc) install
-
-# ------ Install glog ------
-cd ${CURRDIR}
-git clone --branch v0.6.0 --depth 1 https://github.com/google/glog.git
-cd glog
-mkdir build && cd build
-cmake ..
-make -j$(nproc) install
-
-# ------ Install Eigen ------
-cd ${CURRDIR}
-EIGEN_VERSION="3.3.9"
-export EIGEN_DIR="${CURRDIR}/eigen"
-wget https://gitlab.com/libeigen/eigen/-/archive/${EIGEN_VERSION}/eigen-${EIGEN_VERSION}.tar.gz
-tar -xvzf eigen-${EIGEN_VERSION}.tar.gz
-mv eigen-${EIGEN_VERSION} ${EIGEN_DIR}
-cd ${EIGEN_DIR}
-mkdir build && cd build
-cmake ..
-
-# ------ Install CERES solver ------
-cd ${CURRDIR}
-git clone https://ceres-solver.googlesource.com/ceres-solver
-cd ceres-solver
-git checkout $(git describe --tags) # Checkout the latest release
-mkdir build && cd build
-cmake .. -DBUILD_TESTING=OFF \
-         -DBUILD_EXAMPLES=OFF \
-         -DEigen3_DIR="${EIGEN_DIR}/cmake/"
-make -j$(nproc) install
+git clone --branch sarlinpe/libraw-jaspter-nodefaults https://github.com/sarlinpe/vcpkg ${VCPKG_INSTALLATION_ROOT}
+cd ${VCPKG_INSTALLATION_ROOT}
+./bootstrap-vcpkg.sh
+./vcpkg install --recurse --clean-after-build --triplet=${VCPKG_TARGET_TRIPLET} \
+    boost-algorithm \
+    boost-filesystem \
+    boost-graph \
+    boost-heap \
+    boost-program-options \
+    boost-property-map \
+    boost-property-tree \
+    boost-regex \
+    boost-system \
+    ceres[lapack,suitesparse] \
+    eigen3 \
+    flann \
+    jasper[core] \
+    freeimage \
+    metis \
+    gflags \
+    glog \
+    gtest \
+    sqlite3
+# We force the core option of jasper to disable the unwanted opengl option.
+./vcpkg integrate install
 
-# ------ Build COLMAP ------
 cd ${CURRDIR}
 git clone https://github.com/colmap/colmap.git
 cd colmap
 git checkout ${COLMAP_COMMIT_ID}
 mkdir build && cd build
-CXXFLAGS="-fPIC" CFLAGS="-fPIC" cmake .. -DCMAKE_BUILD_TYPE=Release \
-         -DBoost_USE_STATIC_LIBS=OFF \
-         -DBOOST_ROOT=/usr/local \
-         -DCUDA_ENABLED=OFF \
-         -DCGAL_ENABLED=OFF \
-         -DGUI_ENABLED=OFF \
-         -DEIGEN3_INCLUDE_DIRS=$EIGEN_DIR
-make -j$(nproc) install
+CXXFLAGS="-fPIC" CFLAGS="-fPIC" cmake .. -GNinja \
+    -DCUDA_ENABLED=OFF \
+    -DCGAL_ENABLED=OFF \
+    -DGUI_ENABLED=OFF \
+    -DCMAKE_BUILD_TYPE=Release \
+    -DCMAKE_TOOLCHAIN_FILE="${CMAKE_TOOLCHAIN_FILE}" \
+    -DVCPKG_TARGET_TRIPLET=${VCPKG_TARGET_TRIPLET} \
+    -DCMAKE_EXE_LINKER_FLAGS_INIT="-ldl"
+ninja install
+
+ccache --show-stats --verbose
+ccache --evict-older-than 1d
+ccache --show-stats --verbose
diff --git a/package/install-colmap-macos.sh b/package/install-colmap-macos.sh
@@ -13,7 +13,7 @@ brew remove swiftlint
 brew remove node@18
 
 brew update
-brew install git cmake ninja llvm
+brew install git cmake ninja llvm ccache
 
 cd ${CURRDIR}
 #git clone https://github.com/microsoft/vcpkg ${VCPKG_INSTALLATION_ROOT}
@@ -51,8 +51,14 @@ export ARCHFLAGS="-arch ${CIBW_ARCHS_MACOS}"
 cmake .. -GNinja -DGUI_ENABLED=OFF \
     -DCUDA_ENABLED=OFF \
     -DCGAL_ENABLED=OFF \
+    -DCMAKE_BUILD_TYPE=Release \
+    -DCCACHE_ENABLED=ON \
     -DCMAKE_TOOLCHAIN_FILE="${CMAKE_TOOLCHAIN_FILE}" \
     -DVCPKG_TARGET_TRIPLET=${VCPKG_TARGET_TRIPLET} \
     -DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES} \
     `if [[ ${CIBW_ARCHS_MACOS} == "arm64" ]]; then echo "-DSIMD_ENABLED=OFF"; fi`
 ninja install
+
+ccache --show-stats --verbose
+ccache --evict-older-than 1d
+ccache --show-stats --verbose
diff --git a/package/install-colmap-windows.ps1 b/package/install-colmap-windows.ps1
@@ -1,10 +1,30 @@
 $CURRDIR = $PWD
-echo "CMAKE_TOOLCHAIN_FILE = ${env:CMAKE_TOOLCHAIN_FILE}"
-echo "CIBW_CONFIG_SETTINGS_WINDOWS = ${env:CIBW_CONFIG_SETTINGS_WINDOWS}"
 
-curl.exe -L -o "ninja.zip" "https://github.com/ninja-build/ninja/releases/download/v1.10.2/ninja-win.zip"
-Expand-Archive -LiteralPath "${CURRDIR}/ninja.zip" -DestinationPath ${CURRDIR}
-$NINJA_PATH = "${CURRDIR}/ninja.exe"
+$COMPILER_TOOLS_DIR = "${env:COMPILER_CACHE_DIR}/bin"
+New-Item -ItemType Directory -Force -Path ${COMPILER_TOOLS_DIR}
+$env:Path = "${COMPILER_TOOLS_DIR};" + $env:Path
+
+$NINJA_PATH = "${COMPILER_TOOLS_DIR}/ninja.exe"
+If (!(Test-Path -path ${NINJA_PATH} -PathType Leaf)) {
+    $zip_path = "${env:TEMP}/ninja.zip"
+    $url = "https://github.com/ninja-build/ninja/releases/download/v1.10.2/ninja-win.zip"
+    curl.exe -L -o ${zip_path} ${url}
+    Expand-Archive -LiteralPath ${zip_path} -DestinationPath ${COMPILER_TOOLS_DIR}
+    Remove-Item ${zip_path}
+}
+If (!(Test-Path -path "${COMPILER_TOOLS_DIR}/ccache.exe" -PathType Leaf)) {
+    # For some reason this CI runs an earlier PowerShell version that is
+    # not compatible with colmap/.azure-pipelines/install-ccache.ps1
+    $folder = "ccache-4.8-windows-x86_64"
+    $url = "https://github.com/ccache/ccache/releases/download/v4.8/${folder}.zip"
+    $zip_path = "${env:TEMP}/${folder}.zip"
+    $folder_path = "${env:TEMP}/${folder}"
+    curl.exe -L -o ${zip_path} ${url}
+    Expand-Archive -LiteralPath ${zip_path} -DestinationPath "$env:TEMP"
+    Move-Item -Force "${folder_path}/ccache.exe" ${COMPILER_TOOLS_DIR}
+    Remove-Item ${zip_path}
+    Remove-Item -Recurse ${folder_path}
+}
 
 cd ${CURRDIR}
 git clone https://github.com/colmap/colmap.git
@@ -32,3 +52,7 @@ cmake .. `
   -DCMAKE_TOOLCHAIN_FILE="${env:CMAKE_TOOLCHAIN_FILE}" `
   -DVCPKG_TARGET_TRIPLET="x64-windows"
 & ${NINJA_PATH} install
+
+ccache --show-stats --verbose
+ccache --evict-older-than 1d
+ccache --show-stats --verbose
diff --git a/pyproject.toml b/pyproject.toml
@@ -30,12 +30,11 @@ wheel.expand-macos-universal-tags = true
 [tool.cibuildwheel]
 build = "cp3{8,9,10,11}-{macosx,manylinux,win}*"
 archs = ["auto64"]
-environment = { COLMAP_COMMIT_ID="c206c7a333c8b2ef818135bdfe9d31a28bd2eb2b" }
+environment = { COLMAP_COMMIT_ID="3.9" }
 test-command = "python -c \"import pycolmap; print(pycolmap.__version__)\""
 
 [tool.cibuildwheel.linux]
 before-all = "{project}/package/install-colmap-centos.sh"
-config-settings = "cmake.define.EIGEN3_INCLUDE_DIRS=eigen"
 
 [tool.cibuildwheel.macos]
 before-all = "{project}/package/install-colmap-macos.sh"