From b38ef3e95c2f5e9a56f5ffd0e4d4c2327bdfaae7 Mon Sep 17 00:00:00 2001 From: Paul-Edouard Sarlin Date: Wed, 3 Jan 2024 11:32:40 +0100 Subject: [PATCH 01/24] Test ccache for macOS --- .github/workflows/build-new.yml | 26 ++++++++++++++++++++++---- package/install-colmap-macos.sh | 6 +++++- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-new.yml b/.github/workflows/build-new.yml index 2355d3f6..8d77cc0b 100644 --- a/.github/workflows/build-new.yml +++ b/.github/workflows/build-new.yml @@ -22,10 +22,21 @@ jobs: {os: macos-13, arch: arm64}, {os: windows-latest}, ] + env: + COMPILER_CACHE_VERSION: 1 + COMPILER_CACHE_DIR: ${{ github.workspace }}/compiler-cache + CCACHE_DIR: ${{ github.workspace }}/compiler-cache/ccache + CCACHE_BASEDIR: ${{ github.workspace }} steps: + - uses: actions/cache@v3 + id: cache-builds + with: + key: ccache-${{ env.COMPILER_CACHE_VERSION }}-${{ matrix.config.os }}-${{ matrix.config.arch }}-${{ github.run_id }}-${{ github.run_number }} + restore-keys: ccache-${{ env.COMPILER_CACHE_VERSION }}-${{ matrix.config.os }}-${{ matrix.config.arch }} + path: ${{ env.COMPILER_CACHE_DIR }} - uses: actions/checkout@v4 - - name: Set macOS env - if: ${{ startsWith(matrix.config.os, 'macos') }} + - name: Set env (macOS) + if: runner.os == 'macOS' run: | if [[ ${{ matrix.config.arch }} == "x86_64" ]]; then VCPKG_TARGET_TRIPLET="x64-osx" @@ -48,8 +59,8 @@ jobs: CONFIG_SETTINGS="${CONFIG_SETTINGS} cmake.define.VCPKG_TARGET_TRIPLET=${VCPKG_TARGET_TRIPLET}" CONFIG_SETTINGS="${CONFIG_SETTINGS} cmake.define.CMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES}" echo "CIBW_CONFIG_SETTINGS_MACOS=${CONFIG_SETTINGS}" >> "$GITHUB_ENV" - - name: Set Windows env - if: ${{ startsWith(matrix.os, 'windows') }} + - name: Set env (Windows) + if: runner.os == 'Windows' shell: pwsh run: | $CMAKE_TOOLCHAIN_FILE = "${env:VCPKG_INSTALLATION_ROOT}/scripts/buildsystems/vcpkg.cmake" @@ -70,6 +81,13 @@ jobs: with: name: pycolmap-${{ matrix.config.os }}-${{ matrix.config.arch }} path: wheelhouse/pycolmap-*.whl + - name: Cleanup compiler cache (macOS) + if: runner.os == 'macOS' + run: | + set -x + ccache --show-stats --verbose + ccache --evict-older-than 7d + ccache --show-stats --verbose pypi-publish: name: Publish wheels to PyPI diff --git a/package/install-colmap-macos.sh b/package/install-colmap-macos.sh index 75e75a81..4e8f5276 100755 --- a/package/install-colmap-macos.sh +++ b/package/install-colmap-macos.sh @@ -13,7 +13,10 @@ brew remove swiftlint brew remove node@18 brew update -brew install git cmake ninja llvm +brew install git cmake ninja llvm ccache +export CMAKE_CXX_COMPILER_LAUNCHER="ccache" +export CMAKE_C_COMPILER_LAUNCHER="ccache" +export VCPKG_KEEP_ENV_VARS="CMAKE_CXX_COMPILER_LAUNCHER;CMAKE_C_COMPILER_LAUNCHER" cd ${CURRDIR} #git clone https://github.com/microsoft/vcpkg ${VCPKG_INSTALLATION_ROOT} @@ -51,6 +54,7 @@ export ARCHFLAGS="-arch ${CIBW_ARCHS_MACOS}" cmake .. -GNinja -DGUI_ENABLED=OFF \ -DCUDA_ENABLED=OFF \ -DCGAL_ENABLED=OFF \ + -DCCACHE_ENABLED=ON \ -DCMAKE_TOOLCHAIN_FILE="${CMAKE_TOOLCHAIN_FILE}" \ -DVCPKG_TARGET_TRIPLET=${VCPKG_TARGET_TRIPLET} \ -DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES} \ From 952f379ba521c06e4769e9168e0c8f316b528758 Mon Sep 17 00:00:00 2001 From: Paul-Edouard Sarlin Date: Wed, 3 Jan 2024 12:13:08 +0100 Subject: [PATCH 02/24] Move cache location --- .github/workflows/build-new.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-new.yml b/.github/workflows/build-new.yml index 8d77cc0b..f7f1d2ea 100644 --- a/.github/workflows/build-new.yml +++ b/.github/workflows/build-new.yml @@ -24,8 +24,8 @@ jobs: ] env: COMPILER_CACHE_VERSION: 1 - COMPILER_CACHE_DIR: ${{ github.workspace }}/compiler-cache - CCACHE_DIR: ${{ github.workspace }}/compiler-cache/ccache + COMPILER_CACHE_DIR: ${{ github.workspace }}/../compiler-cache + CCACHE_DIR: ${{ github.workspace }}/../compiler-cache/ccache CCACHE_BASEDIR: ${{ github.workspace }} steps: - uses: actions/cache@v3 From a4a24770f9dedf5228e7371a127fef25deeee3bd Mon Sep 17 00:00:00 2001 From: Paul-Edouard Sarlin Date: Wed, 3 Jan 2024 13:41:20 +0100 Subject: [PATCH 03/24] Revert --- .github/workflows/build-new.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-new.yml b/.github/workflows/build-new.yml index f7f1d2ea..8d77cc0b 100644 --- a/.github/workflows/build-new.yml +++ b/.github/workflows/build-new.yml @@ -24,8 +24,8 @@ jobs: ] env: COMPILER_CACHE_VERSION: 1 - COMPILER_CACHE_DIR: ${{ github.workspace }}/../compiler-cache - CCACHE_DIR: ${{ github.workspace }}/../compiler-cache/ccache + COMPILER_CACHE_DIR: ${{ github.workspace }}/compiler-cache + CCACHE_DIR: ${{ github.workspace }}/compiler-cache/ccache CCACHE_BASEDIR: ${{ github.workspace }} steps: - uses: actions/cache@v3 From bfe7154f47744ff03733ed7d814f0e846e5a6b99 Mon Sep 17 00:00:00 2001 From: Paul-Edouard Sarlin Date: Wed, 3 Jan 2024 14:49:50 +0100 Subject: [PATCH 04/24] vcpkg binary caching --- .github/workflows/build-new.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-new.yml b/.github/workflows/build-new.yml index 8d77cc0b..01ed8a55 100644 --- a/.github/workflows/build-new.yml +++ b/.github/workflows/build-new.yml @@ -17,10 +17,10 @@ jobs: strategy: matrix: config: [ - {os: ubuntu-latest}, + #{os: ubuntu-latest}, {os: macos-13, arch: x86_64}, {os: macos-13, arch: arm64}, - {os: windows-latest}, + #{os: windows-latest}, ] env: COMPILER_CACHE_VERSION: 1 @@ -59,6 +59,10 @@ jobs: CONFIG_SETTINGS="${CONFIG_SETTINGS} cmake.define.VCPKG_TARGET_TRIPLET=${VCPKG_TARGET_TRIPLET}" CONFIG_SETTINGS="${CONFIG_SETTINGS} cmake.define.CMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES}" echo "CIBW_CONFIG_SETTINGS_MACOS=${CONFIG_SETTINGS}" >> "$GITHUB_ENV" + + # vcpkg binary caching + VCPKG_BINARY_SOURCES="clear;files,${COMPILER_CACHE_DIR}/vcpkg,readwrite" + echo "VCPKG_BINARY_SOURCES=${VCPKG_BINARY_SOURCES}" >> "$GITHUB_ENV" - name: Set env (Windows) if: runner.os == 'Windows' shell: pwsh From 29615a158940bd0c51d0af3cd0c794a9b8a8a244 Mon Sep 17 00:00:00 2001 From: Paul-Edouard Sarlin Date: Wed, 3 Jan 2024 15:34:01 +0100 Subject: [PATCH 05/24] Dummy From 3963aefb4a71fd37efdba732ee81c87c6fae6f6e Mon Sep 17 00:00:00 2001 From: Paul-Edouard Sarlin Date: Wed, 3 Jan 2024 15:41:26 +0100 Subject: [PATCH 06/24] Check why cache is not used --- package/install-colmap-macos.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/package/install-colmap-macos.sh b/package/install-colmap-macos.sh index 4e8f5276..f96b54dd 100755 --- a/package/install-colmap-macos.sh +++ b/package/install-colmap-macos.sh @@ -25,6 +25,7 @@ git clone --branch sarlinpe/lapack-osx https://github.com/sarlinpe/vcpkg ${VCPKG cd ${VCPKG_INSTALLATION_ROOT} ./bootstrap-vcpkg.sh ./vcpkg install --recurse --clean-after-build --triplet=${VCPKG_TARGET_TRIPLET} \ + --debug \ boost-algorithm \ boost-filesystem \ boost-graph \ From a9fa283d186fdfa72150ba566dc16df5e0b61615 Mon Sep 17 00:00:00 2001 From: Paul-Edouard Sarlin Date: Wed, 3 Jan 2024 16:11:25 +0100 Subject: [PATCH 07/24] Fail if cannot laod cache --- .github/workflows/build-new.yml | 2 +- package/install-colmap-macos.sh | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-new.yml b/.github/workflows/build-new.yml index 01ed8a55..c6f76882 100644 --- a/.github/workflows/build-new.yml +++ b/.github/workflows/build-new.yml @@ -19,7 +19,7 @@ jobs: config: [ #{os: ubuntu-latest}, {os: macos-13, arch: x86_64}, - {os: macos-13, arch: arm64}, + #{os: macos-13, arch: arm64}, #{os: windows-latest}, ] env: diff --git a/package/install-colmap-macos.sh b/package/install-colmap-macos.sh index f96b54dd..9ee0c207 100755 --- a/package/install-colmap-macos.sh +++ b/package/install-colmap-macos.sh @@ -22,10 +22,15 @@ cd ${CURRDIR} #git clone https://github.com/microsoft/vcpkg ${VCPKG_INSTALLATION_ROOT} git clone --branch sarlinpe/lapack-osx https://github.com/sarlinpe/vcpkg ${VCPKG_INSTALLATION_ROOT} +ls -l "${COMPILER_CACHE_DIR}/vcpkg/*" +du -hs "${COMPILER_CACHE_DIR}/vcpkg/*" + cd ${VCPKG_INSTALLATION_ROOT} ./bootstrap-vcpkg.sh ./vcpkg install --recurse --clean-after-build --triplet=${VCPKG_TARGET_TRIPLET} \ --debug \ + --x-abi-tools-use-exact-versions \ + --only-binarycaching \ boost-algorithm \ boost-filesystem \ boost-graph \ From ec013fe0e72d2e3a2fad1b7ea3b92b2a9eb48007 Mon Sep 17 00:00:00 2001 From: Paul-Edouard Sarlin Date: Wed, 3 Jan 2024 16:16:35 +0100 Subject: [PATCH 08/24] Fix --- .github/workflows/build-new.yml | 4 +++- package/install-colmap-macos.sh | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-new.yml b/.github/workflows/build-new.yml index c6f76882..2fd054be 100644 --- a/.github/workflows/build-new.yml +++ b/.github/workflows/build-new.yml @@ -61,7 +61,9 @@ jobs: echo "CIBW_CONFIG_SETTINGS_MACOS=${CONFIG_SETTINGS}" >> "$GITHUB_ENV" # vcpkg binary caching - VCPKG_BINARY_SOURCES="clear;files,${COMPILER_CACHE_DIR}/vcpkg,readwrite" + VCPKG_CACHE_DIR="${COMPILER_CACHE_DIR}/vcpkg" + mkdir -p ${VCPKG_CACHE_DIR} + VCPKG_BINARY_SOURCES="clear;files,${VCPKG_CACHE_DIR},readwrite" echo "VCPKG_BINARY_SOURCES=${VCPKG_BINARY_SOURCES}" >> "$GITHUB_ENV" - name: Set env (Windows) if: runner.os == 'Windows' diff --git a/package/install-colmap-macos.sh b/package/install-colmap-macos.sh index 9ee0c207..03de5314 100755 --- a/package/install-colmap-macos.sh +++ b/package/install-colmap-macos.sh @@ -22,8 +22,8 @@ cd ${CURRDIR} #git clone https://github.com/microsoft/vcpkg ${VCPKG_INSTALLATION_ROOT} git clone --branch sarlinpe/lapack-osx https://github.com/sarlinpe/vcpkg ${VCPKG_INSTALLATION_ROOT} -ls -l "${COMPILER_CACHE_DIR}/vcpkg/*" du -hs "${COMPILER_CACHE_DIR}/vcpkg/*" +ls -l "${COMPILER_CACHE_DIR}/vcpkg/*" cd ${VCPKG_INSTALLATION_ROOT} ./bootstrap-vcpkg.sh From 322ad6212f63a4bffb793a0c8c6868298dc029ec Mon Sep 17 00:00:00 2001 From: Paul-Edouard Sarlin Date: Wed, 3 Jan 2024 16:24:09 +0100 Subject: [PATCH 09/24] Restore cache after checkout --- .github/workflows/build-new.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-new.yml b/.github/workflows/build-new.yml index 2fd054be..12bdf73f 100644 --- a/.github/workflows/build-new.yml +++ b/.github/workflows/build-new.yml @@ -28,13 +28,13 @@ jobs: CCACHE_DIR: ${{ github.workspace }}/compiler-cache/ccache CCACHE_BASEDIR: ${{ github.workspace }} steps: + - uses: actions/checkout@v4 - uses: actions/cache@v3 id: cache-builds with: key: ccache-${{ env.COMPILER_CACHE_VERSION }}-${{ matrix.config.os }}-${{ matrix.config.arch }}-${{ github.run_id }}-${{ github.run_number }} restore-keys: ccache-${{ env.COMPILER_CACHE_VERSION }}-${{ matrix.config.os }}-${{ matrix.config.arch }} path: ${{ env.COMPILER_CACHE_DIR }} - - uses: actions/checkout@v4 - name: Set env (macOS) if: runner.os == 'macOS' run: | From 0948d3ca82508984cbcfc0e781cde6fed5e355c2 Mon Sep 17 00:00:00 2001 From: Paul-Edouard Sarlin Date: Wed, 3 Jan 2024 16:31:49 +0100 Subject: [PATCH 10/24] More debug logging --- package/install-colmap-macos.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/package/install-colmap-macos.sh b/package/install-colmap-macos.sh index 03de5314..89c0285a 100755 --- a/package/install-colmap-macos.sh +++ b/package/install-colmap-macos.sh @@ -18,13 +18,15 @@ export CMAKE_CXX_COMPILER_LAUNCHER="ccache" export CMAKE_C_COMPILER_LAUNCHER="ccache" export VCPKG_KEEP_ENV_VARS="CMAKE_CXX_COMPILER_LAUNCHER;CMAKE_C_COMPILER_LAUNCHER" +du -hs "${CURRDIR}/*" +du -hs "${COMPILER_CACHE_DIR}/*" +du -hs "${COMPILER_CACHE_DIR}/vcpkg/*" +ls -l "${COMPILER_CACHE_DIR}/vcpkg/*" + cd ${CURRDIR} #git clone https://github.com/microsoft/vcpkg ${VCPKG_INSTALLATION_ROOT} git clone --branch sarlinpe/lapack-osx https://github.com/sarlinpe/vcpkg ${VCPKG_INSTALLATION_ROOT} -du -hs "${COMPILER_CACHE_DIR}/vcpkg/*" -ls -l "${COMPILER_CACHE_DIR}/vcpkg/*" - cd ${VCPKG_INSTALLATION_ROOT} ./bootstrap-vcpkg.sh ./vcpkg install --recurse --clean-after-build --triplet=${VCPKG_TARGET_TRIPLET} \ From 7b2b51c76c2960c6ca7913116abd518245feb78f Mon Sep 17 00:00:00 2001 From: Paul-Edouard Sarlin Date: Wed, 3 Jan 2024 16:40:01 +0100 Subject: [PATCH 11/24] More debug logging --- package/install-colmap-macos.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/package/install-colmap-macos.sh b/package/install-colmap-macos.sh index 89c0285a..7e91f767 100755 --- a/package/install-colmap-macos.sh +++ b/package/install-colmap-macos.sh @@ -18,10 +18,12 @@ export CMAKE_CXX_COMPILER_LAUNCHER="ccache" export CMAKE_C_COMPILER_LAUNCHER="ccache" export VCPKG_KEEP_ENV_VARS="CMAKE_CXX_COMPILER_LAUNCHER;CMAKE_C_COMPILER_LAUNCHER" -du -hs "${CURRDIR}/*" -du -hs "${COMPILER_CACHE_DIR}/*" -du -hs "${COMPILER_CACHE_DIR}/vcpkg/*" -ls -l "${COMPILER_CACHE_DIR}/vcpkg/*" +ls -l ${CURRDIR} +ls -l ${COMPILER_CACHE_DIR} +du -hs ${CURRDIR}/* +du -hs ${COMPILER_CACHE_DIR}/* +du -hs ${COMPILER_CACHE_DIR}/vcpkg/* +ls -l ${COMPILER_CACHE_DIR}/vcpkg/* cd ${CURRDIR} #git clone https://github.com/microsoft/vcpkg ${VCPKG_INSTALLATION_ROOT} From 65bcb0293ff2b163608d870b3992359c2a47f3c8 Mon Sep 17 00:00:00 2001 From: Paul-Edouard Sarlin Date: Wed, 3 Jan 2024 17:00:55 +0100 Subject: [PATCH 12/24] Rebuild cache from scratch --- .github/workflows/build-new.yml | 5 ++--- package/install-colmap-macos.sh | 15 +++------------ 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build-new.yml b/.github/workflows/build-new.yml index 12bdf73f..5e81d345 100644 --- a/.github/workflows/build-new.yml +++ b/.github/workflows/build-new.yml @@ -32,8 +32,8 @@ jobs: - uses: actions/cache@v3 id: cache-builds with: - key: ccache-${{ env.COMPILER_CACHE_VERSION }}-${{ matrix.config.os }}-${{ matrix.config.arch }}-${{ github.run_id }}-${{ github.run_number }} - restore-keys: ccache-${{ env.COMPILER_CACHE_VERSION }}-${{ matrix.config.os }}-${{ matrix.config.arch }} + key: v${{ env.COMPILER_CACHE_VERSION }}-${{ matrix.config.os }}-${{ matrix.config.arch }}-${{ github.run_id }}-${{ github.run_number }} + restore-keys: v${{ env.COMPILER_CACHE_VERSION }}-${{ matrix.config.os }}-${{ matrix.config.arch }} path: ${{ env.COMPILER_CACHE_DIR }} - name: Set env (macOS) if: runner.os == 'macOS' @@ -62,7 +62,6 @@ jobs: # vcpkg binary caching VCPKG_CACHE_DIR="${COMPILER_CACHE_DIR}/vcpkg" - mkdir -p ${VCPKG_CACHE_DIR} VCPKG_BINARY_SOURCES="clear;files,${VCPKG_CACHE_DIR},readwrite" echo "VCPKG_BINARY_SOURCES=${VCPKG_BINARY_SOURCES}" >> "$GITHUB_ENV" - name: Set env (Windows) diff --git a/package/install-colmap-macos.sh b/package/install-colmap-macos.sh index 7e91f767..82c4fbe0 100755 --- a/package/install-colmap-macos.sh +++ b/package/install-colmap-macos.sh @@ -14,16 +14,10 @@ brew remove node@18 brew update brew install git cmake ninja llvm ccache -export CMAKE_CXX_COMPILER_LAUNCHER="ccache" -export CMAKE_C_COMPILER_LAUNCHER="ccache" -export VCPKG_KEEP_ENV_VARS="CMAKE_CXX_COMPILER_LAUNCHER;CMAKE_C_COMPILER_LAUNCHER" -ls -l ${CURRDIR} -ls -l ${COMPILER_CACHE_DIR} -du -hs ${CURRDIR}/* -du -hs ${COMPILER_CACHE_DIR}/* -du -hs ${COMPILER_CACHE_DIR}/vcpkg/* -ls -l ${COMPILER_CACHE_DIR}/vcpkg/* +#ls -l ${COMPILER_CACHE_DIR} +#ls -l ${COMPILER_CACHE_DIR}/vcpkg/* +echo ${VCPKG_BINARY_SOURCES} cd ${CURRDIR} #git clone https://github.com/microsoft/vcpkg ${VCPKG_INSTALLATION_ROOT} @@ -32,9 +26,6 @@ git clone --branch sarlinpe/lapack-osx https://github.com/sarlinpe/vcpkg ${VCPKG cd ${VCPKG_INSTALLATION_ROOT} ./bootstrap-vcpkg.sh ./vcpkg install --recurse --clean-after-build --triplet=${VCPKG_TARGET_TRIPLET} \ - --debug \ - --x-abi-tools-use-exact-versions \ - --only-binarycaching \ boost-algorithm \ boost-filesystem \ boost-graph \ From df4c09d809c3d183a9d783cc2ae103010969d536 Mon Sep 17 00:00:00 2001 From: Paul-Edouard Sarlin Date: Wed, 3 Jan 2024 17:55:49 +0100 Subject: [PATCH 13/24] Test caching --- package/install-colmap-macos.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/package/install-colmap-macos.sh b/package/install-colmap-macos.sh index 82c4fbe0..c07eb430 100755 --- a/package/install-colmap-macos.sh +++ b/package/install-colmap-macos.sh @@ -15,8 +15,8 @@ brew remove node@18 brew update brew install git cmake ninja llvm ccache -#ls -l ${COMPILER_CACHE_DIR} -#ls -l ${COMPILER_CACHE_DIR}/vcpkg/* +ls -l ${COMPILER_CACHE_DIR} +ls -l ${COMPILER_CACHE_DIR}/vcpkg/* echo ${VCPKG_BINARY_SOURCES} cd ${CURRDIR} @@ -26,6 +26,8 @@ git clone --branch sarlinpe/lapack-osx https://github.com/sarlinpe/vcpkg ${VCPKG cd ${VCPKG_INSTALLATION_ROOT} ./bootstrap-vcpkg.sh ./vcpkg install --recurse --clean-after-build --triplet=${VCPKG_TARGET_TRIPLET} \ + --debug \ + --only-binarycaching \ boost-algorithm \ boost-filesystem \ boost-graph \ From 831260bae6661420e538ba8d6610eb3bd6f922ae Mon Sep 17 00:00:00 2001 From: Paul-Edouard Sarlin Date: Wed, 3 Jan 2024 18:14:30 +0100 Subject: [PATCH 14/24] Remove debug logs --- .github/workflows/build-new.yml | 2 +- package/install-colmap-macos.sh | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/build-new.yml b/.github/workflows/build-new.yml index 5e81d345..3d4e2611 100644 --- a/.github/workflows/build-new.yml +++ b/.github/workflows/build-new.yml @@ -19,7 +19,7 @@ jobs: config: [ #{os: ubuntu-latest}, {os: macos-13, arch: x86_64}, - #{os: macos-13, arch: arm64}, + {os: macos-13, arch: arm64}, #{os: windows-latest}, ] env: diff --git a/package/install-colmap-macos.sh b/package/install-colmap-macos.sh index c07eb430..aed35ab8 100755 --- a/package/install-colmap-macos.sh +++ b/package/install-colmap-macos.sh @@ -15,10 +15,6 @@ brew remove node@18 brew update brew install git cmake ninja llvm ccache -ls -l ${COMPILER_CACHE_DIR} -ls -l ${COMPILER_CACHE_DIR}/vcpkg/* -echo ${VCPKG_BINARY_SOURCES} - cd ${CURRDIR} #git clone https://github.com/microsoft/vcpkg ${VCPKG_INSTALLATION_ROOT} git clone --branch sarlinpe/lapack-osx https://github.com/sarlinpe/vcpkg ${VCPKG_INSTALLATION_ROOT} @@ -26,8 +22,6 @@ git clone --branch sarlinpe/lapack-osx https://github.com/sarlinpe/vcpkg ${VCPKG cd ${VCPKG_INSTALLATION_ROOT} ./bootstrap-vcpkg.sh ./vcpkg install --recurse --clean-after-build --triplet=${VCPKG_TARGET_TRIPLET} \ - --debug \ - --only-binarycaching \ boost-algorithm \ boost-filesystem \ boost-graph \ From ecbf40a7af10608eebe4e9878b267c8d8e72c17e Mon Sep 17 00:00:00 2001 From: Paul-Edouard Sarlin Date: Wed, 3 Jan 2024 18:34:52 +0100 Subject: [PATCH 15/24] Enable caching on Windows --- .github/workflows/build-new.yml | 8 +++++++- package/install-colmap-windows.ps1 | 21 +++++++++++++++------ 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-new.yml b/.github/workflows/build-new.yml index 3d4e2611..e5169599 100644 --- a/.github/workflows/build-new.yml +++ b/.github/workflows/build-new.yml @@ -20,7 +20,7 @@ jobs: #{os: ubuntu-latest}, {os: macos-13, arch: x86_64}, {os: macos-13, arch: arm64}, - #{os: windows-latest}, + {os: windows-latest}, ] env: COMPILER_CACHE_VERSION: 1 @@ -72,11 +72,17 @@ jobs: echo "CMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}" >> "${env:GITHUB_ENV}" $VCPKG_TARGET_TRIPLET = "x64-windows" echo "VCPKG_TARGET_TRIPLET=${VCPKG_TARGET_TRIPLET}" >> "${env:GITHUB_ENV}" + # Fix: cibuildhweel cannot interpolate env variables. $CMAKE_TOOLCHAIN_FILE = $CMAKE_TOOLCHAIN_FILE.replace('\', '/') $CONFIG_SETTINGS = "cmake.define.CMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}" $CONFIG_SETTINGS = "${CONFIG_SETTINGS} cmake.define.VCPKG_TARGET_TRIPLET=${VCPKG_TARGET_TRIPLET}" echo "CIBW_CONFIG_SETTINGS_WINDOWS=${CONFIG_SETTINGS}" >> "${env:GITHUB_ENV}" + + # vcpkg binary caching + $VCPKG_CACHE_DIR = "${env:COMPILER_CACHE_DIR}/vcpkg" + $VCPKG_BINARY_SOURCES = "clear;files,${VCPKG_CACHE_DIR},readwrite" + echo "VCPKG_BINARY_SOURCES=${VCPKG_BINARY_SOURCES}" >> "${env:GITHUB_ENV}" - name: Build wheels uses: pypa/cibuildwheel@v2.16.2 env: diff --git a/package/install-colmap-windows.ps1 b/package/install-colmap-windows.ps1 index 4c887b44..ad5b4069 100755 --- a/package/install-colmap-windows.ps1 +++ b/package/install-colmap-windows.ps1 @@ -1,16 +1,25 @@ $CURRDIR = $PWD -echo "CMAKE_TOOLCHAIN_FILE = ${env:CMAKE_TOOLCHAIN_FILE}" -echo "CIBW_CONFIG_SETTINGS_WINDOWS = ${env:CIBW_CONFIG_SETTINGS_WINDOWS}" - -curl.exe -L -o "ninja.zip" "https://github.com/ninja-build/ninja/releases/download/v1.10.2/ninja-win.zip" -Expand-Archive -LiteralPath "${CURRDIR}/ninja.zip" -DestinationPath ${CURRDIR} -$NINJA_PATH = "${CURRDIR}/ninja.exe" cd ${CURRDIR} git clone https://github.com/colmap/colmap.git cd colmap git checkout "${env:COLMAP_COMMIT_ID}" +$COMPILER_TOOLS_DIR = "${env:COMPILER_CACHE_DIR}/bin" +New-Item -ItemType Directory -Force -Path ${COMPILER_TOOLS_DIR} +$env:Path = "${COMPILER_TOOLS_DIR};" + $env:Path + +$NINJA_PATH = "${COMPILER_TOOLS_DIR}/ninja.exe" +If (!(Test-Path -path ${NINJA_PATH} -PathType Leaf)) { + $NINJA_ZIP = "${env:TEMP}/ninja.zip" + curl.exe -L -o ${NINJA_ZIP} "https://github.com/ninja-build/ninja/releases/download/v1.10.2/ninja-win.zip" + Expand-Archive -LiteralPath ${NINJA_ZIP} -DestinationPath ${COMPILER_TOOLS_DIR} + Remove-Item ${NINJA_ZIP} +} +If (!(Test-Path -path "${COMPILER_TOOLS_DIR}/ccache.exe" -PathType Leaf)) { + .azure-pipelines/install-ccache.ps1 -Destination ${COMPILER_TOOLS_DIR} +} + & "./scripts/shell/enter_vs_dev_shell.ps1" [System.Collections.ArrayList]$DEPS = Get-Content -Path ".azure-pipelines/build-windows-vcpkg.txt" From bada722ef2e84d8b3c5c6b6937e7b171002522dd Mon Sep 17 00:00:00 2001 From: Paul-Edouard Sarlin Date: Wed, 3 Jan 2024 21:13:11 +0100 Subject: [PATCH 16/24] Dummy From 42681df5e0c34ae3c24978ee83dd2192586bf42e Mon Sep 17 00:00:00 2001 From: Paul-Edouard Sarlin Date: Wed, 3 Jan 2024 21:29:02 +0100 Subject: [PATCH 17/24] Debug why ccache doesn't work on Win --- package/install-colmap-windows.ps1 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/package/install-colmap-windows.ps1 b/package/install-colmap-windows.ps1 index ad5b4069..a9995c62 100755 --- a/package/install-colmap-windows.ps1 +++ b/package/install-colmap-windows.ps1 @@ -7,7 +7,9 @@ git checkout "${env:COLMAP_COMMIT_ID}" $COMPILER_TOOLS_DIR = "${env:COMPILER_CACHE_DIR}/bin" New-Item -ItemType Directory -Force -Path ${COMPILER_TOOLS_DIR} +echo $env:Path $env:Path = "${COMPILER_TOOLS_DIR};" + $env:Path +echo $env:Path $NINJA_PATH = "${COMPILER_TOOLS_DIR}/ninja.exe" If (!(Test-Path -path ${NINJA_PATH} -PathType Leaf)) { @@ -19,6 +21,7 @@ If (!(Test-Path -path ${NINJA_PATH} -PathType Leaf)) { If (!(Test-Path -path "${COMPILER_TOOLS_DIR}/ccache.exe" -PathType Leaf)) { .azure-pipelines/install-ccache.ps1 -Destination ${COMPILER_TOOLS_DIR} } +Dir -Recurse ${COMPILER_TOOLS_DIR} | Select Fullname & "./scripts/shell/enter_vs_dev_shell.ps1" From 0f72c4405e39b0d68e41c545ffd9bcc1ec0251d9 Mon Sep 17 00:00:00 2001 From: Paul-Edouard Sarlin Date: Wed, 3 Jan 2024 21:47:37 +0100 Subject: [PATCH 18/24] Fix ccache download --- package/install-colmap-windows.ps1 | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/package/install-colmap-windows.ps1 b/package/install-colmap-windows.ps1 index a9995c62..bcf62332 100755 --- a/package/install-colmap-windows.ps1 +++ b/package/install-colmap-windows.ps1 @@ -7,9 +7,7 @@ git checkout "${env:COLMAP_COMMIT_ID}" $COMPILER_TOOLS_DIR = "${env:COMPILER_CACHE_DIR}/bin" New-Item -ItemType Directory -Force -Path ${COMPILER_TOOLS_DIR} -echo $env:Path $env:Path = "${COMPILER_TOOLS_DIR};" + $env:Path -echo $env:Path $NINJA_PATH = "${COMPILER_TOOLS_DIR}/ninja.exe" If (!(Test-Path -path ${NINJA_PATH} -PathType Leaf)) { @@ -19,7 +17,16 @@ If (!(Test-Path -path ${NINJA_PATH} -PathType Leaf)) { Remove-Item ${NINJA_ZIP} } If (!(Test-Path -path "${COMPILER_TOOLS_DIR}/ccache.exe" -PathType Leaf)) { - .azure-pipelines/install-ccache.ps1 -Destination ${COMPILER_TOOLS_DIR} + $CCACHE_ZIP = "${env:TEMP}/ccache.zip" + $url = "https://github.com/ccache/ccache/releases/download/v4.8/ccache-4.8-windows-x86_64.zip" + $expectedSha256 = "A2B3BAB4BB8318FFC5B3E4074DC25636258BC7E4B51261F7D9BEF8127FDA8309" + curl.exe -L -o ${CCACHE_ZIP} ${url} + $hash = Get-FileHash ${CCACHE_ZIP} -Algorithm "sha256" + if ($hash.Hash -ne ${expectedSha256}) { + throw "File ${CCACHE_ZIP} hash $hash.Hash did not match expected hash ${expectedSha256}" + } + Expand-Archive -LiteralPath ${CCACHE_ZIP} -DestinationPath ${COMPILER_TOOLS_DIR} + Remove-Item ${CCACHE_ZIP} } Dir -Recurse ${COMPILER_TOOLS_DIR} | Select Fullname From 9ab82a29713725cea589e0f79c3ed059b71c515d Mon Sep 17 00:00:00 2001 From: Paul-Edouard Sarlin Date: Wed, 3 Jan 2024 21:56:35 +0100 Subject: [PATCH 19/24] Fix ccache download --- package/install-colmap-windows.ps1 | 34 ++++++++++++++---------------- 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/package/install-colmap-windows.ps1 b/package/install-colmap-windows.ps1 index bcf62332..27a57fbc 100755 --- a/package/install-colmap-windows.ps1 +++ b/package/install-colmap-windows.ps1 @@ -1,35 +1,33 @@ $CURRDIR = $PWD -cd ${CURRDIR} -git clone https://github.com/colmap/colmap.git -cd colmap -git checkout "${env:COLMAP_COMMIT_ID}" - $COMPILER_TOOLS_DIR = "${env:COMPILER_CACHE_DIR}/bin" New-Item -ItemType Directory -Force -Path ${COMPILER_TOOLS_DIR} $env:Path = "${COMPILER_TOOLS_DIR};" + $env:Path $NINJA_PATH = "${COMPILER_TOOLS_DIR}/ninja.exe" If (!(Test-Path -path ${NINJA_PATH} -PathType Leaf)) { - $NINJA_ZIP = "${env:TEMP}/ninja.zip" - curl.exe -L -o ${NINJA_ZIP} "https://github.com/ninja-build/ninja/releases/download/v1.10.2/ninja-win.zip" - Expand-Archive -LiteralPath ${NINJA_ZIP} -DestinationPath ${COMPILER_TOOLS_DIR} - Remove-Item ${NINJA_ZIP} + $zip_path = "${env:TEMP}/ninja.zip" + $url = "https://github.com/ninja-build/ninja/releases/download/v1.10.2/ninja-win.zip" + curl.exe -L -o ${zip_path} ${url} + Expand-Archive -LiteralPath ${zip_path} -DestinationPath ${COMPILER_TOOLS_DIR} + Remove-Item ${zip_path} } If (!(Test-Path -path "${COMPILER_TOOLS_DIR}/ccache.exe" -PathType Leaf)) { - $CCACHE_ZIP = "${env:TEMP}/ccache.zip" + # For some reason this CI runs an earlier PowerShell version that is + # not compatible with colmap/.azure-pipelines/install-ccache.ps1 + $zip_path = "${env:TEMP}/ccache.zip" $url = "https://github.com/ccache/ccache/releases/download/v4.8/ccache-4.8-windows-x86_64.zip" - $expectedSha256 = "A2B3BAB4BB8318FFC5B3E4074DC25636258BC7E4B51261F7D9BEF8127FDA8309" - curl.exe -L -o ${CCACHE_ZIP} ${url} - $hash = Get-FileHash ${CCACHE_ZIP} -Algorithm "sha256" - if ($hash.Hash -ne ${expectedSha256}) { - throw "File ${CCACHE_ZIP} hash $hash.Hash did not match expected hash ${expectedSha256}" - } - Expand-Archive -LiteralPath ${CCACHE_ZIP} -DestinationPath ${COMPILER_TOOLS_DIR} - Remove-Item ${CCACHE_ZIP} + curl.exe -L -o ${zip_path} ${url} + Expand-Archive -LiteralPath ${zip_path} -DestinationPath ${COMPILER_TOOLS_DIR} + Remove-Item ${zip_path} } Dir -Recurse ${COMPILER_TOOLS_DIR} | Select Fullname +cd ${CURRDIR} +git clone https://github.com/colmap/colmap.git +cd colmap +git checkout "${env:COLMAP_COMMIT_ID}" + & "./scripts/shell/enter_vs_dev_shell.ps1" [System.Collections.ArrayList]$DEPS = Get-Content -Path ".azure-pipelines/build-windows-vcpkg.txt" From 84746caa1121f9f345a242fb612373a0fa67d9eb Mon Sep 17 00:00:00 2001 From: Paul-Edouard Sarlin Date: Wed, 3 Jan 2024 22:01:37 +0100 Subject: [PATCH 20/24] Fix ccache download --- package/install-colmap-windows.ps1 | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/package/install-colmap-windows.ps1 b/package/install-colmap-windows.ps1 index 27a57fbc..c6527610 100755 --- a/package/install-colmap-windows.ps1 +++ b/package/install-colmap-windows.ps1 @@ -15,11 +15,15 @@ If (!(Test-Path -path ${NINJA_PATH} -PathType Leaf)) { If (!(Test-Path -path "${COMPILER_TOOLS_DIR}/ccache.exe" -PathType Leaf)) { # For some reason this CI runs an earlier PowerShell version that is # not compatible with colmap/.azure-pipelines/install-ccache.ps1 - $zip_path = "${env:TEMP}/ccache.zip" - $url = "https://github.com/ccache/ccache/releases/download/v4.8/ccache-4.8-windows-x86_64.zip" + $folder = "ccache-4.8-windows-x86_64" + $url = "https://github.com/ccache/ccache/releases/download/v4.8/${folder}.zip" + $zip_path = "${env:TEMP}/${folder}.zip" + $folder_path = "${env:TEMP}/${folder}" curl.exe -L -o ${zip_path} ${url} - Expand-Archive -LiteralPath ${zip_path} -DestinationPath ${COMPILER_TOOLS_DIR} + Expand-Archive -LiteralPath ${zip_path} -DestinationPath "$env:TEMP" + Move-Item -Force "${folder_path}/ccache.exe" ${COMPILER_TOOLS_DIR} Remove-Item ${zip_path} + Remove-Item -Recurse ${folder_path} } Dir -Recurse ${COMPILER_TOOLS_DIR} | Select Fullname From d9a325321698e756f590a8af5dc7270bcb61ecec Mon Sep 17 00:00:00 2001 From: Paul-Edouard Sarlin Date: Wed, 3 Jan 2024 22:23:47 +0100 Subject: [PATCH 21/24] Cleanup --- .github/workflows/build-new.yml | 9 +-------- package/install-colmap-macos.sh | 4 ++++ package/install-colmap-windows.ps1 | 5 ++++- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build-new.yml b/.github/workflows/build-new.yml index e5169599..c37ec3e0 100644 --- a/.github/workflows/build-new.yml +++ b/.github/workflows/build-new.yml @@ -17,7 +17,7 @@ jobs: strategy: matrix: config: [ - #{os: ubuntu-latest}, + {os: ubuntu-latest}, {os: macos-13, arch: x86_64}, {os: macos-13, arch: arm64}, {os: windows-latest}, @@ -92,13 +92,6 @@ jobs: with: name: pycolmap-${{ matrix.config.os }}-${{ matrix.config.arch }} path: wheelhouse/pycolmap-*.whl - - name: Cleanup compiler cache (macOS) - if: runner.os == 'macOS' - run: | - set -x - ccache --show-stats --verbose - ccache --evict-older-than 7d - ccache --show-stats --verbose pypi-publish: name: Publish wheels to PyPI diff --git a/package/install-colmap-macos.sh b/package/install-colmap-macos.sh index aed35ab8..1f952a73 100755 --- a/package/install-colmap-macos.sh +++ b/package/install-colmap-macos.sh @@ -57,3 +57,7 @@ cmake .. -GNinja -DGUI_ENABLED=OFF \ -DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES} \ `if [[ ${CIBW_ARCHS_MACOS} == "arm64" ]]; then echo "-DSIMD_ENABLED=OFF"; fi` ninja install + +ccache --show-stats --verbose +ccache --evict-older-than 1d +ccache --show-stats --verbose diff --git a/package/install-colmap-windows.ps1 b/package/install-colmap-windows.ps1 index c6527610..f871469d 100755 --- a/package/install-colmap-windows.ps1 +++ b/package/install-colmap-windows.ps1 @@ -25,7 +25,6 @@ If (!(Test-Path -path "${COMPILER_TOOLS_DIR}/ccache.exe" -PathType Leaf)) { Remove-Item ${zip_path} Remove-Item -Recurse ${folder_path} } -Dir -Recurse ${COMPILER_TOOLS_DIR} | Select Fullname cd ${CURRDIR} git clone https://github.com/colmap/colmap.git @@ -53,3 +52,7 @@ cmake .. ` -DCMAKE_TOOLCHAIN_FILE="${env:CMAKE_TOOLCHAIN_FILE}" ` -DVCPKG_TARGET_TRIPLET="x64-windows" & ${NINJA_PATH} install + +ccache --show-stats --verbose +ccache --evict-older-than 1d +ccache --show-stats --verbose From 720fcae4bc2bc9c444d2898dbe2f7ebc232d1b88 Mon Sep 17 00:00:00 2001 From: Paul-Edouard Sarlin Date: Sun, 7 Jan 2024 20:02:38 +0100 Subject: [PATCH 22/24] Force release build type for macOS --- package/install-colmap-macos.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/package/install-colmap-macos.sh b/package/install-colmap-macos.sh index 1f952a73..95c32c2b 100755 --- a/package/install-colmap-macos.sh +++ b/package/install-colmap-macos.sh @@ -51,6 +51,7 @@ export ARCHFLAGS="-arch ${CIBW_ARCHS_MACOS}" cmake .. -GNinja -DGUI_ENABLED=OFF \ -DCUDA_ENABLED=OFF \ -DCGAL_ENABLED=OFF \ + -DCMAKE_BUILD_TYPE=Release \ -DCCACHE_ENABLED=ON \ -DCMAKE_TOOLCHAIN_FILE="${CMAKE_TOOLCHAIN_FILE}" \ -DVCPKG_TARGET_TRIPLET=${VCPKG_TARGET_TRIPLET} \ From 8ebadb265b20746105a5af45f8445ea6cb772971 Mon Sep 17 00:00:00 2001 From: Paul-Edouard Sarlin Date: Sun, 7 Jan 2024 20:04:00 +0100 Subject: [PATCH 23/24] Build with COLMAP 3.9 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 94be7554..a0ab0a00 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,7 +30,7 @@ wheel.expand-macos-universal-tags = true [tool.cibuildwheel] build = "cp3{8,9,10,11}-{macosx,manylinux,win}*" archs = ["auto64"] -environment = { COLMAP_COMMIT_ID="c206c7a333c8b2ef818135bdfe9d31a28bd2eb2b" } +environment = { COLMAP_COMMIT_ID="3.9" } test-command = "python -c \"import pycolmap; print(pycolmap.__version__)\"" [tool.cibuildwheel.linux] From 14a5d6975b0c527c98ee77628f0df2f70abd5ad0 Mon Sep 17 00:00:00 2001 From: Paul-Edouard Sarlin Date: Mon, 8 Jan 2024 08:44:46 +0100 Subject: [PATCH 24/24] Fetch boost from GitHub --- package/install-colmap-centos.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package/install-colmap-centos.sh b/package/install-colmap-centos.sh index aaab3a97..7480a67c 100755 --- a/package/install-colmap-centos.sh +++ b/package/install-colmap-centos.sh @@ -25,8 +25,8 @@ yum install -y \ # ------ Install boost ------ cd ${CURRDIR} -export BOOST_FILENAME=boost_1_71_0 -wget -nv https://boostorg.jfrog.io/artifactory/main/release/1.71.0/source/${BOOST_FILENAME}.tar.gz +export BOOST_FILENAME="boost-1.84.0" +wget -nv https://github.com/boostorg/boost/releases/download/${BOOST_FILENAME}/${BOOST_FILENAME}.tar.gz tar xzf ${BOOST_FILENAME}.tar.gz cd ${BOOST_FILENAME} ./bootstrap.sh --with-libraries=filesystem,system,program_options,graph,test --without-icu