Enable building FAISS main statically (#241)

Port rapidsai/raft#2323 PR from RAFT [Cleans up a collection of anti-patterns in the cuvs CMake code while also enabling building faiss from latest main] Authors: - Tarang Jain (https://github.com/tarang-jain) Approvers: - Robert Maynard (https://github.com/robertmaynard) - Corey J. Nolet (https://github.com/cjnolet) - Paul Taylor (https://github.com/trxcllnt) - Ray Douglass (https://github.com/raydouglass) URL: #241
rapidsai · Jul 27, 2024 · b882590 · b882590
1 parent 337c696
commit b882590
Show file tree

Hide file tree

Showing 10 changed files with 286 additions and 267 deletions.
diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile
@@ -5,6 +5,13 @@ ARG PYTHON_PACKAGE_MANAGER=conda
 
 FROM ${BASE} as pip-base
 
+RUN apt update -y \
+ && DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends \
+    # faiss dependencies
+    libblas-dev \
+    liblapack-dev \
+ && rm -rf /tmp/* /var/tmp/* /var/cache/apt/* /var/lib/apt/lists/*;
+
 ENV DEFAULT_VIRTUAL_ENV=rapids
 
 FROM ${BASE} as conda-base

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
@@ -682,13 +682,13 @@ rapids_export(
 # * build test executable ----------------------------------------------------
 
 if(BUILD_TESTS OR BUILD_C_TESTS)
-  include(internal/CMakeLists.txt)
-  include(test/CMakeLists.txt)
+  add_subdirectory(internal)
+  add_subdirectory(test)
 endif()
 
 # ##################################################################################################
 # * build ann benchmark executable -----------------------------------------------
 
 if(BUILD_ANN_BENCH)
-  include(bench/ann/CMakeLists.txt)
+  add_subdirectory(bench/ann/)
 endif()
diff --git a/cpp/bench/ann/CMakeLists.txt b/cpp/bench/ann/CMakeLists.txt
@@ -12,6 +12,8 @@
 # the License.
 # =============================================================================
 
+list(APPEND CMAKE_MODULE_PATH "${CUVS_SOURCE_DIR}")
+
 # ##################################################################################################
 # * benchmark options ------------------------------------------------------------------------------
 
@@ -39,31 +41,18 @@ option(CUVS_ANN_BENCH_SINGLE_EXE
 
 find_package(Threads REQUIRED)
 
+set(CUVS_ANN_BENCH_USE_FAISS ON)
+set(CUVS_FAISS_ENABLE_GPU ON)
+set(CUVS_USE_FAISS_STATIC ON)
+
 if(BUILD_CPU_ONLY)
   set(CUVS_FAISS_ENABLE_GPU OFF)
-  set(CUVS_ANN_BENCH_USE_FAISS_GPU_FLAT OFF)
-  set(CUVS_ANN_BENCH_USE_FAISS_GPU_IVF_FLAT OFF)
-  set(CUVS_ANN_BENCH_USE_FAISS_GPU_IVF_PQ OFF)
   set(CUVS_ANN_BENCH_USE_CUVS_IVF_FLAT OFF)
   set(CUVS_ANN_BENCH_USE_CUVS_IVF_PQ OFF)
   set(CUVS_ANN_BENCH_USE_CUVS_CAGRA OFF)
   set(CUVS_ANN_BENCH_USE_CUVS_BRUTE_FORCE OFF)
   set(CUVS_ANN_BENCH_USE_CUVS_CAGRA_HNSWLIB OFF)
   set(CUVS_ANN_BENCH_USE_GGNN OFF)
-else()
-  set(CUVS_FAISS_ENABLE_GPU ON)
-endif()
-
-set(CUVS_ANN_BENCH_USE_FAISS OFF)
-if(CUVS_ANN_BENCH_USE_FAISS_GPU_FLAT
-   OR CUVS_ANN_BENCH_USE_FAISS_GPU_IVF_PQ
-   OR CUVS_ANN_BENCH_USE_FAISS_GPU_IVF_FLAT
-   OR CUVS_ANN_BENCH_USE_FAISS_CPU_FLAT
-   OR CUVS_ANN_BENCH_USE_FAISS_CPU_IVF_PQ
-   OR CUVS_ANN_BENCH_USE_FAISS_CPU_IVF_FLAT
-)
-  set(CUVS_ANN_BENCH_USE_FAISS ON)
-  set(CUVS_USE_FAISS_STATIC ON)
 endif()
 
 set(CUVS_ANN_BENCH_USE_CUVS OFF)
@@ -80,21 +69,17 @@ endif()
 # * Fetch requirements -------------------------------------------------------------
 
 if(CUVS_ANN_BENCH_USE_HNSWLIB OR CUVS_ANN_BENCH_USE_CUVS_CAGRA_HNSWLIB)
-  include(cmake/thirdparty/get_hnswlib.cmake)
+  include(cmake/thirdparty/get_hnswlib)
 endif()
 
-include(cmake/thirdparty/get_nlohmann_json.cmake)
+include(cmake/thirdparty/get_nlohmann_json)
 
 if(CUVS_ANN_BENCH_USE_GGNN)
-  include(cmake/thirdparty/get_ggnn.cmake)
+  include(cmake/thirdparty/get_ggnn)
 endif()
 
 if(CUVS_ANN_BENCH_USE_FAISS)
-  # We need to ensure that faiss has all the conda information. So we currently use the very ugly
-  # hammer of `link_libraries` to ensure that all targets in this directory and the faiss directory
-  # will have the conda includes/link dirs
-  link_libraries($<TARGET_NAME_IF_EXISTS:conda_env>)
-  include(cmake/thirdparty/get_faiss.cmake)
+  include(cmake/thirdparty/get_faiss)
 endif()
 
 # ##################################################################################################
@@ -154,8 +139,6 @@ function(ConfigureAnnBench)
             $<$<BOOL:${GPU_BUILD}>:CUDA::cudart_static>
             $<TARGET_NAME_IF_EXISTS:OpenMP::OpenMP_CXX>
             $<TARGET_NAME_IF_EXISTS:conda_env>
-            -static-libgcc
-            -static-libstdc++
   )
 
   set_target_properties(
@@ -210,131 +193,123 @@ endif()
 
 if(CUVS_ANN_BENCH_USE_HNSWLIB)
   ConfigureAnnBench(
-    NAME HNSWLIB PATH bench/ann/src/hnswlib/hnswlib_benchmark.cpp LINKS hnswlib::hnswlib
+    NAME HNSWLIB PATH src/hnswlib/hnswlib_benchmark.cpp LINKS hnswlib::hnswlib
   )
 
 endif()
 
 if(CUVS_ANN_BENCH_USE_CUVS_IVF_PQ)
   ConfigureAnnBench(
-    NAME CUVS_IVF_PQ PATH bench/ann/src/cuvs/cuvs_benchmark.cu
-    $<$<BOOL:${CUVS_ANN_BENCH_USE_CUVS_IVF_PQ}>:bench/ann/src/cuvs/cuvs_ivf_pq.cu> LINKS cuvs
+    NAME CUVS_IVF_PQ
+    PATH
+    src/cuvs/cuvs_benchmark.cu
+    src/cuvs/cuvs_ivf_pq.cu
+    LINKS cuvs
   )
 endif()
 
 if(CUVS_ANN_BENCH_USE_CUVS_IVF_FLAT)
   ConfigureAnnBench(
-    NAME CUVS_IVF_FLAT PATH bench/ann/src/cuvs/cuvs_benchmark.cu
-    $<$<BOOL:${CUVS_ANN_BENCH_USE_CUVS_IVF_FLAT}>:bench/ann/src/cuvs/cuvs_ivf_flat.cu> LINKS cuvs
+    NAME CUVS_IVF_FLAT
+    PATH
+    src/cuvs/cuvs_benchmark.cu
+    src/cuvs/cuvs_ivf_flat.cu
+    LINKS
+    cuvs
   )
 endif()
 
 if(CUVS_ANN_BENCH_USE_CUVS_BRUTE_FORCE)
-  ConfigureAnnBench(NAME CUVS_BRUTE_FORCE PATH bench/ann/src/cuvs/cuvs_benchmark.cu LINKS cuvs)
+  ConfigureAnnBench(NAME CUVS_BRUTE_FORCE PATH src/cuvs/cuvs_benchmark.cu LINKS cuvs)
 endif()
 
 if(CUVS_ANN_BENCH_USE_CUVS_CAGRA)
   ConfigureAnnBench(
     NAME
     CUVS_CAGRA
     PATH
-    bench/ann/src/cuvs/cuvs_benchmark.cu
-    $<$<BOOL:${CUVS_ANN_BENCH_USE_CUVS_CAGRA}>:bench/ann/src/cuvs/cuvs_cagra_float.cu>
-    $<$<BOOL:${CUVS_ANN_BENCH_USE_CUVS_CAGRA}>:bench/ann/src/cuvs/cuvs_cagra_half.cu>
-    $<$<BOOL:${CUVS_ANN_BENCH_USE_CUVS_CAGRA}>:bench/ann/src/cuvs/cuvs_cagra_int8_t.cu>
-    $<$<BOOL:${CUVS_ANN_BENCH_USE_CUVS_CAGRA}>:bench/ann/src/cuvs/cuvs_cagra_uint8_t.cu>
+    src/cuvs/cuvs_benchmark.cu
+    src/cuvs/cuvs_cagra_float.cu
+    src/cuvs/cuvs_cagra_half.cu
+    src/cuvs/cuvs_cagra_int8_t.cu
+    src/cuvs/cuvs_cagra_uint8_t.cu
     LINKS
     cuvs
   )
 endif()
 
 if(CUVS_ANN_BENCH_USE_CUVS_CAGRA_HNSWLIB)
   ConfigureAnnBench(
-    NAME CUVS_CAGRA_HNSWLIB PATH bench/ann/src/cuvs/cuvs_cagra_hnswlib.cu LINKS cuvs
+    NAME CUVS_CAGRA_HNSWLIB PATH src/cuvs/cuvs_cagra_hnswlib.cu LINKS cuvs
     hnswlib::hnswlib
   )
 endif()
 
-set(CUVS_FAISS_TARGETS faiss::faiss)
-if(TARGET faiss::faiss_avx2)
-  set(CUVS_FAISS_TARGETS faiss::faiss_avx2)
-endif()
-
 message("CUVS_FAISS_TARGETS: ${CUVS_FAISS_TARGETS}")
 message("CUDAToolkit_LIBRARY_DIR: ${CUDAToolkit_LIBRARY_DIR}")
 if(CUVS_ANN_BENCH_USE_FAISS_CPU_FLAT)
   ConfigureAnnBench(
-    NAME FAISS_CPU_FLAT PATH bench/ann/src/faiss/faiss_cpu_benchmark.cpp LINKS
+    NAME FAISS_CPU_FLAT PATH src/faiss/faiss_cpu_benchmark.cpp LINKS
     ${CUVS_FAISS_TARGETS}
   )
 endif()
 
 if(CUVS_ANN_BENCH_USE_FAISS_CPU_IVF_FLAT)
   ConfigureAnnBench(
-    NAME FAISS_CPU_IVF_FLAT PATH bench/ann/src/faiss/faiss_cpu_benchmark.cpp LINKS
+    NAME FAISS_CPU_IVF_FLAT PATH src/faiss/faiss_cpu_benchmark.cpp LINKS
     ${CUVS_FAISS_TARGETS}
   )
 endif()
 
 if(CUVS_ANN_BENCH_USE_FAISS_CPU_IVF_PQ)
   ConfigureAnnBench(
-    NAME FAISS_CPU_IVF_PQ PATH bench/ann/src/faiss/faiss_cpu_benchmark.cpp LINKS
+    NAME FAISS_CPU_IVF_PQ PATH src/faiss/faiss_cpu_benchmark.cpp LINKS
     ${CUVS_FAISS_TARGETS}
   )
 endif()
 
-if(CUVS_ANN_BENCH_USE_FAISS_GPU_IVF_FLAT)
+if(CUVS_ANN_BENCH_USE_FAISS_GPU_IVF_FLAT AND CUVS_FAISS_ENABLE_GPU)
   ConfigureAnnBench(
-    NAME FAISS_GPU_IVF_FLAT PATH bench/ann/src/faiss/faiss_gpu_benchmark.cu LINKS
+    NAME FAISS_GPU_IVF_FLAT PATH src/faiss/faiss_gpu_benchmark.cu LINKS
     ${CUVS_FAISS_TARGETS}
   )
 endif()
 
-if(CUVS_ANN_BENCH_USE_FAISS_GPU_IVF_PQ)
+if(CUVS_ANN_BENCH_USE_FAISS_GPU_IVF_PQ AND CUVS_FAISS_ENABLE_GPU)
   ConfigureAnnBench(
-    NAME FAISS_GPU_IVF_PQ PATH bench/ann/src/faiss/faiss_gpu_benchmark.cu LINKS
+    NAME FAISS_GPU_IVF_PQ PATH src/faiss/faiss_gpu_benchmark.cu LINKS
     ${CUVS_FAISS_TARGETS}
   )
 endif()
 
-if(CUVS_ANN_BENCH_USE_FAISS_GPU_FLAT)
+if(CUVS_ANN_BENCH_USE_FAISS_GPU_FLAT AND CUVS_FAISS_ENABLE_GPU)
   ConfigureAnnBench(
-    NAME FAISS_GPU_FLAT PATH bench/ann/src/faiss/faiss_gpu_benchmark.cu LINKS ${CUVS_FAISS_TARGETS}
+    NAME FAISS_GPU_FLAT PATH src/faiss/faiss_gpu_benchmark.cu LINKS ${CUVS_FAISS_TARGETS}
   )
 endif()
 
 if(CUVS_ANN_BENCH_USE_GGNN)
-  include(cmake/thirdparty/get_glog.cmake)
+  include(cmake/thirdparty/get_glog)
   ConfigureAnnBench(
-    NAME GGNN PATH bench/ann/src/ggnn/ggnn_benchmark.cu LINKS glog::glog ggnn::ggnn CUDA::curand
+    NAME GGNN PATH src/ggnn/ggnn_benchmark.cu LINKS glog::glog ggnn::ggnn CUDA::curand
   )
 endif()
 
 # ##################################################################################################
 # * Dynamically-loading ANN_BENCH executable -------------------------------------------------------
 if(CUVS_ANN_BENCH_SINGLE_EXE)
-  add_executable(ANN_BENCH bench/ann/src/common/benchmark.cpp)
-
-  # Build and link static version of the GBench to keep ANN_BENCH self-contained.
-  get_target_property(TMP_PROP benchmark::benchmark SOURCES)
-  add_library(benchmark_static STATIC ${TMP_PROP})
-  get_target_property(TMP_PROP benchmark::benchmark INCLUDE_DIRECTORIES)
-  target_include_directories(benchmark_static PUBLIC ${TMP_PROP})
-  get_target_property(TMP_PROP benchmark::benchmark LINK_LIBRARIES)
-  target_link_libraries(benchmark_static PUBLIC ${TMP_PROP})
+  add_executable(ANN_BENCH src/common/benchmark.cpp)
 
   target_include_directories(ANN_BENCH PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
 
   target_link_libraries(
     ANN_BENCH
     PRIVATE raft::raft
             nlohmann_json::nlohmann_json
-            benchmark_static
+            benchmark::benchmark
             dl
-            -static-libgcc
             fmt::fmt-header-only
             spdlog::spdlog_header_only
-            -static-libstdc++
             $<$<BOOL:${NVTX3_HEADERS_FOUND}>:CUDA::nvtx3>
   )
   set_target_properties(

diff --git a/cpp/cmake/patches/faiss_override.json b/cpp/cmake/patches/faiss_override.json
@@ -0,0 +1,9 @@
+{
+    "packages" : {
+      "faiss" : {
+        "version": "1.7.4",
+        "git_url": "https://github.com/facebookresearch/faiss.git",
+        "git_tag": "main"
+      }
+    }
+  }
diff --git a/cpp/cmake/patches/ggnn_override.json b/cpp/cmake/patches/ggnn_override.json
@@ -0,0 +1,16 @@
+{
+    "packages" : {
+      "ggnn" : {
+        "version": "0.5",
+        "git_url": "https://github.com/cgtuebingen/ggnn.git",
+        "git_tag": "release_${version}",
+        "patches" : [
+          {
+            "file" : "${current_json_dir}/ggnn.diff",
+            "issue" : "Correct compilation issues",
+            "fixed_in" : ""
+          }
+        ]
+      }
+    }
+  }
diff --git a/cpp/cmake/patches/hnswlib_override.json b/cpp/cmake/patches/hnswlib_override.json
@@ -0,0 +1,16 @@
+{
+    "packages" : {
+      "hnswlib" : {
+        "version": "0.6.2",
+        "git_url": "https://github.com/nmslib/hnswlib.git",
+        "git_tag": "v${version}",
+        "patches" : [
+          {
+            "file" : "${current_json_dir}/hnswlib.diff",
+            "issue" : "Correct compilation issues",
+            "fixed_in" : ""
+          }
+        ]
+      }
+    }
+  }