From 0a017da2938747a92c5731c019f057d5ef7df7a8 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Fri, 15 Nov 2024 13:49:45 -0500 Subject: [PATCH] Removing unused diff files --- cpp/cmake/patches/faiss_override.json | 9 - cpp/cmake/patches/ggnn.diff | 230 ------------------------ cpp/cmake/patches/ggnn_override.json | 16 -- cpp/cmake/patches/hnswlib.diff | 188 ------------------- cpp/cmake/patches/hnswlib_override.json | 16 -- 5 files changed, 459 deletions(-) delete mode 100644 cpp/cmake/patches/faiss_override.json delete mode 100644 cpp/cmake/patches/ggnn.diff delete mode 100644 cpp/cmake/patches/ggnn_override.json delete mode 100644 cpp/cmake/patches/hnswlib.diff delete mode 100644 cpp/cmake/patches/hnswlib_override.json diff --git a/cpp/cmake/patches/faiss_override.json b/cpp/cmake/patches/faiss_override.json deleted file mode 100644 index 5d18c77fec..0000000000 --- a/cpp/cmake/patches/faiss_override.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "packages" : { - "faiss" : { - "version": "1.9.0", - "git_url": "https://github.com/facebookresearch/faiss.git", - "git_tag": "v1.9.0" - } - } -} diff --git a/cpp/cmake/patches/ggnn.diff b/cpp/cmake/patches/ggnn.diff deleted file mode 100644 index fc45298803..0000000000 --- a/cpp/cmake/patches/ggnn.diff +++ /dev/null @@ -1,230 +0,0 @@ ---- a/include/ggnn/cache/cuda_simple_knn_sym_cache.cuh -+++ b/include/ggnn/cache/cuda_simple_knn_sym_cache.cuh -@@ -62,7 +62,7 @@ struct SimpleKNNSymCache { - const ValueT dist_half) - : dist_query(dist_query), dist_half(dist_half) {} - -- __device__ __forceinline__ DistQueryAndHalf() {} -+ DistQueryAndHalf() = default; - }; - - struct DistanceAndNorm { -@@ -98,8 +98,7 @@ struct SimpleKNNSymCache { - KeyT cache; - DistQueryAndHalf dist; - bool flag; -- -- __device__ __forceinline__ SyncTempStorage() {} -+ SyncTempStorage() = default; - }; - - public: -diff --git a/include/ggnn/cuda_knn_ggnn_gpu_instance.cuh b/include/ggnn/cuda_knn_ggnn_gpu_instance.cuh -index 8cbaf0d..6eb72ac 100644 ---- a/include/ggnn/cuda_knn_ggnn_gpu_instance.cuh -+++ b/include/ggnn/cuda_knn_ggnn_gpu_instance.cuh -@@ -41,7 +41,6 @@ limitations under the License. - #include "ggnn/sym/cuda_knn_sym_query_layer.cuh" - #include "ggnn/utils/cuda_knn_utils.cuh" - #include "ggnn/utils/cuda_knn_constants.cuh" --#include "ggnn/utils/cuda_knn_dataset.cuh" - - template - __global__ void divide(ValueT* res, ValueT* input, ValueT N) { -@@ -98,9 +97,7 @@ struct GGNNGPUInstance { - typedef GGNNGraphDevice GGNNGraphDevice; - typedef GGNNGraphHost GGNNGraphHost; - -- const Dataset* dataset; - GGNNGraphBuffer* ggnn_buffer {nullptr}; -- GGNNQuery ggnn_query; - - // Graph Shards resident on the GPU - std::vector ggnn_shards; -@@ -117,13 +114,12 @@ struct GGNNGPUInstance { - // number of shards that need to be processed by this instance - const int num_parts; - -- GGNNGPUInstance(const int gpu_id, const Dataset* dataset, -+ GGNNGPUInstance(const int gpu_id, - const int N_shard, const int L, - const bool enable_construction, const float tau_build, - const int num_parts=1, const int num_cpu_buffers=1) : - N_shard{N_shard}, L{L}, tau_build{tau_build}, -- dataset{dataset}, gpu_id{gpu_id}, -- ggnn_query{dataset->N_query, D, KQuery, num_parts}, -+ gpu_id{gpu_id}, - num_parts{num_parts} - { - CHECK_LE(L, MAX_LAYER); -@@ -135,7 +131,6 @@ struct GGNNGPUInstance { - CHECK_EQ(current_gpu_id, gpu_id) << "cudaSetDevice() needs to be called in advance!"; - } - -- ggnn_query.loadQueriesAsync(dataset->h_query, 0); - - computeGraphParameters(); - -@@ -186,7 +181,7 @@ struct GGNNGPUInstance { - } - - GGNNGPUInstance(const GGNNGPUInstance& other) -- : dataset{nullptr}, ggnn_query{0, D, KQuery}, -+ : - gpu_id{0}, N_shard{0}, num_parts{0} { - // this exists to allow using vector::emplace_back - // when it triggers a reallocation, this code will be called. -@@ -305,6 +300,7 @@ struct GGNNGPUInstance { - - // io - -+ /* - void waitForDiskIO(const int shard_id) { - auto& cpu_buffer = ggnn_cpu_buffers[shard_id%ggnn_cpu_buffers.size()]; - if (cpu_buffer.disk_io_thread.joinable()) -@@ -468,11 +464,12 @@ struct GGNNGPUInstance { - CHECK_CUDA(cudaDeviceSynchronize()); - CHECK_CUDA(cudaPeekAtLastError()); - } -+ */ - - // graph operations - - template -- void queryLayer(const int shard_id = 0) const { -+ void queryLayer(const BaseT* d_query, int batch_size, KeyT* d_query_result_ids, ValueT* d_query_result_dists, const int shard_id = 0) const { - CHECK_CUDA(cudaSetDevice(gpu_id)); - const auto& shard = ggnn_shards.at(shard_id%ggnn_shards.size()); - -@@ -482,21 +479,21 @@ struct GGNNGPUInstance { - - int* m_dist_statistics = nullptr; - if (DIST_STATS) -- cudaMallocManaged(&m_dist_statistics, dataset->N_query * sizeof(int)); -+ cudaMallocManaged(&m_dist_statistics, batch_size * sizeof(int)); - - QueryKernel query_kernel; - query_kernel.d_base = shard.d_base; -- query_kernel.d_query = ggnn_query.d_query; -+ query_kernel.d_query = d_query; - - query_kernel.d_graph = shard.d_graph; -- query_kernel.d_query_results = ggnn_query.d_query_result_ids; -- query_kernel.d_query_results_dists = ggnn_query.d_query_result_dists; -+ query_kernel.d_query_results = d_query_result_ids; -+ query_kernel.d_query_results_dists = d_query_result_dists; - - query_kernel.d_translation = shard.d_translation; - - query_kernel.d_nn1_stats = shard.d_nn1_stats; - -- query_kernel.N = dataset->N_query; -+ query_kernel.N = batch_size; - query_kernel.N_offset = 0; - - query_kernel.d_dist_stats = m_dist_statistics; -@@ -771,6 +768,16 @@ struct GGNNGPUInstance { - sym(layer, shard_id); - } - } -+ -+ void set_stream(cudaStream_t stream) { -+ assert(ggnn_shards.size() == 1); -+ ggnn_shards.at(0).stream = stream; -+ } -+ -+ void set_base_data(const BaseT* dataset) { -+ assert(ggnn_shards.size() == 1); -+ ggnn_shards.at(0).d_base = dataset; -+ } - }; - - #endif // INCLUDE_GGNN_CUDA_KNN_GGNN_GPU_INSTANCE_CUH_ -diff --git a/include/ggnn/graph/cuda_knn_ggnn_graph_device.cuh b/include/ggnn/graph/cuda_knn_ggnn_graph_device.cuh -index c94a8f1..781226d 100644 ---- a/include/ggnn/graph/cuda_knn_ggnn_graph_device.cuh -+++ b/include/ggnn/graph/cuda_knn_ggnn_graph_device.cuh -@@ -50,7 +50,7 @@ struct GGNNGraphDevice { - ValueT* d_nn1_stats; - - /// base data pointer for the shard. -- BaseT* d_base; -+ const BaseT* d_base; - - /// combined memory pool - char* d_memory; -@@ -69,7 +69,9 @@ struct GGNNGraphDevice { - const size_t selection_translation_size = align8(ST_all * sizeof(KeyT)); - const size_t nn1_stats_size = align8(2 * sizeof(ValueT)); - total_graph_size = graph_size + 2 * selection_translation_size + nn1_stats_size; -- base_size = align8(static_cast(N) * D * sizeof(BaseT)); -+ // base_size = align8(static_cast(N) * D * sizeof(BaseT)); -+ (void) N; -+ (void) D; - - const size_t total_size = base_size+total_graph_size; - -@@ -86,8 +88,7 @@ struct GGNNGraphDevice { - CHECK_CUDA(cudaMalloc(&d_memory, total_size)); - - size_t pos = 0; -- d_base = reinterpret_cast(d_memory+pos); -- pos += base_size; -+ d_base = nullptr; - d_graph = reinterpret_cast(d_memory+pos); - pos += graph_size; - d_translation = reinterpret_cast(d_memory+pos); -@@ -99,14 +100,14 @@ struct GGNNGraphDevice { - - CHECK_EQ(pos, total_size); - -- CHECK_CUDA(cudaStreamCreate(&stream)); -+ // CHECK_CUDA(cudaStreamCreate(&stream)); - - CHECK_CUDA(cudaPeekAtLastError()); - CHECK_CUDA(cudaDeviceSynchronize()); - CHECK_CUDA(cudaPeekAtLastError()); - } - -- GGNNGraphDevice(const GGNNGraphDevice& other) { -+ GGNNGraphDevice(const GGNNGraphDevice&) { - // this exists to allow using vector::emplace_back - // when it triggers a reallocation, this code will be called. - // always make sure that enough memory is reserved ahead of time. -@@ -116,7 +117,7 @@ struct GGNNGraphDevice { - ~GGNNGraphDevice() { - cudaFree(d_memory); - -- CHECK_CUDA(cudaStreamDestroy(stream)); -+ // CHECK_CUDA(cudaStreamDestroy(stream)); - } - }; - -diff --git a/include/ggnn/graph/cuda_knn_ggnn_graph_host.cuh b/include/ggnn/graph/cuda_knn_ggnn_graph_host.cuh -index 2055f9e..ef5843a 100644 ---- a/include/ggnn/graph/cuda_knn_ggnn_graph_host.cuh -+++ b/include/ggnn/graph/cuda_knn_ggnn_graph_host.cuh -@@ -92,7 +92,7 @@ struct GGNNGraphHost { - CHECK_CUDA(cudaPeekAtLastError()); - } - -- GGNNGraphHost(const GGNNGraphHost& other) { -+ GGNNGraphHost(const GGNNGraphHost&) { - // this exists to allow using vector::emplace_back - // when it triggers a reallocation, this code will be called. - // always make sure that enough memory is reserved ahead of time. -diff --git a/include/ggnn/select/cuda_knn_wrs_select_layer.cuh b/include/ggnn/select/cuda_knn_wrs_select_layer.cuh -index 49d76a1..eef69e6 100644 ---- a/include/ggnn/select/cuda_knn_wrs_select_layer.cuh -+++ b/include/ggnn/select/cuda_knn_wrs_select_layer.cuh -@@ -22,7 +22,6 @@ limitations under the License. - #include - #include - --#include - #include - - #include "ggnn/utils/cuda_knn_constants.cuh" --- -2.43.0 - diff --git a/cpp/cmake/patches/ggnn_override.json b/cpp/cmake/patches/ggnn_override.json deleted file mode 100644 index 768fae8b0c..0000000000 --- a/cpp/cmake/patches/ggnn_override.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "packages" : { - "ggnn" : { - "version": "0.5", - "git_url": "https://github.com/cgtuebingen/ggnn.git", - "git_tag": "release_${version}", - "patches" : [ - { - "file" : "${current_json_dir}/ggnn.diff", - "issue" : "Correct compilation issues", - "fixed_in" : "" - } - ] - } - } -} diff --git a/cpp/cmake/patches/hnswlib.diff b/cpp/cmake/patches/hnswlib.diff deleted file mode 100644 index e7f89a8cc9..0000000000 --- a/cpp/cmake/patches/hnswlib.diff +++ /dev/null @@ -1,188 +0,0 @@ ---- a/hnswlib/hnswalg.h -+++ b/hnswlib/hnswalg.h -@@ -3,6 +3,7 @@ - #include "visited_list_pool.h" - #include "hnswlib.h" - #include -+#include - #include - #include - #include -@@ -16,6 +17,8 @@ namespace hnswlib { - template - class HierarchicalNSW : public AlgorithmInterface { - public: -+ bool base_layer_only{false}; -+ int num_seeds=32; - static const tableint max_update_element_locks = 65536; - HierarchicalNSW(SpaceInterface *s) { - } -@@ -56,7 +59,7 @@ namespace hnswlib { - visited_list_pool_ = new VisitedListPool(1, max_elements); - - //initializations for special treatment of the first node -- enterpoint_node_ = -1; -+ enterpoint_node_ = std::numeric_limits::max(); - maxlevel_ = -1; - - linkLists_ = (char **) malloc(sizeof(void *) * max_elements_); -@@ -527,7 +530,7 @@ namespace hnswlib { - tableint *datal = (tableint *) (data + 1); - for (int i = 0; i < size; i++) { - tableint cand = datal[i]; -- if (cand < 0 || cand > max_elements_) -+ if (cand > max_elements_) - throw std::runtime_error("cand error"); - dist_t d = fstdistfunc_(query_data, getDataByInternalId(cand), dist_func_param_); - -@@ -1067,7 +1070,7 @@ namespace hnswlib { - tableint *datal = (tableint *) (data + 1); - for (int i = 0; i < size; i++) { - tableint cand = datal[i]; -- if (cand < 0 || cand > max_elements_) -+ if (cand > max_elements_) - throw std::runtime_error("cand error"); - dist_t d = fstdistfunc_(data_point, getDataByInternalId(cand), dist_func_param_); - if (d < curdist) { -@@ -1119,28 +1122,41 @@ namespace hnswlib { - tableint currObj = enterpoint_node_; - dist_t curdist = fstdistfunc_(query_data, getDataByInternalId(enterpoint_node_), dist_func_param_); - -- for (int level = maxlevel_; level > 0; level--) { -- bool changed = true; -- while (changed) { -- changed = false; -- unsigned int *data; -+ if (base_layer_only) { -+ // You can increase the number of seeds when testing large-scale dataset, num_seeds = 48 for 100M-scale -+ for (int i = 0; i < num_seeds; i++) { -+ tableint obj = i * (max_elements_ / num_seeds); -+ dist_t dist = fstdistfunc_(query_data, getDataByInternalId(obj), dist_func_param_); -+ if (dist < curdist) { -+ curdist = dist; -+ currObj = obj; -+ } -+ } -+ } -+ else{ -+ for (int level = maxlevel_; level > 0; level--) { -+ bool changed = true; -+ while (changed) { -+ changed = false; -+ unsigned int *data; - -- data = (unsigned int *) get_linklist(currObj, level); -- int size = getListCount(data); -- metric_hops++; -- metric_distance_computations+=size; -+ data = (unsigned int *) get_linklist(currObj, level); -+ int size = getListCount(data); -+ metric_hops++; -+ metric_distance_computations+=size; - -- tableint *datal = (tableint *) (data + 1); -- for (int i = 0; i < size; i++) { -- tableint cand = datal[i]; -- if (cand < 0 || cand > max_elements_) -- throw std::runtime_error("cand error"); -- dist_t d = fstdistfunc_(query_data, getDataByInternalId(cand), dist_func_param_); -+ tableint *datal = (tableint *) (data + 1); -+ for (int i = 0; i < size; i++) { -+ tableint cand = datal[i]; -+ if (cand > max_elements_) -+ throw std::runtime_error("cand error"); -+ dist_t d = fstdistfunc_(query_data, getDataByInternalId(cand), dist_func_param_); - -- if (d < curdist) { -- curdist = d; -- currObj = cand; -- changed = true; -+ if (d < curdist) { -+ curdist = d; -+ currObj = cand; -+ changed = true; -+ } - } - } - } -diff --git a/hnswlib/space_l2.h b/hnswlib/space_l2.h -index 4413537..c3240f3 100644 ---- a/hnswlib/space_l2.h -+++ b/hnswlib/space_l2.h -@@ -252,13 +252,14 @@ namespace hnswlib { - ~L2Space() {} - }; - -+ template - static int - L2SqrI4x(const void *__restrict pVect1, const void *__restrict pVect2, const void *__restrict qty_ptr) { - - size_t qty = *((size_t *) qty_ptr); - int res = 0; -- unsigned char *a = (unsigned char *) pVect1; -- unsigned char *b = (unsigned char *) pVect2; -+ T *a = (T *) pVect1; -+ T *b = (T *) pVect2; - - qty = qty >> 2; - for (size_t i = 0; i < qty; i++) { -@@ -279,11 +280,12 @@ namespace hnswlib { - return (res); - } - -+ template - static int L2SqrI(const void* __restrict pVect1, const void* __restrict pVect2, const void* __restrict qty_ptr) { - size_t qty = *((size_t*)qty_ptr); - int res = 0; -- unsigned char* a = (unsigned char*)pVect1; -- unsigned char* b = (unsigned char*)pVect2; -+ T* a = (T*)pVect1; -+ T* b = (T*)pVect2; - - for(size_t i = 0; i < qty; i++) - { -@@ -294,6 +296,7 @@ namespace hnswlib { - return (res); - } - -+ template - class L2SpaceI : public SpaceInterface { - - DISTFUNC fstdistfunc_; -@@ -302,10 +305,10 @@ namespace hnswlib { - public: - L2SpaceI(size_t dim) { - if(dim % 4 == 0) { -- fstdistfunc_ = L2SqrI4x; -+ fstdistfunc_ = L2SqrI4x; - } - else { -- fstdistfunc_ = L2SqrI; -+ fstdistfunc_ = L2SqrI; - } - dim_ = dim; - data_size_ = dim * sizeof(unsigned char); -diff --git a/hnswlib/visited_list_pool.h b/hnswlib/visited_list_pool.h -index 5e1a4a5..4195ebd 100644 ---- a/hnswlib/visited_list_pool.h -+++ b/hnswlib/visited_list_pool.h -@@ -3,6 +3,7 @@ - #include - #include - #include -+#include - - namespace hnswlib { - typedef unsigned short int vl_type; -@@ -14,7 +15,7 @@ namespace hnswlib { - unsigned int numelements; - - VisitedList(int numelements1) { -- curV = -1; -+ curV = std::numeric_limits::max(); - numelements = numelements1; - mass = new vl_type[numelements]; - } --- -2.43.0 - diff --git a/cpp/cmake/patches/hnswlib_override.json b/cpp/cmake/patches/hnswlib_override.json deleted file mode 100644 index d6ab8a18a5..0000000000 --- a/cpp/cmake/patches/hnswlib_override.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "packages" : { - "hnswlib" : { - "version": "0.6.2", - "git_url": "https://github.com/nmslib/hnswlib.git", - "git_tag": "v${version}", - "patches" : [ - { - "file" : "${current_json_dir}/hnswlib.diff", - "issue" : "Correct compilation issues", - "fixed_in" : "" - } - ] - } - } -}