From 4c04a0e669e496fa48a9e17cf79dc9f8771dc7fe Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Mon, 20 Nov 2023 22:04:31 -0500 Subject: [PATCH] Renaming remaining namespaces and files --- cpp/bench/ann/src/common/ann_types.hpp | 21 +- cpp/bench/ann/src/common/benchmark.cpp | 10 +- cpp/bench/ann/src/common/benchmark.hpp | 4 +- cpp/bench/ann/src/common/conf.hpp | 4 +- cpp/bench/ann/src/common/cuda_stub.hpp | 4 +- cpp/bench/ann/src/common/dataset.hpp | 4 +- cpp/bench/ann/src/common/util.hpp | 4 +- .../ann/src/faiss/faiss_cpu_benchmark.cpp | 58 +- cpp/bench/ann/src/faiss/faiss_cpu_wrapper.h | 10 +- .../ann/src/faiss/faiss_gpu_benchmark.cu | 58 +- cpp/bench/ann/src/faiss/faiss_gpu_wrapper.h | 10 +- cpp/bench/ann/src/ggnn/ggnn_benchmark.cu | 45 +- cpp/bench/ann/src/ggnn/ggnn_wrapper.cuh | 4 +- .../ann/src/hnswlib/hnswlib_benchmark.cpp | 46 +- cpp/bench/ann/src/hnswlib/hnswlib_wrapper.h | 4 +- .../src/raft/raft_ann_bench_param_parser.h | 62 +- cpp/bench/ann/src/raft/raft_ann_bench_utils.h | 16 +- cpp/bench/ann/src/raft/raft_benchmark.cu | 45 +- cpp/bench/ann/src/raft/raft_cagra.cu | 4 +- cpp/bench/ann/src/raft/raft_cagra_hnswlib.cu | 30 +- .../ann/src/raft/raft_cagra_hnswlib_wrapper.h | 4 +- cpp/bench/ann/src/raft/raft_cagra_wrapper.h | 44 +- cpp/bench/ann/src/raft/raft_ivf_flat.cu | 4 +- .../ann/src/raft/raft_ivf_flat_wrapper.h | 26 +- cpp/bench/ann/src/raft/raft_ivf_pq.cu | 4 +- cpp/bench/ann/src/raft/raft_ivf_pq_wrapper.h | 18 +- cpp/bench/ann/src/raft/raft_wrapper.h | 22 +- cpp/bench/micro/cluster/kmeans.cu | 16 +- cpp/bench/micro/cluster/kmeans_balanced.cu | 12 +- cpp/bench/micro/common/benchmark.hpp | 8 +- cpp/bench/micro/distance/distance_common.cuh | 12 +- cpp/bench/micro/distance/distance_cosine.cu | 6 +- cpp/bench/micro/distance/distance_exp_l2.cu | 8 +- cpp/bench/micro/distance/distance_l1.cu | 6 +- cpp/bench/micro/distance/distance_unexp_l2.cu | 8 +- cpp/bench/micro/distance/fused_l2_nn.cu | 8 +- cpp/bench/micro/distance/kernels.cu | 10 +- cpp/bench/micro/distance/masked_nn.cu | 16 +- .../micro/distance/tune_pairwise/bench.cu | 6 +- .../micro/distance/tune_pairwise/kernel.cu | 12 +- .../micro/distance/tune_pairwise/kernel.cuh | 10 +- cpp/bench/micro/matrix/select_k.cu | 6 +- cpp/bench/micro/neighbors/cagra_bench.cuh | 22 +- cpp/bench/micro/neighbors/knn.cuh | 66 +-- .../knn/brute_force_float_int64_t.cu | 4 +- .../knn/brute_force_float_uint32_t.cu | 4 +- .../neighbors/knn/cagra_float_uint32_t.cu | 4 +- .../knn/ivf_flat_filter_float_int64_t.cu | 4 +- .../neighbors/knn/ivf_flat_float_int64_t.cu | 4 +- .../neighbors/knn/ivf_flat_int8_t_int64_t.cu | 4 +- .../neighbors/knn/ivf_flat_uint8_t_int64_t.cu | 4 +- .../knn/ivf_pq_filter_float_int64_t.cu | 4 +- .../neighbors/knn/ivf_pq_float_int64_t.cu | 4 +- .../neighbors/knn/ivf_pq_int8_t_int64_t.cu | 4 +- .../neighbors/knn/ivf_pq_uint8_t_int64_t.cu | 4 +- cpp/bench/micro/neighbors/refine.cuh | 18 +- .../micro/neighbors/refine_float_int64_t.cu | 6 +- .../micro/neighbors/refine_uint8_t_int64_t.cu | 6 +- .../cuvs/cluster/detail/connectivities.cuh | 16 +- cpp/include/cuvs/cluster/detail/kmeans.cuh | 24 +- .../cluster/detail/kmeans_auto_find_k.cuh | 8 +- .../cuvs/cluster/detail/kmeans_balanced.cuh | 32 +- .../cuvs/cluster/detail/kmeans_common.cuh | 24 +- cpp/include/cuvs/cluster/detail/mst.cuh | 4 +- .../cuvs/cluster/detail/single_linkage.cuh | 2 +- cpp/include/cuvs/cluster/kmeans.cuh | 42 +- cpp/include/cuvs/cluster/kmeans_balanced.cuh | 40 +- .../cuvs/cluster/kmeans_balanced_types.hpp | 4 +- cpp/include/cuvs/cluster/kmeans_types.hpp | 14 +- cpp/include/cuvs/cluster/single_linkage.cuh | 18 +- .../cuvs/distance/detail/compress_to_bits.cuh | 4 +- cpp/include/cuvs/distance/detail/distance.cuh | 22 +- .../distance/detail/distance_ops/all_ops.cuh | 28 +- .../distance/detail/distance_ops/canberra.cuh | 4 +- .../detail/distance_ops/correlation.cuh | 4 +- .../distance/detail/distance_ops/cosine.cuh | 4 +- .../distance/detail/distance_ops/cutlass.cuh | 4 +- .../distance/detail/distance_ops/hamming.cuh | 4 +- .../detail/distance_ops/hellinger.cuh | 4 +- .../detail/distance_ops/jensen_shannon.cuh | 4 +- .../detail/distance_ops/kl_divergence.cuh | 4 +- .../cuvs/distance/detail/distance_ops/l1.cuh | 4 +- .../distance/detail/distance_ops/l2_exp.cuh | 4 +- .../distance/detail/distance_ops/l2_unexp.cuh | 4 +- .../distance/detail/distance_ops/l_inf.cuh | 4 +- .../distance/detail/distance_ops/lp_unexp.cuh | 4 +- .../detail/distance_ops/russel_rao.cuh | 4 +- .../distance/detail/distance_ops/template.cuh | 4 +- .../detail/fused_distance_nn/cutlass_base.cuh | 4 +- .../detail/fused_distance_nn/epilogue.cuh | 6 +- .../distance/detail/fused_distance_nn/gemm.h | 4 +- .../cuvs/distance/detail/fused_l2_nn.cuh | 14 +- .../distance/detail/kernels/gram_matrix.cuh | 12 +- .../detail/kernels/kernel_factory.cuh | 6 +- .../detail/kernels/kernel_matrices.cuh | 10 +- .../distance/detail/kernels/rbf_fin_op.cuh | 6 +- .../distance/detail/masked_distance_base.cuh | 2 +- .../cuvs/distance/detail/masked_nn.cuh | 6 +- .../detail/pairwise_distance_cutlass_base.cuh | 2 +- .../detail/pairwise_matrix/dispatch-ext.cuh | 76 +-- .../detail/pairwise_matrix/dispatch-inl.cuh | 16 +- .../pairwise_matrix/dispatch_layout.cuh | 6 +- .../detail/pairwise_matrix/dispatch_sm60.cuh | 8 +- .../detail/pairwise_matrix/dispatch_sm80.cuh | 8 +- .../detail/pairwise_matrix/kernel_sm60.cuh | 8 +- .../detail/pairwise_matrix/params.cuh | 4 +- cpp/include/cuvs/distance/distance-ext.cuh | 542 +++++++++--------- cpp/include/cuvs/distance/distance-inl.cuh | 30 +- cpp/include/cuvs/distance/distance_types.hpp | 6 +- cpp/include/cuvs/distance/fused_l2_nn-ext.cuh | 4 +- cpp/include/cuvs/distance/fused_l2_nn-inl.cuh | 4 +- .../cuvs/distance/fused_l2_nn_helpers.cuh | 6 +- cpp/include/cuvs/distance/kernels.cuh | 14 +- cpp/include/cuvs/distance/masked_nn.cuh | 14 +- cpp/include/cuvs/distance/specializations.cuh | 22 - .../distance/specializations/distance.cuh | 22 - .../specializations/fused_l2_nn_min.cuh | 22 - cpp/include/cuvs/neighbors/ann_types.hpp | 8 +- cpp/include/cuvs/neighbors/ball_cover-ext.cuh | 28 +- cpp/include/cuvs/neighbors/ball_cover-inl.cuh | 50 +- .../cuvs/neighbors/ball_cover_types.hpp | 14 +- .../cuvs/neighbors/brute_force-ext.cuh | 40 +- .../cuvs/neighbors/brute_force-inl.cuh | 46 +- cpp/include/cuvs/neighbors/brute_force.cuh | 12 +- .../cuvs/neighbors/brute_force_types.hpp | 24 +- cpp/include/cuvs/neighbors/cagra.cuh | 36 +- .../cuvs/neighbors/cagra_serialize.cuh | 16 +- cpp/include/cuvs/neighbors/cagra_types.hpp | 36 +- .../cuvs/neighbors/detail/cagra/bitonic.hpp | 4 +- .../neighbors/detail/cagra/cagra_build.cuh | 22 +- .../neighbors/detail/cagra/cagra_search.cuh | 22 +- .../detail/cagra/cagra_serialize.cuh | 8 +- .../detail/cagra/compute_distance.hpp | 4 +- .../neighbors/detail/cagra/device_common.hpp | 4 +- .../cuvs/neighbors/detail/cagra/factory.cuh | 8 +- .../cuvs/neighbors/detail/cagra/fragment.hpp | 4 +- .../neighbors/detail/cagra/graph_core.cuh | 4 +- .../cuvs/neighbors/detail/cagra/hashmap.hpp | 4 +- .../detail/cagra/search_multi_cta.cuh | 4 +- .../cagra/search_multi_cta_kernel-ext.cuh | 30 +- .../cagra/search_multi_cta_kernel-inl.cuh | 10 +- .../detail/cagra/search_multi_kernel.cuh | 10 +- .../neighbors/detail/cagra/search_plan.cuh | 8 +- .../detail/cagra/search_single_cta.cuh | 4 +- .../cagra/search_single_cta_kernel-ext.cuh | 30 +- .../cagra/search_single_cta_kernel-inl.cuh | 12 +- .../neighbors/detail/cagra/topk_by_radix.cuh | 4 +- .../detail/cagra/topk_for_cagra/topk.h | 4 +- .../detail/cagra/topk_for_cagra/topk_core.cuh | 4 +- .../cuvs/neighbors/detail/cagra/utils.hpp | 4 +- .../cuvs/neighbors/detail/div_utils.hpp | 4 +- .../detail/faiss_select/Comparators.cuh | 4 +- .../detail/faiss_select/DistanceUtils.h | 4 +- .../detail/faiss_select/MergeNetworkBlock.cuh | 8 +- .../detail/faiss_select/MergeNetworkUtils.cuh | 4 +- .../detail/faiss_select/MergeNetworkWarp.cuh | 8 +- .../neighbors/detail/faiss_select/Select.cuh | 10 +- .../detail/faiss_select/StaticUtils.h | 4 +- .../faiss_select/key_value_block_select.cuh | 8 +- .../cuvs/neighbors/detail/ivf_flat_build.cuh | 32 +- .../detail/ivf_flat_interleaved_scan-ext.cuh | 24 +- .../detail/ivf_flat_interleaved_scan-inl.cuh | 26 +- .../neighbors/detail/ivf_flat_search-ext.cuh | 20 +- .../neighbors/detail/ivf_flat_search-inl.cuh | 40 +- .../neighbors/detail/ivf_flat_serialize.cuh | 12 +- .../cuvs/neighbors/detail/ivf_pq_build.cuh | 36 +- .../neighbors/detail/ivf_pq_codepacking.cuh | 8 +- .../detail/ivf_pq_compute_similarity-ext.cuh | 60 +- .../detail/ivf_pq_compute_similarity-inl.cuh | 32 +- .../detail/ivf_pq_dummy_block_sort.cuh | 4 +- .../cuvs/neighbors/detail/ivf_pq_fp_8bit.cuh | 8 +- .../cuvs/neighbors/detail/ivf_pq_search.cuh | 36 +- .../neighbors/detail/ivf_pq_serialize.cuh | 16 +- .../cuvs/neighbors/detail/knn_brute_force.cuh | 64 +-- .../detail/knn_brute_force_batch_k_query.cuh | 22 +- .../cuvs/neighbors/detail/knn_merge_parts.cuh | 8 +- .../cuvs/neighbors/detail/nn_descent.cuh | 8 +- .../cuvs/neighbors/detail/refine_common.hpp | 6 +- .../cuvs/neighbors/detail/refine_device.cuh | 24 +- .../cuvs/neighbors/detail/refine_host-ext.hpp | 6 +- .../cuvs/neighbors/detail/refine_host-inl.hpp | 10 +- .../neighbors/detail/selection_faiss-ext.cuh | 8 +- .../neighbors/detail/selection_faiss-inl.cuh | 8 +- .../detail/selection_faiss_helpers.cuh | 4 +- .../cuvs/neighbors/epsilon_neighborhood.cuh | 8 +- cpp/include/cuvs/neighbors/ivf_flat-ext.cuh | 64 +-- cpp/include/cuvs/neighbors/ivf_flat-inl.cuh | 46 +- .../cuvs/neighbors/ivf_flat_codepacker.hpp | 8 +- .../cuvs/neighbors/ivf_flat_helpers.cuh | 14 +- .../cuvs/neighbors/ivf_flat_serialize.cuh | 8 +- cpp/include/cuvs/neighbors/ivf_flat_types.hpp | 22 +- cpp/include/cuvs/neighbors/ivf_list.hpp | 6 +- cpp/include/cuvs/neighbors/ivf_list_types.hpp | 4 +- cpp/include/cuvs/neighbors/ivf_pq-ext.cuh | 60 +- cpp/include/cuvs/neighbors/ivf_pq-inl.cuh | 22 +- cpp/include/cuvs/neighbors/ivf_pq_helpers.cuh | 32 +- .../cuvs/neighbors/ivf_pq_serialize.cuh | 8 +- cpp/include/cuvs/neighbors/ivf_pq_types.hpp | 18 +- .../cuvs/neighbors/neighbors_types.hpp | 4 +- cpp/include/cuvs/neighbors/nn_descent.cuh | 16 +- .../cuvs/neighbors/nn_descent_types.hpp | 14 +- cpp/include/cuvs/neighbors/refine-ext.cuh | 18 +- cpp/include/cuvs/neighbors/refine-inl.cuh | 8 +- cpp/include/cuvs/neighbors/sample_filter.cuh | 4 +- .../cuvs/neighbors/sample_filter_types.hpp | 6 +- cpp/include/cuvs/spatial/knn/ann.cuh | 2 +- cpp/include/cuvs/spatial/knn/ann_common.h | 10 +- cpp/include/cuvs/spatial/knn/ann_types.hpp | 4 +- cpp/include/cuvs/spatial/knn/ball_cover.cuh | 10 +- .../cuvs/spatial/knn/ball_cover_types.hpp | 6 +- .../cuvs/spatial/knn/detail/ann_quantized.cuh | 22 +- .../cuvs/spatial/knn/detail/ann_utils.cuh | 2 +- .../cuvs/spatial/knn/detail/ball_cover.cuh | 6 +- .../knn/detail/ball_cover/registers-inl.cuh | 6 +- .../spatial/knn/detail/fused_l2_knn-ext.cuh | 6 +- .../spatial/knn/detail/fused_l2_knn-inl.cuh | 36 +- .../spatial/knn/detail/haversine_distance.cuh | 6 +- .../cuvs/spatial/knn/detail/processing.cuh | 2 +- .../cuvs/spatial/knn/epsilon_neighborhood.cuh | 8 +- cpp/include/cuvs/spatial/knn/ivf_flat.cuh | 10 +- .../cuvs/spatial/knn/ivf_flat_types.hpp | 12 +- cpp/include/cuvs/spatial/knn/ivf_pq.cuh | 10 +- cpp/include/cuvs/spatial/knn/ivf_pq_types.hpp | 12 +- cpp/include/cuvs/spatial/knn/knn.cuh | 8 +- cpp/include/cuvs/spectral/cluster_solvers.cuh | 6 +- .../spectral/cluster_solvers_deprecated.cuh | 6 +- .../stats/detail/batched/silhouette_score.cuh | 6 +- cpp/include/cuvs/stats/detail/scores.cuh | 2 +- .../cuvs/stats/detail/silhouette_score.cuh | 8 +- .../stats/detail/trustworthiness_score.cuh | 8 +- cpp/include/cuvs/stats/silhouette_score.cuh | 12 +- .../cuvs/stats/trustworthiness_score.cuh | 4 +- cpp/include/cuvs_runtime/cluster/kmeans.hpp | 16 +- .../cuvs_runtime/distance/fused_l2_nn.hpp | 2 +- .../distance/pairwise_distance.hpp | 6 +- cpp/include/cuvs_runtime/neighbors/cagra.hpp | 32 +- .../cuvs_runtime/neighbors/ivf_flat.hpp | 28 +- cpp/include/cuvs_runtime/neighbors/ivf_pq.hpp | 24 +- .../cuvs_internal/matrix/select_k.cuh | 2 +- .../cuvs_internal/neighbors/naive_knn.cuh | 24 +- .../cuvs_internal/neighbors/refine_helper.cuh | 8 +- cpp/src/cuvs_runtime/cluster/cluster_cost.cuh | 14 +- .../cluster/cluster_cost_double.cu | 6 +- .../cluster/cluster_cost_float.cu | 6 +- .../cuvs_runtime/cluster/kmeans_fit_double.cu | 10 +- .../cuvs_runtime/cluster/kmeans_fit_float.cu | 10 +- .../cluster/kmeans_init_plus_plus_double.cu | 10 +- .../cluster/kmeans_init_plus_plus_float.cu | 10 +- .../cuvs_runtime/cluster/update_centroids.cuh | 10 +- .../cluster/update_centroids_double.cu | 6 +- .../cluster/update_centroids_float.cu | 6 +- .../cuvs_runtime/distance/fused_l2_min_arg.cu | 10 +- .../distance/pairwise_distance.cu | 16 +- .../matrix/select_k_float_int64_t.cu | 4 +- .../brute_force_knn_int64_t_float.cu | 8 +- cpp/src/cuvs_runtime/neighbors/cagra_build.cu | 34 +- .../cuvs_runtime/neighbors/cagra_search.cu | 12 +- .../cuvs_runtime/neighbors/cagra_serialize.cu | 24 +- .../cuvs_runtime/neighbors/ivf_flat_build.cu | 26 +- .../cuvs_runtime/neighbors/ivf_flat_search.cu | 12 +- .../neighbors/ivf_flat_serialize.cu | 24 +- cpp/src/cuvs_runtime/neighbors/ivfpq_build.cu | 28 +- .../neighbors/ivfpq_deserialize.cu | 12 +- .../neighbors/ivfpq_search_float_int64_t.cu | 12 +- .../neighbors/ivfpq_search_int8_t_int64_t.cu | 12 +- .../neighbors/ivfpq_search_uint8_t_int64_t.cu | 12 +- .../cuvs_runtime/neighbors/ivfpq_serialize.cu | 12 +- .../neighbors/refine_d_int64_t_float.cu | 8 +- .../neighbors/refine_d_int64_t_int8_t.cu | 8 +- .../neighbors/refine_d_int64_t_uint8_t.cu | 8 +- .../neighbors/refine_h_int64_t_float.cu | 8 +- .../neighbors/refine_h_int64_t_int8_t.cu | 8 +- .../neighbors/refine_h_int64_t_uint8_t.cu | 8 +- ...rmat_rectangular_generator_int64_double.cu | 4 +- .../rmat_rectangular_generator_int64_float.cu | 4 +- .../rmat_rectangular_generator_int_double.cu | 4 +- .../rmat_rectangular_generator_int_float.cu | 4 +- .../pairwise_matrix/dispatch_00_generate.py | 40 +- ...patch_canberra_double_double_double_int.cu | 12 +- ...dispatch_canberra_float_float_float_int.cu | 12 +- ...ch_correlation_double_double_double_int.cu | 12 +- ...patch_correlation_float_float_float_int.cu | 12 +- ...ispatch_cosine_double_double_double_int.cu | 14 +- .../dispatch_cosine_float_float_float_int.cu | 14 +- ...ing_unexpanded_double_double_double_int.cu | 12 +- ...amming_unexpanded_float_float_float_int.cu | 12 +- ...inger_expanded_double_double_double_int.cu | 12 +- ...ellinger_expanded_float_float_float_int.cu | 12 +- ...jensen_shannon_double_double_double_int.cu | 12 +- ...ch_jensen_shannon_float_float_float_int.cu | 12 +- ..._kl_divergence_double_double_double_int.cu | 12 +- ...tch_kl_divergence_float_float_float_int.cu | 12 +- .../dispatch_l1_double_double_double_int.cu | 12 +- .../dispatch_l1_float_float_float_int.cu | 12 +- ...ch_l2_expanded_double_double_double_int.cu | 14 +- ...patch_l2_expanded_float_float_float_int.cu | 14 +- ..._l2_unexpanded_double_double_double_int.cu | 12 +- ...tch_l2_unexpanded_float_float_float_int.cu | 12 +- ...dispatch_l_inf_double_double_double_int.cu | 12 +- .../dispatch_l_inf_float_float_float_int.cu | 12 +- ..._lp_unexpanded_double_double_double_int.cu | 12 +- ...tch_lp_unexpanded_float_float_float_int.cu | 12 +- .../detail/pairwise_matrix/dispatch_rbf.cu | 20 +- ...tch_russel_rao_double_double_double_int.cu | 12 +- ...spatch_russel_rao_float_float_float_int.cu | 12 +- cpp/src/distance/distance.cu | 520 ++++++++--------- cpp/src/distance/fused_l2_nn.cu | 6 +- cpp/src/neighbors/ball_cover.cu | 22 +- cpp/src/neighbors/brute_force_00_generate.py | 10 +- .../brute_force_fused_l2_knn_float_int64_t.cu | 6 +- .../neighbors/brute_force_knn_index_float.cu | 14 +- .../brute_force_knn_int64_t_float_int64_t.cu | 6 +- .../brute_force_knn_int64_t_float_uint32_t.cu | 6 +- .../brute_force_knn_int_float_int.cu | 6 +- ...brute_force_knn_uint32_t_float_uint32_t.cu | 6 +- .../cagra/search_multi_cta_00_generate.py | 10 +- ...arch_multi_cta_float_uint32_dim1024_t32.cu | 10 +- ...search_multi_cta_float_uint32_dim128_t8.cu | 10 +- ...earch_multi_cta_float_uint32_dim256_t16.cu | 10 +- ...earch_multi_cta_float_uint32_dim512_t32.cu | 10 +- ...arch_multi_cta_float_uint64_dim1024_t32.cu | 10 +- ...search_multi_cta_float_uint64_dim128_t8.cu | 10 +- ...earch_multi_cta_float_uint64_dim256_t16.cu | 10 +- ...earch_multi_cta_float_uint64_dim512_t32.cu | 10 +- ...earch_multi_cta_int8_uint32_dim1024_t32.cu | 10 +- .../search_multi_cta_int8_uint32_dim128_t8.cu | 10 +- ...search_multi_cta_int8_uint32_dim256_t16.cu | 10 +- ...search_multi_cta_int8_uint32_dim512_t32.cu | 10 +- ...arch_multi_cta_uint8_uint32_dim1024_t32.cu | 10 +- ...search_multi_cta_uint8_uint32_dim128_t8.cu | 10 +- ...earch_multi_cta_uint8_uint32_dim256_t16.cu | 10 +- ...earch_multi_cta_uint8_uint32_dim512_t32.cu | 10 +- .../cagra/search_single_cta_00_generate.py | 10 +- ...rch_single_cta_float_uint32_dim1024_t32.cu | 10 +- ...earch_single_cta_float_uint32_dim128_t8.cu | 10 +- ...arch_single_cta_float_uint32_dim256_t16.cu | 10 +- ...arch_single_cta_float_uint32_dim512_t32.cu | 10 +- ...rch_single_cta_float_uint64_dim1024_t32.cu | 10 +- ...earch_single_cta_float_uint64_dim128_t8.cu | 10 +- ...arch_single_cta_float_uint64_dim256_t16.cu | 10 +- ...arch_single_cta_float_uint64_dim512_t32.cu | 10 +- ...arch_single_cta_int8_uint32_dim1024_t32.cu | 10 +- ...search_single_cta_int8_uint32_dim128_t8.cu | 10 +- ...earch_single_cta_int8_uint32_dim256_t16.cu | 10 +- ...earch_single_cta_int8_uint32_dim512_t32.cu | 10 +- ...rch_single_cta_uint8_uint32_dim1024_t32.cu | 10 +- ...earch_single_cta_uint8_uint32_dim128_t8.cu | 10 +- ...arch_single_cta_uint8_uint32_dim256_t16.cu | 10 +- ...arch_single_cta_uint8_uint32_dim512_t32.cu | 10 +- ...at_interleaved_scan_float_float_int64_t.cu | 12 +- ...interleaved_scan_int8_t_int32_t_int64_t.cu | 12 +- ...terleaved_scan_uint8_t_uint32_t_int64_t.cu | 12 +- cpp/src/neighbors/detail/ivf_flat_search.cu | 14 +- .../ivf_pq_compute_similarity_00_generate.py | 26 +- .../ivf_pq_compute_similarity_float_float.cu | 20 +- ...f_pq_compute_similarity_float_fp8_false.cu | 22 +- ...vf_pq_compute_similarity_float_fp8_true.cu | 22 +- .../ivf_pq_compute_similarity_float_half.cu | 20 +- ...vf_pq_compute_similarity_half_fp8_false.cu | 22 +- ...ivf_pq_compute_similarity_half_fp8_true.cu | 22 +- .../ivf_pq_compute_similarity_half_half.cu | 20 +- .../detail/refine_host_float_float.cpp | 4 +- .../detail/refine_host_int8_t_float.cpp | 4 +- .../detail/refine_host_uint8_t_float.cpp | 4 +- .../detail/selection_faiss_00_generate.py | 4 +- .../detail/selection_faiss_int32_t_float.cu | 4 +- .../detail/selection_faiss_int64_t_double.cu | 4 +- .../detail/selection_faiss_int64_t_half.cu | 4 +- .../detail/selection_faiss_int_double.cu | 4 +- .../detail/selection_faiss_long_float.cu | 4 +- .../detail/selection_faiss_size_t_double.cu | 4 +- .../detail/selection_faiss_size_t_float.cu | 4 +- .../detail/selection_faiss_uint32_t_double.cu | 4 +- .../detail/selection_faiss_uint32_t_float.cu | 4 +- .../detail/selection_faiss_uint32_t_half.cu | 4 +- cpp/src/neighbors/ivf_flat_00_generate.py | 52 +- .../neighbors/ivf_flat_build_float_int64_t.cu | 20 +- .../ivf_flat_build_int8_t_int64_t.cu | 20 +- .../ivf_flat_build_uint8_t_int64_t.cu | 20 +- .../ivf_flat_extend_float_int64_t.cu | 22 +- .../ivf_flat_extend_int8_t_int64_t.cu | 22 +- .../ivf_flat_extend_uint8_t_int64_t.cu | 22 +- .../ivf_flat_search_float_int64_t.cu | 14 +- .../ivf_flat_search_int8_t_int64_t.cu | 14 +- .../ivf_flat_search_uint8_t_int64_t.cu | 14 +- .../neighbors/ivfpq_build_float_int64_t.cu | 14 +- .../neighbors/ivfpq_build_int8_t_int64_t.cu | 14 +- .../neighbors/ivfpq_build_uint8_t_int64_t.cu | 14 +- .../neighbors/ivfpq_extend_float_int64_t.cu | 22 +- .../neighbors/ivfpq_extend_int8_t_int64_t.cu | 22 +- .../neighbors/ivfpq_extend_uint8_t_int64_t.cu | 22 +- .../neighbors/ivfpq_search_float_int64_t.cu | 16 +- .../neighbors/ivfpq_search_int8_t_int64_t.cu | 16 +- .../neighbors/ivfpq_search_uint8_t_int64_t.cu | 16 +- cpp/src/neighbors/refine_00_generate.py | 10 +- cpp/src/neighbors/refine_float_float.cu | 10 +- cpp/src/neighbors/refine_int8_t_float.cu | 10 +- cpp/src/neighbors/refine_uint8_t_float.cu | 10 +- .../knn/detail/fused_l2_knn_int32_t_float.cu | 4 +- .../knn/detail/fused_l2_knn_int64_t_float.cu | 4 +- .../knn/detail/fused_l2_knn_uint32_t_float.cu | 4 +- cpp/template/src/cagra_example.cu | 2 +- cpp/template/src/ivf_flat_example.cu | 8 +- cpp/test/cluster/kmeans_balanced.cu | 4 +- cpp/test/cluster/linkage.cu | 8 +- cpp/test/distance/dist_adj.cu | 8 +- cpp/test/distance/dist_adj.cuh | 24 +- .../distance/dist_adj_distance_instance.cu | 18 +- cpp/test/distance/dist_adj_threshold.cuh | 4 +- cpp/test/distance/dist_canberra.cu | 4 +- cpp/test/distance/dist_correlation.cu | 6 +- cpp/test/distance/dist_cos.cu | 6 +- cpp/test/distance/dist_hamming.cu | 4 +- cpp/test/distance/dist_hellinger.cu | 4 +- cpp/test/distance/dist_inner_product.cu | 4 +- cpp/test/distance/dist_jensen_shannon.cu | 4 +- cpp/test/distance/dist_kl_divergence.cu | 4 +- cpp/test/distance/dist_l1.cu | 4 +- cpp/test/distance/dist_l2_exp.cu | 6 +- cpp/test/distance/dist_l2_sqrt_exp.cu | 4 +- cpp/test/distance/dist_l2_unexp.cu | 4 +- cpp/test/distance/dist_l_inf.cu | 4 +- cpp/test/distance/dist_lp_unexp.cu | 4 +- cpp/test/distance/dist_russell_rao.cu | 4 +- cpp/test/distance/distance_base.cuh | 86 +-- cpp/test/distance/fused_l2_nn.cu | 4 +- cpp/test/distance/gram.cu | 8 +- cpp/test/distance/gram_base.cuh | 4 +- cpp/test/distance/masked_nn.cu | 18 +- .../distance/masked_nn_compress_to_bits.cu | 10 +- cpp/test/ext_headers/00_generate.py | 24 +- ...istance_detail_pairwise_matrix_dispatch.cu | 2 +- .../ext_headers/raft_distance_distance.cu | 2 +- .../ext_headers/raft_distance_fused_l2_nn.cu | 2 +- .../ext_headers/raft_neighbors_ball_cover.cu | 2 +- .../ext_headers/raft_neighbors_brute_force.cu | 2 +- ...ghbors_detail_ivf_flat_interleaved_scan.cu | 2 +- .../raft_neighbors_detail_ivf_flat_search.cu | 2 +- ...ghbors_detail_ivf_pq_compute_similarity.cu | 2 +- .../raft_neighbors_detail_selection_faiss.cu | 2 +- .../ext_headers/raft_neighbors_ivf_flat.cu | 2 +- cpp/test/ext_headers/raft_neighbors_ivf_pq.cu | 2 +- cpp/test/ext_headers/raft_neighbors_refine.cu | 2 +- cpp/test/neighbors/ann_cagra.cuh | 30 +- .../ann_cagra/search_kernel_uint64_t.cuh | 22 +- .../neighbors/ann_cagra/test_float_int64_t.cu | 4 +- .../ann_cagra/test_float_uint32_t.cu | 4 +- .../ann_cagra/test_int8_t_uint32_t.cu | 4 +- .../ann_cagra/test_uint8_t_uint32_t.cu | 4 +- cpp/test/neighbors/ann_ivf_flat.cuh | 98 ++-- .../ann_ivf_flat/test_filter_float_int64_t.cu | 4 +- .../ann_ivf_flat/test_float_int64_t.cu | 4 +- .../ann_ivf_flat/test_int8_t_int64_t.cu | 4 +- .../ann_ivf_flat/test_uint8_t_int64_t.cu | 4 +- cpp/test/neighbors/ann_ivf_pq.cuh | 16 +- .../ann_ivf_pq/test_filter_float_int64_t.cu | 4 +- .../ann_ivf_pq/test_filter_int8_t_int64_t.cu | 4 +- .../ann_ivf_pq/test_float_int64_t.cu | 4 +- .../ann_ivf_pq/test_float_uint32_t.cu | 4 +- .../ann_ivf_pq/test_int8_t_int64_t.cu | 4 +- .../ann_ivf_pq/test_uint8_t_int64_t.cu | 4 +- cpp/test/neighbors/ann_nn_descent.cuh | 10 +- .../ann_nn_descent/test_float_uint32_t.cu | 4 +- .../ann_nn_descent/test_int8_t_uint32_t.cu | 4 +- .../ann_nn_descent/test_uint8_t_uint32_t.cu | 4 +- cpp/test/neighbors/ann_utils.cuh | 52 +- cpp/test/neighbors/ball_cover.cu | 38 +- cpp/test/neighbors/fused_l2_knn.cu | 42 +- cpp/test/neighbors/haversine.cu | 2 +- cpp/test/neighbors/knn.cu | 10 +- cpp/test/neighbors/refine.cu | 18 +- cpp/test/neighbors/selection.cu | 6 +- cpp/test/neighbors/tiled_knn.cu | 78 +-- cpp/test/sparse/dist_coo_spmv.cu | 28 +- cpp/test/sparse/distance.cu | 44 +- cpp/test/sparse/gram.cu | 10 +- cpp/test/sparse/neighbors/brute_force.cu | 6 +- .../sparse/neighbors/cross_component_nn.cu | 4 +- cpp/test/sparse/neighbors/knn_graph.cu | 2 +- cpp/test/stats/neighborhood_recall.cu | 12 +- cpp/test/stats/silhouette_score.cu | 20 +- cpp/test/stats/trustworthiness.cu | 4 +- 482 files changed, 3631 insertions(+), 3702 deletions(-) delete mode 100644 cpp/include/cuvs/distance/specializations.cuh delete mode 100644 cpp/include/cuvs/distance/specializations/distance.cuh delete mode 100644 cpp/include/cuvs/distance/specializations/fused_l2_nn_min.cuh diff --git a/cpp/bench/ann/src/common/ann_types.hpp b/cpp/bench/ann/src/common/ann_types.hpp index e964a81ef..8d745b9d2 100644 --- a/cpp/bench/ann/src/common/ann_types.hpp +++ b/cpp/bench/ann/src/common/ann_types.hpp @@ -22,7 +22,7 @@ #include #include -namespace raft::bench::ann { +namespace cuvs::bench { enum Objective { THROUGHPUT, // See how many vectors we can push through @@ -43,9 +43,9 @@ enum class Metric { inline auto parse_metric(const std::string& metric_str) -> Metric { if (metric_str == "inner_product") { - return raft::bench::ann::Metric::kInnerProduct; + return cuvs::bench::Metric::kInnerProduct; } else if (metric_str == "euclidean") { - return raft::bench::ann::Metric::kEuclidean; + return cuvs::bench::Metric::kEuclidean; } else { throw std::runtime_error("invalid metric: '" + metric_str + "'"); } @@ -127,12 +127,11 @@ class ANN : public AnnBase { virtual void set_search_dataset(const T* /*dataset*/, size_t /*nrow*/){}; }; -} // namespace raft::bench::ann +} // namespace cuvs::bench -#define REGISTER_ALGO_INSTANCE(DataT) \ - template auto raft::bench::ann::create_algo( \ - const std::string&, const std::string&, int, const nlohmann::json&, const std::vector&) \ - ->std::unique_ptr>; \ - template auto raft::bench::ann::create_search_param(const std::string&, \ - const nlohmann::json&) \ - ->std::unique_ptr::AnnSearchParam>; +#define REGISTER_ALGO_INSTANCE(DataT) \ + template auto cuvs::bench::create_algo( \ + const std::string&, const std::string&, int, const nlohmann::json&, const std::vector&) \ + ->std::unique_ptr>; \ + template auto cuvs::bench::create_search_param(const std::string&, const nlohmann::json&) \ + ->std::unique_ptr::AnnSearchParam>; diff --git a/cpp/bench/ann/src/common/benchmark.cpp b/cpp/bench/ann/src/common/benchmark.cpp index 6424a3647..e3db4c3fb 100644 --- a/cpp/bench/ann/src/common/benchmark.cpp +++ b/cpp/bench/ann/src/common/benchmark.cpp @@ -26,7 +26,7 @@ #include #include -namespace raft::bench::ann { +namespace cuvs::bench { struct lib_handle { void* handle{nullptr}; @@ -72,7 +72,7 @@ auto create_algo(const std::string& algo, const std::string& distance, int dim, const nlohmann::json& conf, - const std::vector& dev_list) -> std::unique_ptr> + const std::vector& dev_list) -> std::unique_ptr> { static auto fname = get_fun_name(reinterpret_cast(&create_algo)); auto handle = load_lib(algo); @@ -85,7 +85,7 @@ auto create_algo(const std::string& algo, } template -std::unique_ptr::AnnSearchParam> create_search_param( +std::unique_ptr::AnnSearchParam> create_search_param( const std::string& algo, const nlohmann::json& conf) { static auto fname = get_fun_name(reinterpret_cast(&create_search_param)); @@ -98,7 +98,7 @@ std::unique_ptr::AnnSearchParam> create_search return fun(algo, conf); } -}; // namespace raft::bench::ann +}; // namespace cuvs::bench REGISTER_ALGO_INSTANCE(float); REGISTER_ALGO_INSTANCE(std::int8_t); @@ -106,4 +106,4 @@ REGISTER_ALGO_INSTANCE(std::uint8_t); #include "benchmark.hpp" -int main(int argc, char** argv) { return raft::bench::ann::run_main(argc, argv); } +int main(int argc, char** argv) { return cuvs::bench::run_main(argc, argv); } diff --git a/cpp/bench/ann/src/common/benchmark.hpp b/cpp/bench/ann/src/common/benchmark.hpp index a2e77323c..5e88491b4 100644 --- a/cpp/bench/ann/src/common/benchmark.hpp +++ b/cpp/bench/ann/src/common/benchmark.hpp @@ -38,7 +38,7 @@ #include #include #include -namespace raft::bench::ann { +namespace cuvs::bench { std::mutex init_mutex; std::condition_variable cond_var; @@ -711,4 +711,4 @@ inline auto run_main(int argc, char** argv) -> int current_algo.reset(); return 0; } -}; // namespace raft::bench::ann +}; // namespace cuvs::bench diff --git a/cpp/bench/ann/src/common/conf.hpp b/cpp/bench/ann/src/common/conf.hpp index 405b00a74..9b635790e 100644 --- a/cpp/bench/ann/src/common/conf.hpp +++ b/cpp/bench/ann/src/common/conf.hpp @@ -26,7 +26,7 @@ #define JSON_DIAGNOSTICS 1 #include -namespace raft::bench::ann { +namespace cuvs::bench { class Configuration { public: @@ -153,4 +153,4 @@ class Configuration { std::vector indices_; }; -} // namespace raft::bench::ann +} // namespace cuvs::bench diff --git a/cpp/bench/ann/src/common/cuda_stub.hpp b/cpp/bench/ann/src/common/cuda_stub.hpp index d4d02d5f3..c05b6108e 100644 --- a/cpp/bench/ann/src/common/cuda_stub.hpp +++ b/cpp/bench/ann/src/common/cuda_stub.hpp @@ -43,7 +43,7 @@ typedef void* cudaStream_t; typedef void* cudaEvent_t; #endif -namespace raft::bench::ann { +namespace cuvs::bench { struct cuda_lib_handle { void* handle{nullptr}; @@ -231,4 +231,4 @@ RAFT_DECLARE_CUDART(cudaGetDeviceProperties); #undef RAFT_DECLARE_CUDART #endif -}; // namespace raft::bench::ann +}; // namespace cuvs::bench diff --git a/cpp/bench/ann/src/common/dataset.hpp b/cpp/bench/ann/src/common/dataset.hpp index ccc5915b3..0b55966e3 100644 --- a/cpp/bench/ann/src/common/dataset.hpp +++ b/cpp/bench/ann/src/common/dataset.hpp @@ -36,7 +36,7 @@ typedef uint16_t half; #include #include -namespace raft::bench::ann { +namespace cuvs::bench { // http://big-ann-benchmarks.com/index.html: // binary format that starts with 8 bytes of data consisting of num_points(uint32_t) @@ -498,4 +498,4 @@ void BinDataset::map_base_set_() const this->mapped_base_set_ = base_file_.map(); } -} // namespace raft::bench::ann +} // namespace cuvs::bench diff --git a/cpp/bench/ann/src/common/util.hpp b/cpp/bench/ann/src/common/util.hpp index 544aa789a..47d0c3a28 100644 --- a/cpp/bench/ann/src/common/util.hpp +++ b/cpp/bench/ann/src/common/util.hpp @@ -36,7 +36,7 @@ #include #include -namespace raft::bench::ann { +namespace cuvs::bench { template struct buf { @@ -345,4 +345,4 @@ void log_error(Ts&&... vs) log_("error", std::forward(vs)...); } -} // namespace raft::bench::ann +} // namespace cuvs::bench diff --git a/cpp/bench/ann/src/faiss/faiss_cpu_benchmark.cpp b/cpp/bench/ann/src/faiss/faiss_cpu_benchmark.cpp index 82122e8fc..34eb1e458 100644 --- a/cpp/bench/ann/src/faiss/faiss_cpu_benchmark.cpp +++ b/cpp/bench/ann/src/faiss/faiss_cpu_benchmark.cpp @@ -27,11 +27,11 @@ #define JSON_DIAGNOSTICS 1 #include -namespace raft::bench::ann { +namespace cuvs::bench { template void parse_base_build_param(const nlohmann::json& conf, - typename raft::bench::ann::FaissCpu::BuildParam& param) + typename cuvs::bench::FaissCpu::BuildParam& param) { param.nlist = conf.at("nlist"); if (conf.contains("ratio")) { param.ratio = conf.at("ratio"); } @@ -39,14 +39,14 @@ void parse_base_build_param(const nlohmann::json& conf, template void parse_build_param(const nlohmann::json& conf, - typename raft::bench::ann::FaissCpuIVFFlat::BuildParam& param) + typename cuvs::bench::FaissCpuIVFFlat::BuildParam& param) { parse_base_build_param(conf, param); } template void parse_build_param(const nlohmann::json& conf, - typename raft::bench::ann::FaissCpuIVFPQ::BuildParam& param) + typename cuvs::bench::FaissCpuIVFPQ::BuildParam& param) { parse_base_build_param(conf, param); param.M = conf.at("M"); @@ -64,7 +64,7 @@ void parse_build_param(const nlohmann::json& conf, template void parse_build_param(const nlohmann::json& conf, - typename raft::bench::ann::FaissCpuIVFSQ::BuildParam& param) + typename cuvs::bench::FaissCpuIVFSQ::BuildParam& param) { parse_base_build_param(conf, param); param.quantizer_type = conf.at("quantizer_type"); @@ -72,7 +72,7 @@ void parse_build_param(const nlohmann::json& conf, template void parse_search_param(const nlohmann::json& conf, - typename raft::bench::ann::FaissCpu::SearchParam& param) + typename cuvs::bench::FaissCpu::SearchParam& param) { param.nprobe = conf.at("nprobe"); if (conf.contains("refine_ratio")) { param.refine_ratio = conf.at("refine_ratio"); } @@ -80,9 +80,9 @@ void parse_search_param(const nlohmann::json& conf, } template class Algo> -std::unique_ptr> make_algo(raft::bench::ann::Metric metric, - int dim, - const nlohmann::json& conf) +std::unique_ptr> make_algo(cuvs::bench::Metric metric, + int dim, + const nlohmann::json& conf) { typename Algo::BuildParam param; parse_build_param(conf, param); @@ -90,10 +90,10 @@ std::unique_ptr> make_algo(raft::bench::ann::Metric met } template class Algo> -std::unique_ptr> make_algo(raft::bench::ann::Metric metric, - int dim, - const nlohmann::json& conf, - const std::vector& dev_list) +std::unique_ptr> make_algo(cuvs::bench::Metric metric, + int dim, + const nlohmann::json& conf, + const std::vector& dev_list) { typename Algo::BuildParam param; parse_build_param(conf, param); @@ -103,27 +103,27 @@ std::unique_ptr> make_algo(raft::bench::ann::Metric met } template -std::unique_ptr> create_algo(const std::string& algo, - const std::string& distance, - int dim, - const nlohmann::json& conf, - const std::vector& dev_list) +std::unique_ptr> create_algo(const std::string& algo, + const std::string& distance, + int dim, + const nlohmann::json& conf, + const std::vector& dev_list) { // stop compiler warning; not all algorithms support multi-GPU so it may not be used (void)dev_list; - std::unique_ptr> ann; + std::unique_ptr> ann; if constexpr (std::is_same_v) { - raft::bench::ann::Metric metric = parse_metric(distance); + cuvs::bench::Metric metric = parse_metric(distance); if (algo == "faiss_cpu_ivf_flat") { - ann = make_algo(metric, dim, conf, dev_list); + ann = make_algo(metric, dim, conf, dev_list); } else if (algo == "faiss_cpu_ivf_pq") { - ann = make_algo(metric, dim, conf); + ann = make_algo(metric, dim, conf); } else if (algo == "faiss_cpu_ivf_sq") { - ann = make_algo(metric, dim, conf); + ann = make_algo(metric, dim, conf); } else if (algo == "faiss_cpu_flat") { - ann = std::make_unique>(metric, dim); + ann = std::make_unique>(metric, dim); } } @@ -135,22 +135,22 @@ std::unique_ptr> create_algo(const std::string& algo, } template -std::unique_ptr::AnnSearchParam> create_search_param( +std::unique_ptr::AnnSearchParam> create_search_param( const std::string& algo, const nlohmann::json& conf) { if (algo == "faiss_cpu_ivf_flat" || algo == "faiss_cpu_ivf_pq" || algo == "faiss_cpu_ivf_sq") { - auto param = std::make_unique::SearchParam>(); + auto param = std::make_unique::SearchParam>(); parse_search_param(conf, *param); return param; } else if (algo == "faiss_cpu_flat") { - auto param = std::make_unique::AnnSearchParam>(); + auto param = std::make_unique::AnnSearchParam>(); return param; } // else throw std::runtime_error("invalid algo: '" + algo + "'"); } -} // namespace raft::bench::ann +} // namespace cuvs::bench REGISTER_ALGO_INSTANCE(float); REGISTER_ALGO_INSTANCE(std::int8_t); @@ -158,5 +158,5 @@ REGISTER_ALGO_INSTANCE(std::uint8_t); #ifdef CUVS_BENCH_BUILD_MAIN #include "../common/benchmark.hpp" -int main(int argc, char** argv) { return raft::bench::ann::run_main(argc, argv); } +int main(int argc, char** argv) { return cuvs::bench::run_main(argc, argv); } #endif diff --git a/cpp/bench/ann/src/faiss/faiss_cpu_wrapper.h b/cpp/bench/ann/src/faiss/faiss_cpu_wrapper.h index 755fe9f19..18c2e7673 100644 --- a/cpp/bench/ann/src/faiss/faiss_cpu_wrapper.h +++ b/cpp/bench/ann/src/faiss/faiss_cpu_wrapper.h @@ -35,11 +35,11 @@ namespace { -faiss::MetricType parse_metric_type(raft::bench::ann::Metric metric) +faiss::MetricType parse_metric_type(cuvs::bench::Metric metric) { - if (metric == raft::bench::ann::Metric::kInnerProduct) { + if (metric == cuvs::bench::Metric::kInnerProduct) { return faiss::METRIC_INNER_PRODUCT; - } else if (metric == raft::bench::ann::Metric::kEuclidean) { + } else if (metric == cuvs::bench::Metric::kEuclidean) { return faiss::METRIC_L2; } else { throw std::runtime_error("faiss supports only metric type of inner product and L2"); @@ -47,7 +47,7 @@ faiss::MetricType parse_metric_type(raft::bench::ann::Metric metric) } } // namespace -namespace raft::bench::ann { +namespace cuvs::bench { template class FaissCpu : public ANN { @@ -310,4 +310,4 @@ class FaissCpuFlat : public FaissCpu { void load(const std::string& file) override { this->template load_(file); } }; -} // namespace raft::bench::ann +} // namespace cuvs::bench diff --git a/cpp/bench/ann/src/faiss/faiss_gpu_benchmark.cu b/cpp/bench/ann/src/faiss/faiss_gpu_benchmark.cu index 77ad2c102..4fd7749b0 100644 --- a/cpp/bench/ann/src/faiss/faiss_gpu_benchmark.cu +++ b/cpp/bench/ann/src/faiss/faiss_gpu_benchmark.cu @@ -28,11 +28,11 @@ #define JSON_DIAGNOSTICS 1 #include -namespace raft::bench::ann { +namespace cuvs::bench { template void parse_base_build_param(const nlohmann::json& conf, - typename raft::bench::ann::FaissGpu::BuildParam& param) + typename cuvs::bench::FaissGpu::BuildParam& param) { param.nlist = conf.at("nlist"); if (conf.contains("ratio")) { param.ratio = conf.at("ratio"); } @@ -40,14 +40,14 @@ void parse_base_build_param(const nlohmann::json& conf, template void parse_build_param(const nlohmann::json& conf, - typename raft::bench::ann::FaissGpuIVFFlat::BuildParam& param) + typename cuvs::bench::FaissGpuIVFFlat::BuildParam& param) { parse_base_build_param(conf, param); } template void parse_build_param(const nlohmann::json& conf, - typename raft::bench::ann::FaissGpuIVFPQ::BuildParam& param) + typename cuvs::bench::FaissGpuIVFPQ::BuildParam& param) { parse_base_build_param(conf, param); param.M = conf.at("M"); @@ -65,7 +65,7 @@ void parse_build_param(const nlohmann::json& conf, template void parse_build_param(const nlohmann::json& conf, - typename raft::bench::ann::FaissGpuIVFSQ::BuildParam& param) + typename cuvs::bench::FaissGpuIVFSQ::BuildParam& param) { parse_base_build_param(conf, param); param.quantizer_type = conf.at("quantizer_type"); @@ -73,16 +73,16 @@ void parse_build_param(const nlohmann::json& conf, template void parse_search_param(const nlohmann::json& conf, - typename raft::bench::ann::FaissGpu::SearchParam& param) + typename cuvs::bench::FaissGpu::SearchParam& param) { param.nprobe = conf.at("nprobe"); if (conf.contains("refine_ratio")) { param.refine_ratio = conf.at("refine_ratio"); } } template class Algo> -std::unique_ptr> make_algo(raft::bench::ann::Metric metric, - int dim, - const nlohmann::json& conf) +std::unique_ptr> make_algo(cuvs::bench::Metric metric, + int dim, + const nlohmann::json& conf) { typename Algo::BuildParam param; parse_build_param(conf, param); @@ -90,10 +90,10 @@ std::unique_ptr> make_algo(raft::bench::ann::Metric met } template class Algo> -std::unique_ptr> make_algo(raft::bench::ann::Metric metric, - int dim, - const nlohmann::json& conf, - const std::vector& dev_list) +std::unique_ptr> make_algo(cuvs::bench::Metric metric, + int dim, + const nlohmann::json& conf, + const std::vector& dev_list) { typename Algo::BuildParam param; parse_build_param(conf, param); @@ -103,27 +103,27 @@ std::unique_ptr> make_algo(raft::bench::ann::Metric met } template -std::unique_ptr> create_algo(const std::string& algo, - const std::string& distance, - int dim, - const nlohmann::json& conf, - const std::vector& dev_list) +std::unique_ptr> create_algo(const std::string& algo, + const std::string& distance, + int dim, + const nlohmann::json& conf, + const std::vector& dev_list) { // stop compiler warning; not all algorithms support multi-GPU so it may not be used (void)dev_list; - std::unique_ptr> ann; + std::unique_ptr> ann; if constexpr (std::is_same_v) { - raft::bench::ann::Metric metric = parse_metric(distance); + cuvs::bench::Metric metric = parse_metric(distance); if (algo == "faiss_gpu_ivf_flat") { - ann = make_algo(metric, dim, conf, dev_list); + ann = make_algo(metric, dim, conf, dev_list); } else if (algo == "faiss_gpu_ivf_pq") { - ann = make_algo(metric, dim, conf); + ann = make_algo(metric, dim, conf); } else if (algo == "faiss_gpu_ivf_sq") { - ann = make_algo(metric, dim, conf); + ann = make_algo(metric, dim, conf); } else if (algo == "faiss_gpu_flat") { - ann = std::make_unique>(metric, dim); + ann = std::make_unique>(metric, dim); } } @@ -135,22 +135,22 @@ std::unique_ptr> create_algo(const std::string& algo, } template -std::unique_ptr::AnnSearchParam> create_search_param( +std::unique_ptr::AnnSearchParam> create_search_param( const std::string& algo, const nlohmann::json& conf) { if (algo == "faiss_gpu_ivf_flat" || algo == "faiss_gpu_ivf_pq" || algo == "faiss_gpu_ivf_sq") { - auto param = std::make_unique::SearchParam>(); + auto param = std::make_unique::SearchParam>(); parse_search_param(conf, *param); return param; } else if (algo == "faiss_gpu_flat") { - auto param = std::make_unique::AnnSearchParam>(); + auto param = std::make_unique::AnnSearchParam>(); return param; } // else throw std::runtime_error("invalid algo: '" + algo + "'"); } -} // namespace raft::bench::ann +} // namespace cuvs::bench REGISTER_ALGO_INSTANCE(float); REGISTER_ALGO_INSTANCE(std::int8_t); @@ -158,5 +158,5 @@ REGISTER_ALGO_INSTANCE(std::uint8_t); #ifdef CUVS_BENCH_BUILD_MAIN #include "../common/benchmark.hpp" -int main(int argc, char** argv) { return raft::bench::ann::run_main(argc, argv); } +int main(int argc, char** argv) { return cuvs::bench::run_main(argc, argv); } #endif diff --git a/cpp/bench/ann/src/faiss/faiss_gpu_wrapper.h b/cpp/bench/ann/src/faiss/faiss_gpu_wrapper.h index 4f13ff8a4..23e8e182a 100644 --- a/cpp/bench/ann/src/faiss/faiss_gpu_wrapper.h +++ b/cpp/bench/ann/src/faiss/faiss_gpu_wrapper.h @@ -46,11 +46,11 @@ namespace { -faiss::MetricType parse_metric_type(raft::bench::ann::Metric metric) +faiss::MetricType parse_metric_type(cuvs::bench::Metric metric) { - if (metric == raft::bench::ann::Metric::kInnerProduct) { + if (metric == cuvs::bench::Metric::kInnerProduct) { return faiss::METRIC_INNER_PRODUCT; - } else if (metric == raft::bench::ann::Metric::kEuclidean) { + } else if (metric == cuvs::bench::Metric::kEuclidean) { return faiss::METRIC_L2; } else { throw std::runtime_error("faiss supports only metric type of inner product and L2"); @@ -78,7 +78,7 @@ class OmpSingleThreadScope { } // namespace -namespace raft::bench::ann { +namespace cuvs::bench { template class FaissGpu : public ANN { @@ -430,6 +430,6 @@ class FaissGpuFlat : public FaissGpu { } }; -} // namespace raft::bench::ann +} // namespace cuvs::bench #endif \ No newline at end of file diff --git a/cpp/bench/ann/src/ggnn/ggnn_benchmark.cu b/cpp/bench/ann/src/ggnn/ggnn_benchmark.cu index 01fd1359c..96a1ce332 100644 --- a/cpp/bench/ann/src/ggnn/ggnn_benchmark.cu +++ b/cpp/bench/ann/src/ggnn/ggnn_benchmark.cu @@ -27,11 +27,10 @@ #define JSON_DIAGNOSTICS 1 #include -namespace raft::bench::ann { +namespace cuvs::bench { template -void parse_build_param(const nlohmann::json& conf, - typename raft::bench::ann::Ggnn::BuildParam& param) +void parse_build_param(const nlohmann::json& conf, typename cuvs::bench::Ggnn::BuildParam& param) { param.k = conf.at("k"); @@ -46,7 +45,7 @@ void parse_build_param(const nlohmann::json& conf, template void parse_search_param(const nlohmann::json& conf, - typename raft::bench::ann::Ggnn::SearchParam& param) + typename cuvs::bench::Ggnn::SearchParam& param) { param.tau = conf.at("tau"); @@ -57,9 +56,9 @@ void parse_search_param(const nlohmann::json& conf, } template class Algo> -std::unique_ptr> make_algo(raft::bench::ann::Metric metric, - int dim, - const nlohmann::json& conf) +std::unique_ptr> make_algo(cuvs::bench::Metric metric, + int dim, + const nlohmann::json& conf) { typename Algo::BuildParam param; parse_build_param(conf, param); @@ -67,10 +66,10 @@ std::unique_ptr> make_algo(raft::bench::ann::Metric met } template class Algo> -std::unique_ptr> make_algo(raft::bench::ann::Metric metric, - int dim, - const nlohmann::json& conf, - const std::vector& dev_list) +std::unique_ptr> make_algo(cuvs::bench::Metric metric, + int dim, + const nlohmann::json& conf, + const std::vector& dev_list) { typename Algo::BuildParam param; parse_build_param(conf, param); @@ -80,34 +79,34 @@ std::unique_ptr> make_algo(raft::bench::ann::Metric met } template -std::unique_ptr> create_algo(const std::string& algo, - const std::string& distance, - int dim, - const nlohmann::json& conf, - const std::vector& dev_list) +std::unique_ptr> create_algo(const std::string& algo, + const std::string& distance, + int dim, + const nlohmann::json& conf, + const std::vector& dev_list) { // stop compiler warning; not all algorithms support multi-GPU so it may not be used (void)dev_list; - raft::bench::ann::Metric metric = parse_metric(distance); - std::unique_ptr> ann; + cuvs::bench::Metric metric = parse_metric(distance); + std::unique_ptr> ann; if constexpr (std::is_same_v) {} if constexpr (std::is_same_v) {} - if (algo == "ggnn") { ann = make_algo(metric, dim, conf); } + if (algo == "ggnn") { ann = make_algo(metric, dim, conf); } if (!ann) { throw std::runtime_error("invalid algo: '" + algo + "'"); } return ann; } template -std::unique_ptr::AnnSearchParam> create_search_param( +std::unique_ptr::AnnSearchParam> create_search_param( const std::string& algo, const nlohmann::json& conf) { if (algo == "ggnn") { - auto param = std::make_unique::SearchParam>(); + auto param = std::make_unique::SearchParam>(); parse_search_param(conf, *param); return param; } @@ -115,7 +114,7 @@ std::unique_ptr::AnnSearchParam> create_search throw std::runtime_error("invalid algo: '" + algo + "'"); } -} // namespace raft::bench::ann +} // namespace cuvs::bench REGISTER_ALGO_INSTANCE(float); REGISTER_ALGO_INSTANCE(std::int8_t); @@ -123,5 +122,5 @@ REGISTER_ALGO_INSTANCE(std::uint8_t); #ifdef CUVS_BENCH_BUILD_MAIN #include "../common/benchmark.hpp" -int main(int argc, char** argv) { return raft::bench::ann::run_main(argc, argv); } +int main(int argc, char** argv) { return cuvs::bench::run_main(argc, argv); } #endif diff --git a/cpp/bench/ann/src/ggnn/ggnn_wrapper.cuh b/cpp/bench/ann/src/ggnn/ggnn_wrapper.cuh index 664ec511d..16242136d 100644 --- a/cpp/bench/ann/src/ggnn/ggnn_wrapper.cuh +++ b/cpp/bench/ann/src/ggnn/ggnn_wrapper.cuh @@ -24,7 +24,7 @@ #include #include -namespace raft::bench::ann { +namespace cuvs::bench { template class GgnnImpl; @@ -291,4 +291,4 @@ void GgnnImpl::load(const std::string& file) RAFT_CUDA_TRY(cudaStreamSynchronize(ggnn_device.stream)); } -} // namespace raft::bench::ann +} // namespace cuvs::bench diff --git a/cpp/bench/ann/src/hnswlib/hnswlib_benchmark.cpp b/cpp/bench/ann/src/hnswlib/hnswlib_benchmark.cpp index 8e87c64d6..9c5301a1b 100644 --- a/cpp/bench/ann/src/hnswlib/hnswlib_benchmark.cpp +++ b/cpp/bench/ann/src/hnswlib/hnswlib_benchmark.cpp @@ -28,11 +28,11 @@ #define JSON_DIAGNOSTICS 1 #include -namespace raft::bench::ann { +namespace cuvs::bench { template void parse_build_param(const nlohmann::json& conf, - typename raft::bench::ann::HnswLib::BuildParam& param) + typename cuvs::bench::HnswLib::BuildParam& param) { param.ef_construction = conf.at("efConstruction"); param.M = conf.at("M"); @@ -41,16 +41,16 @@ void parse_build_param(const nlohmann::json& conf, template void parse_search_param(const nlohmann::json& conf, - typename raft::bench::ann::HnswLib::SearchParam& param) + typename cuvs::bench::HnswLib::SearchParam& param) { param.ef = conf.at("ef"); if (conf.contains("numThreads")) { param.num_threads = conf.at("numThreads"); } } template class Algo> -std::unique_ptr> make_algo(raft::bench::ann::Metric metric, - int dim, - const nlohmann::json& conf) +std::unique_ptr> make_algo(cuvs::bench::Metric metric, + int dim, + const nlohmann::json& conf) { typename Algo::BuildParam param; parse_build_param(conf, param); @@ -58,10 +58,10 @@ std::unique_ptr> make_algo(raft::bench::ann::Metric met } template class Algo> -std::unique_ptr> make_algo(raft::bench::ann::Metric metric, - int dim, - const nlohmann::json& conf, - const std::vector& dev_list) +std::unique_ptr> make_algo(cuvs::bench::Metric metric, + int dim, + const nlohmann::json& conf, + const std::vector& dev_list) { typename Algo::BuildParam param; parse_build_param(conf, param); @@ -71,24 +71,24 @@ std::unique_ptr> make_algo(raft::bench::ann::Metric met } template -std::unique_ptr> create_algo(const std::string& algo, - const std::string& distance, - int dim, - const nlohmann::json& conf, - const std::vector& dev_list) +std::unique_ptr> create_algo(const std::string& algo, + const std::string& distance, + int dim, + const nlohmann::json& conf, + const std::vector& dev_list) { // stop compiler warning; not all algorithms support multi-GPU so it may not be used (void)dev_list; - raft::bench::ann::Metric metric = parse_metric(distance); - std::unique_ptr> ann; + cuvs::bench::Metric metric = parse_metric(distance); + std::unique_ptr> ann; if constexpr (std::is_same_v) { - if (algo == "hnswlib") { ann = make_algo(metric, dim, conf); } + if (algo == "hnswlib") { ann = make_algo(metric, dim, conf); } } if constexpr (std::is_same_v) { - if (algo == "hnswlib") { ann = make_algo(metric, dim, conf); } + if (algo == "hnswlib") { ann = make_algo(metric, dim, conf); } } if (!ann) { throw std::runtime_error("invalid algo: '" + algo + "'"); } @@ -96,11 +96,11 @@ std::unique_ptr> create_algo(const std::string& algo, } template -std::unique_ptr::AnnSearchParam> create_search_param( +std::unique_ptr::AnnSearchParam> create_search_param( const std::string& algo, const nlohmann::json& conf) { if (algo == "hnswlib") { - auto param = std::make_unique::SearchParam>(); + auto param = std::make_unique::SearchParam>(); parse_search_param(conf, *param); return param; } @@ -108,7 +108,7 @@ std::unique_ptr::AnnSearchParam> create_search throw std::runtime_error("invalid algo: '" + algo + "'"); } -}; // namespace raft::bench::ann +}; // namespace cuvs::bench REGISTER_ALGO_INSTANCE(float); REGISTER_ALGO_INSTANCE(std::int8_t); @@ -116,5 +116,5 @@ REGISTER_ALGO_INSTANCE(std::uint8_t); #ifdef CUVS_BENCH_BUILD_MAIN #include "../common/benchmark.hpp" -int main(int argc, char** argv) { return raft::bench::ann::run_main(argc, argv); } +int main(int argc, char** argv) { return cuvs::bench::run_main(argc, argv); } #endif diff --git a/cpp/bench/ann/src/hnswlib/hnswlib_wrapper.h b/cpp/bench/ann/src/hnswlib/hnswlib_wrapper.h index 921d72dec..24c1b6d4b 100644 --- a/cpp/bench/ann/src/hnswlib/hnswlib_wrapper.h +++ b/cpp/bench/ann/src/hnswlib/hnswlib_wrapper.h @@ -35,7 +35,7 @@ #include "../common/thread_pool.hpp" #include -namespace raft::bench::ann { +namespace cuvs::bench { template struct hnsw_dist_t { @@ -227,4 +227,4 @@ void HnswLib::get_search_knn_results_(const T* query, } } -}; // namespace raft::bench::ann +}; // namespace cuvs::bench diff --git a/cpp/bench/ann/src/raft/raft_ann_bench_param_parser.h b/cpp/bench/ann/src/raft/raft_ann_bench_param_parser.h index a5d1a75e3..6d9430acd 100644 --- a/cpp/bench/ann/src/raft/raft_ann_bench_param_parser.h +++ b/cpp/bench/ann/src/raft/raft_ann_bench_param_parser.h @@ -25,32 +25,32 @@ #endif #ifdef CUVS_BENCH_USE_RAFT_IVF_FLAT #include "raft_ivf_flat_wrapper.h" -extern template class raft::bench::ann::RaftIvfFlatGpu; -extern template class raft::bench::ann::RaftIvfFlatGpu; -extern template class raft::bench::ann::RaftIvfFlatGpu; +extern template class cuvs::bench::RaftIvfFlatGpu; +extern template class cuvs::bench::RaftIvfFlatGpu; +extern template class cuvs::bench::RaftIvfFlatGpu; #endif #if defined(CUVS_BENCH_USE_RAFT_IVF_PQ) || defined(CUVS_BENCH_USE_RAFT_CAGRA) || \ defined(CUVS_BENCH_USE_RAFT_CAGRA_HNSWLIB) #include "raft_ivf_pq_wrapper.h" #endif #ifdef CUVS_BENCH_USE_RAFT_IVF_PQ -extern template class raft::bench::ann::RaftIvfPQ; -extern template class raft::bench::ann::RaftIvfPQ; -extern template class raft::bench::ann::RaftIvfPQ; +extern template class cuvs::bench::RaftIvfPQ; +extern template class cuvs::bench::RaftIvfPQ; +extern template class cuvs::bench::RaftIvfPQ; #endif #if defined(CUVS_BENCH_USE_RAFT_CAGRA) || defined(CUVS_BENCH_USE_RAFT_CAGRA_HNSWLIB) #include "raft_cagra_wrapper.h" #endif #ifdef CUVS_BENCH_USE_RAFT_CAGRA -extern template class raft::bench::ann::RaftCagra; -extern template class raft::bench::ann::RaftCagra; -extern template class raft::bench::ann::RaftCagra; +extern template class cuvs::bench::RaftCagra; +extern template class cuvs::bench::RaftCagra; +extern template class cuvs::bench::RaftCagra; #endif #ifdef CUVS_BENCH_USE_RAFT_IVF_FLAT template void parse_build_param(const nlohmann::json& conf, - typename raft::bench::ann::RaftIvfFlatGpu::BuildParam& param) + typename cuvs::bench::RaftIvfFlatGpu::BuildParam& param) { param.n_lists = conf.at("nlist"); if (conf.contains("niter")) { param.kmeans_n_iters = conf.at("niter"); } @@ -59,7 +59,7 @@ void parse_build_param(const nlohmann::json& conf, template void parse_search_param(const nlohmann::json& conf, - typename raft::bench::ann::RaftIvfFlatGpu::SearchParam& param) + typename cuvs::bench::RaftIvfFlatGpu::SearchParam& param) { param.ivf_flat_params.n_probes = conf.at("nprobe"); } @@ -69,7 +69,7 @@ void parse_search_param(const nlohmann::json& conf, defined(CUVS_BENCH_USE_RAFT_CAGRA_HNSWLIB) template void parse_build_param(const nlohmann::json& conf, - typename raft::bench::ann::RaftIvfPQ::BuildParam& param) + typename cuvs::bench::RaftIvfPQ::BuildParam& param) { if (conf.contains("nlist")) { param.n_lists = conf.at("nlist"); } if (conf.contains("niter")) { param.kmeans_n_iters = conf.at("niter"); } @@ -79,9 +79,9 @@ void parse_build_param(const nlohmann::json& conf, if (conf.contains("codebook_kind")) { std::string kind = conf.at("codebook_kind"); if (kind == "cluster") { - param.codebook_kind = raft::neighbors::ivf_pq::codebook_gen::PER_CLUSTER; + param.codebook_kind = cuvs::neighbors::ivf_pq::codebook_gen::PER_CLUSTER; } else if (kind == "subspace") { - param.codebook_kind = raft::neighbors::ivf_pq::codebook_gen::PER_SUBSPACE; + param.codebook_kind = cuvs::neighbors::ivf_pq::codebook_gen::PER_SUBSPACE; } else { throw std::runtime_error("codebook_kind: '" + kind + "', should be either 'cluster' or 'subspace'"); @@ -91,7 +91,7 @@ void parse_build_param(const nlohmann::json& conf, template void parse_search_param(const nlohmann::json& conf, - typename raft::bench::ann::RaftIvfPQ::SearchParam& param) + typename cuvs::bench::RaftIvfPQ::SearchParam& param) { if (conf.contains("nprobe")) { param.pq_param.n_probes = conf.at("nprobe"); } if (conf.contains("internalDistanceDtype")) { @@ -135,7 +135,7 @@ void parse_search_param(const nlohmann::json& conf, #if defined(CUVS_BENCH_USE_RAFT_CAGRA) || defined(CUVS_BENCH_USE_RAFT_CAGRA_HNSWLIB) template void parse_build_param(const nlohmann::json& conf, - raft::neighbors::experimental::nn_descent::index_params& param) + cuvs::neighbors::experimental::nn_descent::index_params& param) { if (conf.contains("graph_degree")) { param.graph_degree = conf.at("graph_degree"); } if (conf.contains("intermediate_graph_degree")) { @@ -165,7 +165,7 @@ nlohmann::json collect_conf_with_prefix(const nlohmann::json& conf, template void parse_build_param(const nlohmann::json& conf, - typename raft::bench::ann::RaftCagra::BuildParam& param) + typename cuvs::bench::RaftCagra::BuildParam& param) { if (conf.contains("graph_degree")) { param.cagra_params.graph_degree = conf.at("graph_degree"); @@ -176,27 +176,27 @@ void parse_build_param(const nlohmann::json& conf, } if (conf.contains("graph_build_algo")) { if (conf.at("graph_build_algo") == "IVF_PQ") { - param.cagra_params.build_algo = raft::neighbors::cagra::graph_build_algo::IVF_PQ; + param.cagra_params.build_algo = cuvs::neighbors::cagra::graph_build_algo::IVF_PQ; } else if (conf.at("graph_build_algo") == "NN_DESCENT") { - param.cagra_params.build_algo = raft::neighbors::cagra::graph_build_algo::NN_DESCENT; + param.cagra_params.build_algo = cuvs::neighbors::cagra::graph_build_algo::NN_DESCENT; } } nlohmann::json ivf_pq_build_conf = collect_conf_with_prefix(conf, "ivf_pq_build_"); if (!ivf_pq_build_conf.empty()) { - raft::neighbors::ivf_pq::index_params bparam; + cuvs::neighbors::ivf_pq::index_params bparam; parse_build_param(ivf_pq_build_conf, bparam); param.ivf_pq_build_params = bparam; } nlohmann::json ivf_pq_search_conf = collect_conf_with_prefix(conf, "ivf_pq_search_"); if (!ivf_pq_search_conf.empty()) { - typename raft::bench::ann::RaftIvfPQ::SearchParam sparam; + typename cuvs::bench::RaftIvfPQ::SearchParam sparam; parse_search_param(ivf_pq_search_conf, sparam); param.ivf_pq_search_params = sparam.pq_param; param.ivf_pq_refine_rate = sparam.refine_ratio; } nlohmann::json nn_descent_conf = collect_conf_with_prefix(conf, "nn_descent_"); if (!nn_descent_conf.empty()) { - raft::neighbors::experimental::nn_descent::index_params nn_param; + cuvs::neighbors::experimental::nn_descent::index_params nn_param; nn_param.intermediate_graph_degree = 1.5 * param.cagra_params.intermediate_graph_degree; parse_build_param(nn_descent_conf, nn_param); if (nn_param.graph_degree != param.cagra_params.intermediate_graph_degree) { @@ -206,14 +206,14 @@ void parse_build_param(const nlohmann::json& conf, } } -raft::bench::ann::AllocatorType parse_allocator(std::string mem_type) +cuvs::bench::AllocatorType parse_allocator(std::string mem_type) { if (mem_type == "device") { - return raft::bench::ann::AllocatorType::Device; + return cuvs::bench::AllocatorType::Device; } else if (mem_type == "host_pinned") { - return raft::bench::ann::AllocatorType::HostPinned; + return cuvs::bench::AllocatorType::HostPinned; } else if (mem_type == "host_huge_page") { - return raft::bench::ann::AllocatorType::HostHugePage; + return cuvs::bench::AllocatorType::HostHugePage; } THROW( "Invalid value for memory type %s, must be one of [\"device\", \"host_pinned\", " @@ -223,20 +223,20 @@ raft::bench::ann::AllocatorType parse_allocator(std::string mem_type) template void parse_search_param(const nlohmann::json& conf, - typename raft::bench::ann::RaftCagra::SearchParam& param) + typename cuvs::bench::RaftCagra::SearchParam& param) { if (conf.contains("itopk")) { param.p.itopk_size = conf.at("itopk"); } if (conf.contains("search_width")) { param.p.search_width = conf.at("search_width"); } if (conf.contains("max_iterations")) { param.p.max_iterations = conf.at("max_iterations"); } if (conf.contains("algo")) { if (conf.at("algo") == "single_cta") { - param.p.algo = raft::neighbors::experimental::cagra::search_algo::SINGLE_CTA; + param.p.algo = cuvs::neighbors::experimental::cagra::search_algo::SINGLE_CTA; } else if (conf.at("algo") == "multi_cta") { - param.p.algo = raft::neighbors::experimental::cagra::search_algo::MULTI_CTA; + param.p.algo = cuvs::neighbors::experimental::cagra::search_algo::MULTI_CTA; } else if (conf.at("algo") == "multi_kernel") { - param.p.algo = raft::neighbors::experimental::cagra::search_algo::MULTI_KERNEL; + param.p.algo = cuvs::neighbors::experimental::cagra::search_algo::MULTI_KERNEL; } else if (conf.at("algo") == "auto") { - param.p.algo = raft::neighbors::experimental::cagra::search_algo::AUTO; + param.p.algo = cuvs::neighbors::experimental::cagra::search_algo::AUTO; } else { std::string tmp = conf.at("algo"); THROW("Invalid value for algo: %s", tmp.c_str()); diff --git a/cpp/bench/ann/src/raft/raft_ann_bench_utils.h b/cpp/bench/ann/src/raft/raft_ann_bench_utils.h index cb30c2693..2d656f514 100644 --- a/cpp/bench/ann/src/raft/raft_ann_bench_utils.h +++ b/cpp/bench/ann/src/raft/raft_ann_bench_utils.h @@ -15,12 +15,12 @@ */ #pragma once #include +#include #include #include #include #include #include -#include #include #include #include @@ -28,17 +28,17 @@ #include #include -namespace raft::bench::ann { +namespace cuvs::bench { -inline raft::distance::DistanceType parse_metric_type(raft::bench::ann::Metric metric) +inline cuvs::distance::DistanceType parse_metric_type(cuvs::bench::Metric metric) { - if (metric == raft::bench::ann::Metric::kInnerProduct) { - return raft::distance::DistanceType::InnerProduct; - } else if (metric == raft::bench::ann::Metric::kEuclidean) { + if (metric == cuvs::bench::Metric::kInnerProduct) { + return cuvs::distance::DistanceType::InnerProduct; + } else if (metric == cuvs::bench::Metric::kEuclidean) { // Even for L2 expanded RAFT IVF Flat uses unexpanded formula - return raft::distance::DistanceType::L2Expanded; + return cuvs::distance::DistanceType::L2Expanded; } else { throw std::runtime_error("raft supports only metric type of inner product and L2"); } } -} // namespace raft::bench::ann +} // namespace cuvs::bench diff --git a/cpp/bench/ann/src/raft/raft_benchmark.cu b/cpp/bench/ann/src/raft/raft_benchmark.cu index 5489e7a86..543772619 100644 --- a/cpp/bench/ann/src/raft/raft_benchmark.cu +++ b/cpp/bench/ann/src/raft/raft_benchmark.cu @@ -31,24 +31,24 @@ #define JSON_DIAGNOSTICS 1 #include -namespace raft::bench::ann { +namespace cuvs::bench { template -std::unique_ptr> create_algo(const std::string& algo, - const std::string& distance, - int dim, - const nlohmann::json& conf, - const std::vector& dev_list) +std::unique_ptr> create_algo(const std::string& algo, + const std::string& distance, + int dim, + const nlohmann::json& conf, + const std::vector& dev_list) { // stop compiler warning; not all algorithms support multi-GPU so it may not be used (void)dev_list; - raft::bench::ann::Metric metric = parse_metric(distance); - std::unique_ptr> ann; + cuvs::bench::Metric metric = parse_metric(distance); + std::unique_ptr> ann; if constexpr (std::is_same_v) { #ifdef CUVS_BENCH_USE_RAFT_BFKNN - if (algo == "raft_bfknn") { ann = std::make_unique>(metric, dim); } + if (algo == "raft_bfknn") { ann = std::make_unique>(metric, dim); } #endif } @@ -56,23 +56,23 @@ std::unique_ptr> create_algo(const std::string& algo, #ifdef CUVS_BENCH_USE_RAFT_IVF_FLAT if (algo == "raft_ivf_flat") { - typename raft::bench::ann::RaftIvfFlatGpu::BuildParam param; + typename cuvs::bench::RaftIvfFlatGpu::BuildParam param; parse_build_param(conf, param); - ann = std::make_unique>(metric, dim, param); + ann = std::make_unique>(metric, dim, param); } #endif #ifdef CUVS_BENCH_USE_RAFT_IVF_PQ if (algo == "raft_ivf_pq") { - typename raft::bench::ann::RaftIvfPQ::BuildParam param; + typename cuvs::bench::RaftIvfPQ::BuildParam param; parse_build_param(conf, param); - ann = std::make_unique>(metric, dim, param); + ann = std::make_unique>(metric, dim, param); } #endif #ifdef CUVS_BENCH_USE_RAFT_CAGRA if (algo == "raft_cagra") { - typename raft::bench::ann::RaftCagra::BuildParam param; + typename cuvs::bench::RaftCagra::BuildParam param; parse_build_param(conf, param); - ann = std::make_unique>(metric, dim, param); + ann = std::make_unique>(metric, dim, param); } #endif @@ -82,33 +82,32 @@ std::unique_ptr> create_algo(const std::string& algo, } template -std::unique_ptr::AnnSearchParam> create_search_param( +std::unique_ptr::AnnSearchParam> create_search_param( const std::string& algo, const nlohmann::json& conf) { #ifdef CUVS_BENCH_USE_RAFT_BFKNN if (algo == "raft_brute_force") { - auto param = std::make_unique::AnnSearchParam>(); + auto param = std::make_unique::AnnSearchParam>(); return param; } #endif #ifdef CUVS_BENCH_USE_RAFT_IVF_FLAT if (algo == "raft_ivf_flat") { - auto param = - std::make_unique::SearchParam>(); + auto param = std::make_unique::SearchParam>(); parse_search_param(conf, *param); return param; } #endif #ifdef CUVS_BENCH_USE_RAFT_IVF_PQ if (algo == "raft_ivf_pq") { - auto param = std::make_unique::SearchParam>(); + auto param = std::make_unique::SearchParam>(); parse_search_param(conf, *param); return param; } #endif #ifdef CUVS_BENCH_USE_RAFT_CAGRA if (algo == "raft_cagra") { - auto param = std::make_unique::SearchParam>(); + auto param = std::make_unique::SearchParam>(); parse_search_param(conf, *param); return param; } @@ -118,7 +117,7 @@ std::unique_ptr::AnnSearchParam> create_search throw std::runtime_error("invalid algo: '" + algo + "'"); } -}; // namespace raft::bench::ann +}; // namespace cuvs::bench REGISTER_ALGO_INSTANCE(float); REGISTER_ALGO_INSTANCE(std::int8_t); @@ -135,6 +134,6 @@ int main(int argc, char** argv) &pool_mr); // Updates the current device resource pointer to `pool_mr` rmm::mr::device_memory_resource* mr = rmm::mr::get_current_device_resource(); // Points to `pool_mr` - return raft::bench::ann::run_main(argc, argv); + return cuvs::bench::run_main(argc, argv); } #endif diff --git a/cpp/bench/ann/src/raft/raft_cagra.cu b/cpp/bench/ann/src/raft/raft_cagra.cu index be18af7f2..f424433e0 100644 --- a/cpp/bench/ann/src/raft/raft_cagra.cu +++ b/cpp/bench/ann/src/raft/raft_cagra.cu @@ -15,8 +15,8 @@ */ #include "raft_cagra_wrapper.h" -namespace raft::bench::ann { +namespace cuvs::bench { template class RaftCagra; template class RaftCagra; template class RaftCagra; -} // namespace raft::bench::ann +} // namespace cuvs::bench diff --git a/cpp/bench/ann/src/raft/raft_cagra_hnswlib.cu b/cpp/bench/ann/src/raft/raft_cagra_hnswlib.cu index 7b5ff0eba..3303d059e 100644 --- a/cpp/bench/ann/src/raft/raft_cagra_hnswlib.cu +++ b/cpp/bench/ann/src/raft/raft_cagra_hnswlib.cu @@ -23,34 +23,34 @@ #define JSON_DIAGNOSTICS 1 #include -namespace raft::bench::ann { +namespace cuvs::bench { template void parse_search_param(const nlohmann::json& conf, - typename raft::bench::ann::RaftCagraHnswlib::SearchParam& param) + typename cuvs::bench::RaftCagraHnswlib::SearchParam& param) { param.ef = conf.at("ef"); if (conf.contains("numThreads")) { param.num_threads = conf.at("numThreads"); } } template -std::unique_ptr> create_algo(const std::string& algo, - const std::string& distance, - int dim, - const nlohmann::json& conf, - const std::vector& dev_list) +std::unique_ptr> create_algo(const std::string& algo, + const std::string& distance, + int dim, + const nlohmann::json& conf, + const std::vector& dev_list) { // stop compiler warning; not all algorithms support multi-GPU so it may not be used (void)dev_list; - raft::bench::ann::Metric metric = parse_metric(distance); - std::unique_ptr> ann; + cuvs::bench::Metric metric = parse_metric(distance); + std::unique_ptr> ann; if constexpr (std::is_same_v or std::is_same_v) { if (algo == "raft_cagra_hnswlib") { - typename raft::bench::ann::RaftCagraHnswlib::BuildParam param; + typename cuvs::bench::RaftCagraHnswlib::BuildParam param; parse_build_param(conf, param); - ann = std::make_unique>(metric, dim, param); + ann = std::make_unique>(metric, dim, param); } } @@ -60,12 +60,12 @@ std::unique_ptr> create_algo(const std::string& algo, } template -std::unique_ptr::AnnSearchParam> create_search_param( +std::unique_ptr::AnnSearchParam> create_search_param( const std::string& algo, const nlohmann::json& conf) { if (algo == "raft_cagra_hnswlib") { auto param = - std::make_unique::SearchParam>(); + std::make_unique::SearchParam>(); parse_search_param(conf, *param); return param; } @@ -73,7 +73,7 @@ std::unique_ptr::AnnSearchParam> create_search throw std::runtime_error("invalid algo: '" + algo + "'"); } -} // namespace raft::bench::ann +} // namespace cuvs::bench REGISTER_ALGO_INSTANCE(float); REGISTER_ALGO_INSTANCE(std::int8_t); @@ -90,6 +90,6 @@ int main(int argc, char** argv) &pool_mr); // Updates the current device resource pointer to `pool_mr` rmm::mr::device_memory_resource* mr = rmm::mr::get_current_device_resource(); // Points to `pool_mr` - return raft::bench::ann::run_main(argc, argv); + return cuvs::bench::run_main(argc, argv); } #endif diff --git a/cpp/bench/ann/src/raft/raft_cagra_hnswlib_wrapper.h b/cpp/bench/ann/src/raft/raft_cagra_hnswlib_wrapper.h index 432caecfc..44fa94c8e 100644 --- a/cpp/bench/ann/src/raft/raft_cagra_hnswlib_wrapper.h +++ b/cpp/bench/ann/src/raft/raft_cagra_hnswlib_wrapper.h @@ -19,7 +19,7 @@ #include "raft_cagra_wrapper.h" #include -namespace raft::bench::ann { +namespace cuvs::bench { template class RaftCagraHnswlib : public ANN { @@ -117,4 +117,4 @@ void RaftCagraHnswlib::search( hnswlib_search_->search(queries, batch_size, k, neighbors, distances); } -} // namespace raft::bench::ann +} // namespace cuvs::bench diff --git a/cpp/bench/ann/src/raft/raft_cagra_wrapper.h b/cpp/bench/ann/src/raft/raft_cagra_wrapper.h index a3e481ec5..90c4218c5 100644 --- a/cpp/bench/ann/src/raft/raft_cagra_wrapper.h +++ b/cpp/bench/ann/src/raft/raft_cagra_wrapper.h @@ -16,6 +16,13 @@ #pragma once #include +#include +#include +#include +#include +#include +#include +#include #include #include #include @@ -24,14 +31,7 @@ #include #include #include -#include #include -#include -#include -#include -#include -#include -#include #include #include #include @@ -48,7 +48,7 @@ #include #include -namespace raft::bench::ann { +namespace cuvs::bench { enum class AllocatorType { HostPinned, HostHugePage, Device }; template @@ -57,19 +57,19 @@ class RaftCagra : public ANN { using typename ANN::AnnSearchParam; struct SearchParam : public AnnSearchParam { - raft::neighbors::experimental::cagra::search_params p; + cuvs::neighbors::experimental::cagra::search_params p; AllocatorType graph_mem = AllocatorType::Device; AllocatorType dataset_mem = AllocatorType::Device; auto needs_dataset() const -> bool override { return true; } }; struct BuildParam { - raft::neighbors::cagra::index_params cagra_params; - std::optional nn_descent_params = + cuvs::neighbors::cagra::index_params cagra_params; + std::optional nn_descent_params = std::nullopt; std::optional ivf_pq_refine_rate = std::nullopt; - std::optional ivf_pq_build_params = std::nullopt; - std::optional ivf_pq_search_params = std::nullopt; + std::optional ivf_pq_build_params = std::nullopt; + std::optional ivf_pq_search_params = std::nullopt; }; RaftCagra(Metric metric, int dim, const BuildParam& param, int concurrent_searches = 1) @@ -134,8 +134,8 @@ class RaftCagra : public ANN { AllocatorType dataset_mem_; BuildParam index_params_; bool need_dataset_update_; - raft::neighbors::cagra::search_params search_params_; - std::optional> index_; + cuvs::neighbors::cagra::search_params search_params_; + std::optional> index_; int device_; int dimension_; raft::device_matrix graph_; @@ -151,7 +151,7 @@ void RaftCagra::build(const T* dataset, size_t nrow, cudaStream_t) auto& params = index_params_.cagra_params; - index_.emplace(raft::neighbors::cagra::detail::build(handle_, + index_.emplace(cuvs::neighbors::cagra::detail::build(handle_, params, dataset_view, index_params_.nn_descent_params, @@ -209,7 +209,7 @@ void RaftCagra::set_search_param(const AnnSearchParam& param) allocator_to_string(dataset_mem_).c_str()); auto mr = get_mr(dataset_mem_); - raft::neighbors::cagra::detail::copy_with_padding(handle_, dataset_, input_dataset_v_, mr); + cuvs::neighbors::cagra::detail::copy_with_padding(handle_, dataset_, input_dataset_v_, mr); index_->update_dataset(handle_, make_const_mdspan(dataset_.view())); @@ -237,19 +237,19 @@ void RaftCagra::set_search_dataset(const T* dataset, size_t nrow) template void RaftCagra::save(const std::string& file) const { - raft::neighbors::cagra::serialize(handle_, file, *index_); + cuvs::neighbors::cagra::serialize(handle_, file, *index_); } template void RaftCagra::save_to_hnswlib(const std::string& file) const { - raft::neighbors::cagra::serialize_to_hnswlib(handle_, file, *index_); + cuvs::neighbors::cagra::serialize_to_hnswlib(handle_, file, *index_); } template void RaftCagra::load(const std::string& file) { - index_ = raft::neighbors::cagra::deserialize(handle_, file); + index_ = cuvs::neighbors::cagra::deserialize(handle_, file); } template @@ -270,7 +270,7 @@ void RaftCagra::search( auto neighbors_view = raft::make_device_matrix_view(neighbors_IdxT, batch_size, k); auto distances_view = raft::make_device_matrix_view(distances, batch_size, k); - raft::neighbors::cagra::search( + cuvs::neighbors::cagra::search( handle_, search_params_, *index_, queries_view, neighbors_view, distances_view); if (!std::is_same::value) { @@ -283,4 +283,4 @@ void RaftCagra::search( handle_.sync_stream(); } -} // namespace raft::bench::ann +} // namespace cuvs::bench diff --git a/cpp/bench/ann/src/raft/raft_ivf_flat.cu b/cpp/bench/ann/src/raft/raft_ivf_flat.cu index bcd23723a..4cfe37242 100644 --- a/cpp/bench/ann/src/raft/raft_ivf_flat.cu +++ b/cpp/bench/ann/src/raft/raft_ivf_flat.cu @@ -15,8 +15,8 @@ */ #include "raft_ivf_flat_wrapper.h" -namespace raft::bench::ann { +namespace cuvs::bench { template class RaftIvfFlatGpu; template class RaftIvfFlatGpu; template class RaftIvfFlatGpu; -} // namespace raft::bench::ann +} // namespace cuvs::bench diff --git a/cpp/bench/ann/src/raft/raft_ivf_flat_wrapper.h b/cpp/bench/ann/src/raft/raft_ivf_flat_wrapper.h index 24b3c69bb..7ff420dec 100644 --- a/cpp/bench/ann/src/raft/raft_ivf_flat_wrapper.h +++ b/cpp/bench/ann/src/raft/raft_ivf_flat_wrapper.h @@ -16,6 +16,9 @@ #pragma once #include +#include +#include +#include #include #include #include @@ -23,10 +26,7 @@ #include #include #include -#include #include -#include -#include #include #include #include @@ -38,7 +38,7 @@ #include "raft_ann_bench_utils.h" #include -namespace raft::bench::ann { +namespace cuvs::bench { template class RaftIvfFlatGpu : public ANN { @@ -46,10 +46,10 @@ class RaftIvfFlatGpu : public ANN { using typename ANN::AnnSearchParam; struct SearchParam : public AnnSearchParam { - raft::neighbors::ivf_flat::search_params ivf_flat_params; + cuvs::neighbors::ivf_flat::search_params ivf_flat_params; }; - using BuildParam = raft::neighbors::ivf_flat::index_params; + using BuildParam = cuvs::neighbors::ivf_flat::index_params; RaftIvfFlatGpu(Metric metric, int dim, const BuildParam& param) : ANN(metric, dim), index_params_(param), dimension_(dim) @@ -88,8 +88,8 @@ class RaftIvfFlatGpu : public ANN { private: raft::device_resources handle_; BuildParam index_params_; - raft::neighbors::ivf_flat::search_params search_params_; - std::optional> index_; + cuvs::neighbors::ivf_flat::search_params search_params_; + std::optional> index_; int device_; int dimension_; }; @@ -98,7 +98,7 @@ template void RaftIvfFlatGpu::build(const T* dataset, size_t nrow, cudaStream_t) { index_.emplace( - raft::neighbors::ivf_flat::build(handle_, index_params_, dataset, IdxT(nrow), dimension_)); + cuvs::neighbors::ivf_flat::build(handle_, index_params_, dataset, IdxT(nrow), dimension_)); return; } @@ -113,14 +113,14 @@ void RaftIvfFlatGpu::set_search_param(const AnnSearchParam& param) template void RaftIvfFlatGpu::save(const std::string& file) const { - raft::neighbors::ivf_flat::serialize(handle_, file, *index_); + cuvs::neighbors::ivf_flat::serialize(handle_, file, *index_); return; } template void RaftIvfFlatGpu::load(const std::string& file) { - index_ = raft::neighbors::ivf_flat::deserialize(handle_, file); + index_ = cuvs::neighbors::ivf_flat::deserialize(handle_, file); return; } @@ -129,9 +129,9 @@ void RaftIvfFlatGpu::search( const T* queries, int batch_size, int k, size_t* neighbors, float* distances, cudaStream_t) const { static_assert(sizeof(size_t) == sizeof(IdxT), "IdxT is incompatible with size_t"); - raft::neighbors::ivf_flat::search( + cuvs::neighbors::ivf_flat::search( handle_, search_params_, *index_, queries, batch_size, k, (IdxT*)neighbors, distances); resource::sync_stream(handle_); return; } -} // namespace raft::bench::ann +} // namespace cuvs::bench diff --git a/cpp/bench/ann/src/raft/raft_ivf_pq.cu b/cpp/bench/ann/src/raft/raft_ivf_pq.cu index 2efe14631..8df1bfab2 100644 --- a/cpp/bench/ann/src/raft/raft_ivf_pq.cu +++ b/cpp/bench/ann/src/raft/raft_ivf_pq.cu @@ -15,8 +15,8 @@ */ #include "raft_ivf_pq_wrapper.h" -namespace raft::bench::ann { +namespace cuvs::bench { template class RaftIvfPQ; template class RaftIvfPQ; template class RaftIvfPQ; -} // namespace raft::bench::ann +} // namespace cuvs::bench diff --git a/cpp/bench/ann/src/raft/raft_ivf_pq_wrapper.h b/cpp/bench/ann/src/raft/raft_ivf_pq_wrapper.h index e4004b000..f940507a0 100644 --- a/cpp/bench/ann/src/raft/raft_ivf_pq_wrapper.h +++ b/cpp/bench/ann/src/raft/raft_ivf_pq_wrapper.h @@ -15,6 +15,8 @@ */ #pragma once +#include +#include #include #include #include @@ -22,9 +24,7 @@ #include #include #include -#include #include -#include #include #include #include @@ -37,7 +37,7 @@ #include "raft_ann_bench_utils.h" #include -namespace raft::bench::ann { +namespace cuvs::bench { template class RaftIvfPQ : public ANN { @@ -46,12 +46,12 @@ class RaftIvfPQ : public ANN { using ANN::dim_; struct SearchParam : public AnnSearchParam { - raft::neighbors::ivf_pq::search_params pq_param; + cuvs::neighbors::ivf_pq::search_params pq_param; float refine_ratio = 1.0f; auto needs_dataset() const -> bool override { return refine_ratio > 1.0f; } }; - using BuildParam = raft::neighbors::ivf_pq::index_params; + using BuildParam = cuvs::neighbors::ivf_pq::index_params; RaftIvfPQ(Metric metric, int dim, const BuildParam& param) : ANN(metric, dim), index_params_(param), dimension_(dim) @@ -92,8 +92,8 @@ class RaftIvfPQ : public ANN { raft::device_resources handle_; cudaEvent_t sync_{nullptr}; BuildParam index_params_; - raft::neighbors::ivf_pq::search_params search_params_; - std::optional> index_; + cuvs::neighbors::ivf_pq::search_params search_params_; + std::optional> index_; int device_; int dimension_; float refine_ratio_ = 1.0; @@ -115,7 +115,7 @@ void RaftIvfPQ::save(const std::string& file) const template void RaftIvfPQ::load(const std::string& file) { - auto index_tmp = raft::neighbors::ivf_pq::index(handle_, index_params_, dimension_); + auto index_tmp = cuvs::neighbors::ivf_pq::index(handle_, index_params_, dimension_); raft::runtime::neighbors::ivf_pq::deserialize(handle_, file, &index_tmp); index_.emplace(std::move(index_tmp)); return; @@ -218,4 +218,4 @@ void RaftIvfPQ::search(const T* queries, stream_wait(stream); // RAFT stream -> bench stream } } -} // namespace raft::bench::ann +} // namespace cuvs::bench diff --git a/cpp/bench/ann/src/raft/raft_wrapper.h b/cpp/bench/ann/src/raft/raft_wrapper.h index 499bdf29a..1f9ddd585 100644 --- a/cpp/bench/ann/src/raft/raft_wrapper.h +++ b/cpp/bench/ann/src/raft/raft_wrapper.h @@ -16,9 +16,9 @@ #pragma once #include +#include +#include #include -#include -#include #include #include #include @@ -28,12 +28,12 @@ namespace raft_temp { -inline raft::distance::DistanceType parse_metric_type(raft::bench::ann::Metric metric) +inline cuvs::distance::DistanceType parse_metric_type(cuvs::bench::Metric metric) { - if (metric == raft::bench::ann::Metric::kInnerProduct) { - return raft::distance::DistanceType::InnerProduct; - } else if (metric == raft::bench::ann::Metric::kEuclidean) { - return raft::distance::DistanceType::L2Expanded; + if (metric == cuvs::bench::Metric::kInnerProduct) { + return cuvs::distance::DistanceType::InnerProduct; + } else if (metric == cuvs::bench::Metric::kEuclidean) { + return cuvs::distance::DistanceType::L2Expanded; } else { throw std::runtime_error("raft supports only metric type of inner product and L2"); } @@ -41,7 +41,7 @@ inline raft::distance::DistanceType parse_metric_type(raft::bench::ann::Metric m } // namespace raft_temp -namespace raft::bench::ann { +namespace cuvs::bench { // brute force fused L2 KNN - RAFT template @@ -77,7 +77,7 @@ class RaftGpu : public ANN { void load(const std::string&) override { return; }; protected: - raft::distance::DistanceType metric_type_; + cuvs::distance::DistanceType metric_type_; int device_; const T* dataset_; size_t nrow_; @@ -88,7 +88,7 @@ RaftGpu::RaftGpu(Metric metric, int dim) : ANN(metric, dim), metric_type_(raft_temp::parse_metric_type(metric)) { static_assert(std::is_same_v, "raft support only float type"); - assert(metric_type_ == raft::distance::DistanceType::L2Expanded); + assert(metric_type_ == cuvs::distance::DistanceType::L2Expanded); RAFT_CUDA_TRY(cudaGetDevice(&device_)); } @@ -150,4 +150,4 @@ void RaftGpu::search(const T* queries, metric_type_); } -} // namespace raft::bench::ann +} // namespace cuvs::bench diff --git a/cpp/bench/micro/cluster/kmeans.cu b/cpp/bench/micro/cluster/kmeans.cu index 3147960f7..6f6f7efa7 100644 --- a/cpp/bench/micro/cluster/kmeans.cu +++ b/cpp/bench/micro/cluster/kmeans.cu @@ -15,15 +15,15 @@ */ #include -#include -#include +#include +#include -namespace raft::bench::cluster { +namespace cuvs::bench::cluster { struct KMeansBenchParams { DatasetParams data; BlobsParams blobs; - raft::cluster::KMeansParams kmeans; + cuvs::cluster::KMeansParams kmeans; }; inline auto operator<<(std::ostream& os, const KMeansBenchParams& p) -> std::ostream& @@ -57,7 +57,7 @@ struct KMeans : public BlobsFixture { raft::host_scalar_view n_iter_view = raft::make_host_scalar_view(&n_iter); this->loop_on_state(state, [&]() { - raft::cluster::kmeans_fit_predict(this->handle, + cuvs::cluster::kmeans_fit_predict(this->handle, params.kmeans, X_view, opt_weights_view, @@ -93,11 +93,11 @@ std::vector getKMeansInputs() p.blobs.center_box_min = -10.0; p.blobs.center_box_max = 10.0; p.blobs.seed = 12345ULL; - p.kmeans.init = raft::cluster::KMeansParams::KMeansPlusPlus; + p.kmeans.init = cuvs::cluster::KMeansParams::KMeansPlusPlus; p.kmeans.max_iter = 300; p.kmeans.tol = 1e-4; p.kmeans.verbosity = RAFT_LEVEL_INFO; - p.kmeans.metric = raft::distance::DistanceType::L2Expanded; + p.kmeans.metric = cuvs::distance::DistanceType::L2Expanded; p.kmeans.inertia_check = true; std::vector> row_cols_k = { {1000000, 20, 1000}, @@ -121,4 +121,4 @@ RAFT_BENCH_REGISTER((KMeans), "", getKMeansInputs()); // RAFT_BENCH_REGISTER((KMeans), "", getKMeansInputs()); // RAFT_BENCH_REGISTER((KMeans), "", getKMeansInputs()); -} // namespace raft::bench::cluster +} // namespace cuvs::bench::cluster diff --git a/cpp/bench/micro/cluster/kmeans_balanced.cu b/cpp/bench/micro/cluster/kmeans_balanced.cu index 129578c30..63021ad6a 100644 --- a/cpp/bench/micro/cluster/kmeans_balanced.cu +++ b/cpp/bench/micro/cluster/kmeans_balanced.cu @@ -15,16 +15,16 @@ */ #include -#include +#include #include #include -namespace raft::bench::cluster { +namespace cuvs::bench::cluster { struct KMeansBalancedBenchParams { DatasetParams data; uint32_t n_lists; - raft::cluster::kmeans_balanced_params kb_params; + cuvs::cluster::kmeans_balanced_params kb_params; }; template @@ -36,7 +36,7 @@ struct KMeansBalanced : public fixture { this->loop_on_state(state, [this]() { raft::device_matrix_view X_view = this->X.view(); raft::device_matrix_view centroids_view = this->centroids.view(); - raft::cluster::kmeans_balanced::fit( + cuvs::cluster::kmeans_balanced::fit( this->handle, this->params.kb_params, X_view, centroids_view); }); } @@ -76,7 +76,7 @@ std::vector getKMeansBalancedInputs() KMeansBalancedBenchParams p; p.data.row_major = true; p.kb_params.n_iters = 20; - p.kb_params.metric = raft::distance::DistanceType::L2Expanded; + p.kb_params.metric = cuvs::distance::DistanceType::L2Expanded; std::vector> row_cols = { {100000, 128}, {1000000, 128}, {10000000, 128}, // The following dataset sizes are too large for most GPUs. @@ -96,4 +96,4 @@ std::vector getKMeansBalancedInputs() // Note: the datasets sizes are too large for 32-bit index types. RAFT_BENCH_REGISTER((KMeansBalanced), "", getKMeansBalancedInputs()); -} // namespace raft::bench::cluster +} // namespace cuvs::bench::cluster diff --git a/cpp/bench/micro/common/benchmark.hpp b/cpp/bench/micro/common/benchmark.hpp index d3da3bff6..7eda12dcd 100644 --- a/cpp/bench/micro/common/benchmark.hpp +++ b/cpp/bench/micro/common/benchmark.hpp @@ -34,7 +34,7 @@ #include #include -namespace raft::bench { +namespace cuvs::bench { /** * RAII way to temporary set the pooling memory allocator in rmm. @@ -333,7 +333,7 @@ struct registrar { }; // namespace internal #define RAFT_BENCH_REGISTER_INTERNAL(TestClass, ...) \ - static raft::bench::internal::registrar BENCHMARK_PRIVATE_NAME(registrar)( \ + static cuvs::bench::internal::registrar BENCHMARK_PRIVATE_NAME(registrar)( \ RAFT_STRINGIFY(TestClass), __VA_ARGS__) /** @@ -341,7 +341,7 @@ struct registrar { * for the set of benchmarks to be registered so that the main harness inside * google bench can find these benchmarks and run them. * - * @param TestClass child class of `raft::bench::Fixture` which contains + * @param TestClass child class of `cuvs::bench::Fixture` which contains * the logic to generate the dataset and run training on it * for a given algo. Ideally, once such struct is needed for * every algo to be benchmarked @@ -353,4 +353,4 @@ struct registrar { #define RAFT_BENCH_REGISTER(TestClass, ...) \ RAFT_BENCH_REGISTER_INTERNAL(RAFT_DEPAREN(TestClass), __VA_ARGS__) -} // namespace raft::bench +} // namespace cuvs::bench diff --git a/cpp/bench/micro/distance/distance_common.cuh b/cpp/bench/micro/distance/distance_common.cuh index dff3401b6..a861400d9 100644 --- a/cpp/bench/micro/distance/distance_common.cuh +++ b/cpp/bench/micro/distance/distance_common.cuh @@ -15,18 +15,18 @@ */ #include -#include +#include #include #include -namespace raft::bench::distance { +namespace cuvs::bench::distance { struct distance_params { int m, n, k; bool isRowMajor; }; // struct distance_params -template +template struct distance : public fixture { distance(const distance_params& p) : params(p), @@ -38,7 +38,7 @@ struct distance : public fixture { RAFT_CUDA_TRY(cudaMemsetAsync(x.data(), 0, x.size() * sizeof(T), stream)); RAFT_CUDA_TRY(cudaMemsetAsync(y.data(), 0, y.size() * sizeof(T), stream)); RAFT_CUDA_TRY(cudaMemsetAsync(out.data(), 0, out.size() * sizeof(T), stream)); - worksize = raft::distance::getWorkspaceSize( + worksize = cuvs::distance::getWorkspaceSize( x.data(), y.data(), params.m, params.n, params.k); workspace.resize(worksize, stream); } @@ -46,7 +46,7 @@ struct distance : public fixture { void run_benchmark(::benchmark::State& state) override { loop_on_state(state, [this]() { - raft::distance::distance(handle, + cuvs::distance::distance(handle, x.data(), y.data(), out.data(), @@ -89,4 +89,4 @@ const std::vector dist_input_vecs{ using Name##D = distance; \ RAFT_BENCH_REGISTER(Name##D, "", dist_input_vecs); -} // namespace raft::bench::distance +} // namespace cuvs::bench::distance diff --git a/cpp/bench/micro/distance/distance_cosine.cu b/cpp/bench/micro/distance/distance_cosine.cu index c8ac8067c..68756070d 100644 --- a/cpp/bench/micro/distance/distance_cosine.cu +++ b/cpp/bench/micro/distance/distance_cosine.cu @@ -16,8 +16,8 @@ #include "distance_common.cuh" -namespace raft::bench::distance { +namespace cuvs::bench::distance { -DIST_BENCH_REGISTER(DistanceCosine, raft::distance::DistanceType::CosineExpanded); +DIST_BENCH_REGISTER(DistanceCosine, cuvs::distance::DistanceType::CosineExpanded); -} // namespace raft::bench::distance +} // namespace cuvs::bench::distance diff --git a/cpp/bench/micro/distance/distance_exp_l2.cu b/cpp/bench/micro/distance/distance_exp_l2.cu index 52b7fff05..edda28c5b 100644 --- a/cpp/bench/micro/distance/distance_exp_l2.cu +++ b/cpp/bench/micro/distance/distance_exp_l2.cu @@ -16,9 +16,9 @@ #include "distance_common.cuh" -namespace raft::bench::distance { +namespace cuvs::bench::distance { -DIST_BENCH_REGISTER(DistanceL2Sq, raft::distance::DistanceType::L2Expanded); -DIST_BENCH_REGISTER(DistanceL2Sqrt, raft::distance::DistanceType::L2SqrtExpanded); +DIST_BENCH_REGISTER(DistanceL2Sq, cuvs::distance::DistanceType::L2Expanded); +DIST_BENCH_REGISTER(DistanceL2Sqrt, cuvs::distance::DistanceType::L2SqrtExpanded); -} // namespace raft::bench::distance +} // namespace cuvs::bench::distance diff --git a/cpp/bench/micro/distance/distance_l1.cu b/cpp/bench/micro/distance/distance_l1.cu index e80db48ef..0e57b6677 100644 --- a/cpp/bench/micro/distance/distance_l1.cu +++ b/cpp/bench/micro/distance/distance_l1.cu @@ -16,8 +16,8 @@ #include "distance_common.cuh" -namespace raft::bench::distance { +namespace cuvs::bench::distance { -DIST_BENCH_REGISTER(DistanceL1, raft::distance::DistanceType::L1); +DIST_BENCH_REGISTER(DistanceL1, cuvs::distance::DistanceType::L1); -} // namespace raft::bench::distance +} // namespace cuvs::bench::distance diff --git a/cpp/bench/micro/distance/distance_unexp_l2.cu b/cpp/bench/micro/distance/distance_unexp_l2.cu index 7ac1a8a4b..0f142393f 100644 --- a/cpp/bench/micro/distance/distance_unexp_l2.cu +++ b/cpp/bench/micro/distance/distance_unexp_l2.cu @@ -16,9 +16,9 @@ #include "distance_common.cuh" -namespace raft::bench::distance { +namespace cuvs::bench::distance { -DIST_BENCH_REGISTER(DistanceUnexpL2Sq, raft::distance::DistanceType::L2Unexpanded); -DIST_BENCH_REGISTER(DistanceUnexpL2Sqrt, raft::distance::DistanceType::L2SqrtUnexpanded); +DIST_BENCH_REGISTER(DistanceUnexpL2Sq, cuvs::distance::DistanceType::L2Unexpanded); +DIST_BENCH_REGISTER(DistanceUnexpL2Sqrt, cuvs::distance::DistanceType::L2SqrtUnexpanded); -} // namespace raft::bench::distance +} // namespace cuvs::bench::distance diff --git a/cpp/bench/micro/distance/fused_l2_nn.cu b/cpp/bench/micro/distance/fused_l2_nn.cu index c0ebd6045..98d5aefd5 100644 --- a/cpp/bench/micro/distance/fused_l2_nn.cu +++ b/cpp/bench/micro/distance/fused_l2_nn.cu @@ -15,13 +15,13 @@ */ #include +#include #include -#include #include #include #include -namespace raft::bench::distance { +namespace cuvs::bench::distance { struct fusedl2nn_inputs { int64_t m, n, k; @@ -90,7 +90,7 @@ struct fusedl2nn : public fixture { state.SetLabel(label_stream.str()); loop_on_state(state, [this]() { - raft::distance::fusedL2NNMinReduce(out.data_handle(), + cuvs::distance::fusedL2NNMinReduce(out.data_handle(), x.data_handle(), y.data_handle(), x_norm.data_handle(), @@ -158,4 +158,4 @@ FUSEDL2NN_BENCH(double, int64_t, double); FUSEDL2NN_BENCH(float, int64_t, (raft::KeyValuePair)); FUSEDL2NN_BENCH(double, int64_t, (raft::KeyValuePair)); -} // namespace raft::bench::distance +} // namespace cuvs::bench::distance diff --git a/cpp/bench/micro/distance/kernels.cu b/cpp/bench/micro/distance/kernels.cu index 3f7475966..7078ceb6c 100644 --- a/cpp/bench/micro/distance/kernels.cu +++ b/cpp/bench/micro/distance/kernels.cu @@ -14,19 +14,19 @@ * limitations under the License. */ #include +#include +#include #include #include #include -#include -#include #include #include #include #include -namespace raft::bench::distance::kernels { +namespace cuvs::bench::distance::kernels { -using namespace raft::distance::kernels; +using namespace cuvs::distance::kernels; struct GramTestParams { int m; // m parameter of the GEMM int k; // k parameter of the GEMM @@ -117,4 +117,4 @@ static std::vector getInputs() RAFT_BENCH_REGISTER(GramMatrix, "", getInputs()); RAFT_BENCH_REGISTER(GramMatrix, "", getInputs()); -} // namespace raft::bench::distance::kernels +} // namespace cuvs::bench::distance::kernels diff --git a/cpp/bench/micro/distance/masked_nn.cu b/cpp/bench/micro/distance/masked_nn.cu index 19d78f4cd..7f68747c5 100644 --- a/cpp/bench/micro/distance/masked_nn.cu +++ b/cpp/bench/micro/distance/masked_nn.cu @@ -21,16 +21,16 @@ #include #include +#include #include #include #include #include -#include #include #include #include -namespace raft::bench::distance::masked_nn { +namespace cuvs::bench::distance::masked_nn { // Introduce various sparsity patterns enum AdjacencyPattern { @@ -91,9 +91,9 @@ struct masked_l2_nn : public fixture { using DataT = T; using IdxT = int; using OutT = raft::KeyValuePair; - using RedOpT = raft::distance::MinAndDistanceReduceOp; - using PairRedOpT = raft::distance::KVPMinReduce; - using ParamT = raft::distance::masked_l2_nn_params; + using RedOpT = cuvs::distance::MinAndDistanceReduceOp; + using PairRedOpT = cuvs::distance::KVPMinReduce; + using ParamT = cuvs::distance::masked_l2_nn_params; // Parameters Params params; @@ -122,7 +122,7 @@ struct masked_l2_nn : public fixture { xn.data_handle(), x.data_handle(), p.k, p.m, raft::linalg::L2Norm, true, stream); raft::linalg::rowNorm( yn.data_handle(), y.data_handle(), p.k, p.n, raft::linalg::L2Norm, true, stream); - raft::distance::initialize, int>( + cuvs::distance::initialize, int>( handle, out.data_handle(), p.m, std::numeric_limits::max(), RedOpT{}); dim3 block(32, 32); @@ -139,7 +139,7 @@ struct masked_l2_nn : public fixture { loop_on_state(state, [this, masked_l2_params]() { // It is sufficient to only benchmark the L2-squared metric - raft::distance::masked_l2_nn(handle, + cuvs::distance::masked_l2_nn(handle, masked_l2_params, x.view(), y.view(), @@ -260,4 +260,4 @@ RAFT_BENCH_REGISTER(masked_l2_nn, "", masked_l2_nn_input_vecs); // We don't benchmark double to keep compile times in check when not using the // distance library. -} // namespace raft::bench::distance::masked_nn +} // namespace cuvs::bench::distance::masked_nn diff --git a/cpp/bench/micro/distance/tune_pairwise/bench.cu b/cpp/bench/micro/distance/tune_pairwise/bench.cu index 87159ab1b..e04c37e5e 100644 --- a/cpp/bench/micro/distance/tune_pairwise/bench.cu +++ b/cpp/bench/micro/distance/tune_pairwise/bench.cu @@ -41,11 +41,11 @@ #include "kernel.cuh" // launch_kernel #include // std::min #include // RAFT_BENCH_REGISTER -#include // pairwise_matrix_params +#include // pairwise_matrix_params #include // rmm::device_uvector #include // std::vector -namespace raft::bench::distance::tune { +namespace cuvs::bench::distance::tune { // Max throughput benchmark. // @@ -148,4 +148,4 @@ struct throughput_bench : public fixture { RAFT_BENCH_REGISTER(throughput_bench, "", throughput_params); -} // namespace raft::bench::distance::tune +} // namespace cuvs::bench::distance::tune diff --git a/cpp/bench/micro/distance/tune_pairwise/kernel.cu b/cpp/bench/micro/distance/tune_pairwise/kernel.cu index 3112e1ea9..6970014ae 100644 --- a/cpp/bench/micro/distance/tune_pairwise/kernel.cu +++ b/cpp/bench/micro/distance/tune_pairwise/kernel.cu @@ -15,14 +15,14 @@ */ #include "kernel.cuh" -#include // pairwise_matrix_sm60_wrapper +#include // pairwise_matrix_sm60_wrapper #include // raft::linalg::Policy4x4 #include // raft::util::arch::SM_compute_arch -namespace raft::bench::distance::tune { +namespace cuvs::bench::distance::tune { // Distance op -using OpT = raft::distance::detail::ops::lp_unexp_distance_op; +using OpT = cuvs::distance::detail::ops::lp_unexp_distance_op; constexpr float metric_arg = 2.0; OpT distance_op{metric_arg}; @@ -40,7 +40,7 @@ void launch_kernel(pairwise_matrix_params params, dim3 grid, cudaStream_t stream int smem_size = OpT::shared_mem_size(); // Obtain function pointer to kernel - auto kernel = raft::distance::detail::pairwise_matrix_kernel // lp_unexp_distance_op -#include // pairwise_matrix_params +#include // lp_unexp_distance_op +#include // pairwise_matrix_params -namespace raft::bench::distance::tune { +namespace cuvs::bench::distance::tune { // Launch one specific kernel with the following template parameters constexpr bool row_major = true; @@ -31,7 +31,7 @@ using IdxT = int; using FinOpT = raft::identity_op; using pairwise_matrix_params = - raft::distance::detail::pairwise_matrix_params; + cuvs::distance::detail::pairwise_matrix_params; // Launches kernel void launch_kernel(pairwise_matrix_params, dim3, cudaStream_t); @@ -41,4 +41,4 @@ void get_block_size(int& m, int& n, int& k); int get_max_occupancy(); -} // namespace raft::bench::distance::tune +} // namespace cuvs::bench::distance::tune diff --git a/cpp/bench/micro/matrix/select_k.cu b/cpp/bench/micro/matrix/select_k.cu index 5eee5e3d2..f1c1ec1a6 100644 --- a/cpp/bench/micro/matrix/select_k.cu +++ b/cpp/bench/micro/matrix/select_k.cu @@ -37,7 +37,7 @@ #include namespace raft::matrix { -using namespace raft::bench; // NOLINT +using namespace cuvs::bench; // NOLINT template struct replace_with_mask { @@ -258,7 +258,7 @@ SELECTION_REGISTER(double, int64_t, kWarpDistributedShm); // NOLINT << input.len << "/" << input.k << "/" << input.use_index_input << "/" \ << input.use_memory_pool; \ auto* b = ::benchmark::internal::RegisterBenchmarkInternal( \ - new raft::bench::internal::Fixture(name.str(), input)); \ + new cuvs::bench::internal::Fixture(name.str(), input)); \ b->UseManualTime(); \ b->Unit(benchmark::kMillisecond); \ } @@ -279,7 +279,7 @@ const static size_t MAX_MEMORY = 16 * 1024 * 1024 * 1024ULL; SELECTION_REGISTER_ALGO_INPUT(KeyT, IdxT, kWarpDistributed, input) \ SELECTION_REGISTER_ALGO_INPUT(KeyT, IdxT, kWarpDistributedShm, input) \ } \ - if (input.k <= raft::neighbors::detail::kFaissMaxK()) { \ + if (input.k <= cuvs::neighbors::detail::kFaissMaxK()) { \ SELECTION_REGISTER_ALGO_INPUT(KeyT, IdxT, kFaissBlockSelect, input) \ } \ } \ diff --git a/cpp/bench/micro/neighbors/cagra_bench.cuh b/cpp/bench/micro/neighbors/cagra_bench.cuh index 07e93a347..3be664db8 100644 --- a/cpp/bench/micro/neighbors/cagra_bench.cuh +++ b/cpp/bench/micro/neighbors/cagra_bench.cuh @@ -17,15 +17,15 @@ #pragma once #include -#include -#include +#include +#include #include #include #include #include -namespace raft::bench::neighbors { +namespace cuvs::bench::neighbors { struct params { /** Size of the dataset. */ @@ -77,7 +77,7 @@ struct CagraBench : public fixture { raft::random::uniformInt( handle, state, knn_graph_.data_handle(), knn_graph_.size(), 0, ps.n_samples - 1); - auto metric = raft::distance::DistanceType::L2Expanded; + auto metric = cuvs::distance::DistanceType::L2Expanded; auto removed_indices = raft::make_device_vector(handle, ps.removed_ratio * ps.n_samples); @@ -86,13 +86,13 @@ struct CagraBench : public fixture { thrust::device_pointer_cast(removed_indices.data_handle()), thrust::device_pointer_cast(removed_indices.data_handle() + removed_indices.extent(0))); removed_indices_bitset_.set(handle, removed_indices.view()); - index_.emplace(raft::neighbors::cagra::index( + index_.emplace(cuvs::neighbors::cagra::index( handle, metric, make_const_mdspan(dataset_.view()), make_const_mdspan(knn_graph_.view()))); } void run_benchmark(::benchmark::State& state) override { - raft::neighbors::cagra::search_params search_params; + cuvs::neighbors::cagra::search_params search_params; search_params.max_queries = 1024; search_params.itopk_size = params_.itopk_size; search_params.team_size = 0; @@ -108,14 +108,14 @@ struct CagraBench : public fixture { auto queries_v = make_const_mdspan(queries_.view()); if (params_.removed_ratio > 0) { - auto filter = raft::neighbors::filtering::bitset_filter(removed_indices_bitset_.view()); + auto filter = cuvs::neighbors::filtering::bitset_filter(removed_indices_bitset_.view()); loop_on_state(state, [&]() { - raft::neighbors::cagra::search_with_filtering( + cuvs::neighbors::cagra::search_with_filtering( this->handle, search_params, *this->index_, queries_v, ind_v, dist_v, filter); }); } else { loop_on_state(state, [&]() { - raft::neighbors::cagra::search( + cuvs::neighbors::cagra::search( this->handle, search_params, *this->index_, queries_v, ind_v, dist_v); }); } @@ -145,7 +145,7 @@ struct CagraBench : public fixture { private: const params params_; - std::optional> index_; + std::optional> index_; raft::device_matrix queries_; raft::device_matrix dataset_; raft::device_matrix knn_graph_; @@ -203,4 +203,4 @@ const std::vector kCagraInputs = generate_inputs(); RAFT_BENCH_REGISTER(AnnCagra, #ValT "/" #IdxT, inputs); \ } -} // namespace raft::bench::neighbors +} // namespace cuvs::bench::neighbors diff --git a/cpp/bench/micro/neighbors/knn.cuh b/cpp/bench/micro/neighbors/knn.cuh index 55865b577..39f240cd0 100644 --- a/cpp/bench/micro/neighbors/knn.cuh +++ b/cpp/bench/micro/neighbors/knn.cuh @@ -21,10 +21,10 @@ #include +#include +#include +#include #include -#include -#include -#include #include #include @@ -38,7 +38,7 @@ #include -namespace raft::bench::spatial { +namespace cuvs::bench::spatial { struct params { /** Size of the dataset. */ @@ -141,16 +141,16 @@ template struct ivf_flat_knn { using dist_t = float; - std::optional> index; - raft::neighbors::ivf_flat::index_params index_params; - raft::neighbors::ivf_flat::search_params search_params; + std::optional> index; + cuvs::neighbors::ivf_flat::index_params index_params; + cuvs::neighbors::ivf_flat::search_params search_params; params ps; ivf_flat_knn(const raft::device_resources& handle, const params& ps, const ValT* data) : ps(ps) { index_params.n_lists = 4096; - index_params.metric = raft::distance::DistanceType::L2Expanded; - index.emplace(raft::neighbors::ivf_flat::build( + index_params.metric = cuvs::distance::DistanceType::L2Expanded; + index.emplace(cuvs::neighbors::ivf_flat::build( handle, index_params, data, IdxT(ps.n_samples), uint32_t(ps.n_dims))); } @@ -160,7 +160,7 @@ struct ivf_flat_knn { IdxT* out_idxs) { search_params.n_probes = 20; - raft::neighbors::ivf_flat::search( + cuvs::neighbors::ivf_flat::search( handle, search_params, *index, search_items, ps.n_queries, ps.k, out_idxs, out_dists); } }; @@ -169,17 +169,17 @@ template struct ivf_pq_knn { using dist_t = float; - std::optional> index; - raft::neighbors::ivf_pq::index_params index_params; - raft::neighbors::ivf_pq::search_params search_params; + std::optional> index; + cuvs::neighbors::ivf_pq::index_params index_params; + cuvs::neighbors::ivf_pq::search_params search_params; params ps; ivf_pq_knn(const raft::device_resources& handle, const params& ps, const ValT* data) : ps(ps) { index_params.n_lists = 4096; - index_params.metric = raft::distance::DistanceType::L2Expanded; + index_params.metric = cuvs::distance::DistanceType::L2Expanded; auto data_view = raft::make_device_matrix_view(data, ps.n_samples, ps.n_dims); - index.emplace(raft::neighbors::ivf_pq::build(handle, index_params, data_view)); + index.emplace(cuvs::neighbors::ivf_pq::build(handle, index_params, data_view)); } void search(const raft::device_resources& handle, @@ -193,7 +193,7 @@ struct ivf_pq_knn { auto idxs_view = raft::make_device_matrix_view(out_idxs, ps.n_queries, ps.k); auto dists_view = raft::make_device_matrix_view(out_dists, ps.n_queries, ps.k); - raft::neighbors::ivf_pq::search( + cuvs::neighbors::ivf_pq::search( handle, search_params, *index, queries_view, idxs_view, dists_view); } }; @@ -233,9 +233,9 @@ template struct ivf_flat_filter_knn { using dist_t = float; - std::optional> index; - raft::neighbors::ivf_flat::index_params index_params; - raft::neighbors::ivf_flat::search_params search_params; + std::optional> index; + cuvs::neighbors::ivf_flat::index_params index_params; + cuvs::neighbors::ivf_flat::search_params search_params; raft::core::bitset removed_indices_bitset_; params ps; @@ -243,8 +243,8 @@ struct ivf_flat_filter_knn { : ps(ps), removed_indices_bitset_(handle, ps.n_samples) { index_params.n_lists = 4096; - index_params.metric = raft::distance::DistanceType::L2Expanded; - index.emplace(raft::neighbors::ivf_flat::build( + index_params.metric = cuvs::distance::DistanceType::L2Expanded; + index.emplace(cuvs::neighbors::ivf_flat::build( handle, index_params, data, IdxT(ps.n_samples), uint32_t(ps.n_dims))); auto removed_indices = raft::make_device_vector(handle, ps.removed_ratio * ps.n_samples); @@ -265,13 +265,13 @@ struct ivf_flat_filter_knn { raft::make_device_matrix_view(search_items, ps.n_queries, ps.n_dims); auto neighbors_view = raft::make_device_matrix_view(out_idxs, ps.n_queries, ps.k); auto distance_view = raft::make_device_matrix_view(out_dists, ps.n_queries, ps.k); - auto filter = raft::neighbors::filtering::bitset_filter(removed_indices_bitset_.view()); + auto filter = cuvs::neighbors::filtering::bitset_filter(removed_indices_bitset_.view()); if (ps.removed_ratio > 0) { - raft::neighbors::ivf_flat::search_with_filtering( + cuvs::neighbors::ivf_flat::search_with_filtering( handle, search_params, *index, queries_view, neighbors_view, distance_view, filter); } else { - raft::neighbors::ivf_flat::search( + cuvs::neighbors::ivf_flat::search( handle, search_params, *index, queries_view, neighbors_view, distance_view); } } @@ -281,9 +281,9 @@ template struct ivf_pq_filter_knn { using dist_t = float; - std::optional> index; - raft::neighbors::ivf_pq::index_params index_params; - raft::neighbors::ivf_pq::search_params search_params; + std::optional> index; + cuvs::neighbors::ivf_pq::index_params index_params; + cuvs::neighbors::ivf_pq::search_params search_params; raft::core::bitset removed_indices_bitset_; params ps; @@ -291,9 +291,9 @@ struct ivf_pq_filter_knn { : ps(ps), removed_indices_bitset_(handle, ps.n_samples) { index_params.n_lists = 4096; - index_params.metric = raft::distance::DistanceType::L2Expanded; + index_params.metric = cuvs::distance::DistanceType::L2Expanded; auto data_view = raft::make_device_matrix_view(data, ps.n_samples, ps.n_dims); - index.emplace(raft::neighbors::ivf_pq::build(handle, index_params, data_view)); + index.emplace(cuvs::neighbors::ivf_pq::build(handle, index_params, data_view)); auto removed_indices = raft::make_device_vector(handle, ps.removed_ratio * ps.n_samples); thrust::sequence( @@ -315,13 +315,13 @@ struct ivf_pq_filter_knn { raft::make_device_matrix_view(out_idxs, ps.n_queries, ps.k); auto distance_view = raft::make_device_matrix_view(out_dists, ps.n_queries, ps.k); - auto filter = raft::neighbors::filtering::bitset_filter(removed_indices_bitset_.view()); + auto filter = cuvs::neighbors::filtering::bitset_filter(removed_indices_bitset_.view()); if (ps.removed_ratio > 0) { - raft::neighbors::ivf_pq::search_with_filtering( + cuvs::neighbors::ivf_pq::search_with_filtering( handle, search_params, *index, queries_view, neighbors_view, distance_view, filter); } else { - raft::neighbors::ivf_pq::search( + cuvs::neighbors::ivf_pq::search( handle, search_params, *index, queries_view, neighbors_view, distance_view); } } @@ -506,4 +506,4 @@ inline const std::vector kAllScopes{Scope::BUILD_SEARCH, Scope::SEARCH, S RAFT_BENCH_REGISTER(KNN, #ValT "/" #IdxT "/" #ImplT, inputs, strats, scope); \ } -} // namespace raft::bench::spatial +} // namespace cuvs::bench::spatial diff --git a/cpp/bench/micro/neighbors/knn/brute_force_float_int64_t.cu b/cpp/bench/micro/neighbors/knn/brute_force_float_int64_t.cu index 7df059967..80d1aa574 100644 --- a/cpp/bench/micro/neighbors/knn/brute_force_float_int64_t.cu +++ b/cpp/bench/micro/neighbors/knn/brute_force_float_int64_t.cu @@ -16,8 +16,8 @@ #include "../knn.cuh" -namespace raft::bench::spatial { +namespace cuvs::bench::spatial { KNN_REGISTER(float, int64_t, brute_force_knn, kInputs, kAllStrategies, kScopeFull); -} // namespace raft::bench::spatial +} // namespace cuvs::bench::spatial diff --git a/cpp/bench/micro/neighbors/knn/brute_force_float_uint32_t.cu b/cpp/bench/micro/neighbors/knn/brute_force_float_uint32_t.cu index 9704d39e7..b7abceb51 100644 --- a/cpp/bench/micro/neighbors/knn/brute_force_float_uint32_t.cu +++ b/cpp/bench/micro/neighbors/knn/brute_force_float_uint32_t.cu @@ -16,8 +16,8 @@ #include "../knn.cuh" -namespace raft::bench::spatial { +namespace cuvs::bench::spatial { KNN_REGISTER(float, uint32_t, brute_force_knn, kInputs, kAllStrategies, kScopeFull); -} // namespace raft::bench::spatial +} // namespace cuvs::bench::spatial diff --git a/cpp/bench/micro/neighbors/knn/cagra_float_uint32_t.cu b/cpp/bench/micro/neighbors/knn/cagra_float_uint32_t.cu index 5d762f6e8..1a96f0d89 100644 --- a/cpp/bench/micro/neighbors/knn/cagra_float_uint32_t.cu +++ b/cpp/bench/micro/neighbors/knn/cagra_float_uint32_t.cu @@ -16,8 +16,8 @@ #include "../cagra_bench.cuh" -namespace raft::bench::neighbors { +namespace cuvs::bench::neighbors { CAGRA_REGISTER(float, uint32_t, kCagraInputs); -} // namespace raft::bench::neighbors +} // namespace cuvs::bench::neighbors diff --git a/cpp/bench/micro/neighbors/knn/ivf_flat_filter_float_int64_t.cu b/cpp/bench/micro/neighbors/knn/ivf_flat_filter_float_int64_t.cu index bf5118cea..ff6c78652 100644 --- a/cpp/bench/micro/neighbors/knn/ivf_flat_filter_float_int64_t.cu +++ b/cpp/bench/micro/neighbors/knn/ivf_flat_filter_float_int64_t.cu @@ -17,8 +17,8 @@ #undef RAFT_EXPLICIT_INSTANTIATE_ONLY // Enable instantiation of search with filter #include "../knn.cuh" -namespace raft::bench::spatial { +namespace cuvs::bench::spatial { KNN_REGISTER(float, int64_t, ivf_flat_filter_knn, kInputsFilter, kNoCopyOnly, kScopeFull); -} // namespace raft::bench::spatial +} // namespace cuvs::bench::spatial diff --git a/cpp/bench/micro/neighbors/knn/ivf_flat_float_int64_t.cu b/cpp/bench/micro/neighbors/knn/ivf_flat_float_int64_t.cu index fbbb4f9ac..dfab16356 100644 --- a/cpp/bench/micro/neighbors/knn/ivf_flat_float_int64_t.cu +++ b/cpp/bench/micro/neighbors/knn/ivf_flat_float_int64_t.cu @@ -16,8 +16,8 @@ #include "../knn.cuh" -namespace raft::bench::spatial { +namespace cuvs::bench::spatial { KNN_REGISTER(float, int64_t, ivf_flat_knn, kInputs, kNoCopyOnly, kAllScopes); -} // namespace raft::bench::spatial +} // namespace cuvs::bench::spatial diff --git a/cpp/bench/micro/neighbors/knn/ivf_flat_int8_t_int64_t.cu b/cpp/bench/micro/neighbors/knn/ivf_flat_int8_t_int64_t.cu index 7067dbe1b..42e71ff33 100644 --- a/cpp/bench/micro/neighbors/knn/ivf_flat_int8_t_int64_t.cu +++ b/cpp/bench/micro/neighbors/knn/ivf_flat_int8_t_int64_t.cu @@ -16,8 +16,8 @@ #include "../knn.cuh" -namespace raft::bench::spatial { +namespace cuvs::bench::spatial { KNN_REGISTER(int8_t, int64_t, ivf_flat_knn, kInputs, kNoCopyOnly, kAllScopes); -} // namespace raft::bench::spatial +} // namespace cuvs::bench::spatial diff --git a/cpp/bench/micro/neighbors/knn/ivf_flat_uint8_t_int64_t.cu b/cpp/bench/micro/neighbors/knn/ivf_flat_uint8_t_int64_t.cu index 91fada3c2..c875e6b0d 100644 --- a/cpp/bench/micro/neighbors/knn/ivf_flat_uint8_t_int64_t.cu +++ b/cpp/bench/micro/neighbors/knn/ivf_flat_uint8_t_int64_t.cu @@ -16,8 +16,8 @@ #include "../knn.cuh" -namespace raft::bench::spatial { +namespace cuvs::bench::spatial { KNN_REGISTER(uint8_t, int64_t, ivf_flat_knn, kInputs, kNoCopyOnly, kAllScopes); -} // namespace raft::bench::spatial +} // namespace cuvs::bench::spatial diff --git a/cpp/bench/micro/neighbors/knn/ivf_pq_filter_float_int64_t.cu b/cpp/bench/micro/neighbors/knn/ivf_pq_filter_float_int64_t.cu index 9534515cb..d4ee48988 100644 --- a/cpp/bench/micro/neighbors/knn/ivf_pq_filter_float_int64_t.cu +++ b/cpp/bench/micro/neighbors/knn/ivf_pq_filter_float_int64_t.cu @@ -17,8 +17,8 @@ #undef RAFT_EXPLICIT_INSTANTIATE_ONLY // Enable instantiation of search with filter #include "../knn.cuh" -namespace raft::bench::spatial { +namespace cuvs::bench::spatial { KNN_REGISTER(float, int64_t, ivf_pq_filter_knn, kInputsFilter, kNoCopyOnly, kScopeFull); -} // namespace raft::bench::spatial +} // namespace cuvs::bench::spatial diff --git a/cpp/bench/micro/neighbors/knn/ivf_pq_float_int64_t.cu b/cpp/bench/micro/neighbors/knn/ivf_pq_float_int64_t.cu index 83c4973c3..0d668b413 100644 --- a/cpp/bench/micro/neighbors/knn/ivf_pq_float_int64_t.cu +++ b/cpp/bench/micro/neighbors/knn/ivf_pq_float_int64_t.cu @@ -16,8 +16,8 @@ #include "../knn.cuh" -namespace raft::bench::spatial { +namespace cuvs::bench::spatial { KNN_REGISTER(float, int64_t, ivf_pq_knn, kInputs, kNoCopyOnly, kAllScopes); -} // namespace raft::bench::spatial +} // namespace cuvs::bench::spatial diff --git a/cpp/bench/micro/neighbors/knn/ivf_pq_int8_t_int64_t.cu b/cpp/bench/micro/neighbors/knn/ivf_pq_int8_t_int64_t.cu index 4ea281b11..6af29d9ce 100644 --- a/cpp/bench/micro/neighbors/knn/ivf_pq_int8_t_int64_t.cu +++ b/cpp/bench/micro/neighbors/knn/ivf_pq_int8_t_int64_t.cu @@ -16,8 +16,8 @@ #include "../knn.cuh" -namespace raft::bench::spatial { +namespace cuvs::bench::spatial { KNN_REGISTER(int8_t, int64_t, ivf_pq_knn, kInputs, kNoCopyOnly, kAllScopes); -} // namespace raft::bench::spatial +} // namespace cuvs::bench::spatial diff --git a/cpp/bench/micro/neighbors/knn/ivf_pq_uint8_t_int64_t.cu b/cpp/bench/micro/neighbors/knn/ivf_pq_uint8_t_int64_t.cu index 3313a49ba..db749de99 100644 --- a/cpp/bench/micro/neighbors/knn/ivf_pq_uint8_t_int64_t.cu +++ b/cpp/bench/micro/neighbors/knn/ivf_pq_uint8_t_int64_t.cu @@ -16,8 +16,8 @@ #include "../knn.cuh" -namespace raft::bench::spatial { +namespace cuvs::bench::spatial { KNN_REGISTER(uint8_t, int64_t, ivf_pq_knn, kInputs, kNoCopyOnly, kAllScopes); -} // namespace raft::bench::spatial +} // namespace cuvs::bench::spatial diff --git a/cpp/bench/micro/neighbors/refine.cuh b/cpp/bench/micro/neighbors/refine.cuh index 18d27116f..23e522e69 100644 --- a/cpp/bench/micro/neighbors/refine.cuh +++ b/cpp/bench/micro/neighbors/refine.cuh @@ -20,11 +20,11 @@ #include +#include +#include +#include #include #include -#include -#include -#include #include #include @@ -34,9 +34,9 @@ #include #include -using namespace raft::neighbors; +using namespace cuvs::neighbors; -namespace raft::bench::neighbors { +namespace cuvs::bench::neighbors { template inline auto operator<<(std::ostream& os, const RefineInputs& p) -> std::ostream& @@ -63,7 +63,7 @@ class RefineAnn : public fixture { if (data.p.host_data) { loop_on_state(state, [this]() { - raft::neighbors::refine(handle_, + cuvs::neighbors::refine(handle_, data.dataset_host.view(), data.queries_host.view(), data.candidates_host.view(), @@ -73,7 +73,7 @@ class RefineAnn : public fixture { }); } else { loop_on_state(state, [&]() { - raft::neighbors::refine(handle_, + cuvs::neighbors::refine(handle_, data.dataset.view(), data.queries.view(), data.candidates.view(), @@ -94,7 +94,7 @@ template std::vector> getInputs() { std::vector> out; - raft::distance::DistanceType metric = raft::distance::DistanceType::L2Expanded; + cuvs::distance::DistanceType metric = cuvs::distance::DistanceType::L2Expanded; for (bool host_data : {true, false}) { for (T n_queries : {1000, 10000}) { for (T dim : {128, 512}) { @@ -106,4 +106,4 @@ std::vector> getInputs() return out; } -} // namespace raft::bench::neighbors +} // namespace cuvs::bench::neighbors diff --git a/cpp/bench/micro/neighbors/refine_float_int64_t.cu b/cpp/bench/micro/neighbors/refine_float_int64_t.cu index bbedc1ae6..665f9af2c 100644 --- a/cpp/bench/micro/neighbors/refine_float_int64_t.cu +++ b/cpp/bench/micro/neighbors/refine_float_int64_t.cu @@ -17,9 +17,9 @@ #include "refine.cuh" #include -using namespace raft::neighbors; +using namespace cuvs::neighbors; -namespace raft::bench::neighbors { +namespace cuvs::bench::neighbors { using refine_float_int64 = RefineAnn; RAFT_BENCH_REGISTER(refine_float_int64, "", getInputs()); -} // namespace raft::bench::neighbors +} // namespace cuvs::bench::neighbors diff --git a/cpp/bench/micro/neighbors/refine_uint8_t_int64_t.cu b/cpp/bench/micro/neighbors/refine_uint8_t_int64_t.cu index 4952361f0..06cf5bfad 100644 --- a/cpp/bench/micro/neighbors/refine_uint8_t_int64_t.cu +++ b/cpp/bench/micro/neighbors/refine_uint8_t_int64_t.cu @@ -17,9 +17,9 @@ #include "refine.cuh" #include -using namespace raft::neighbors; +using namespace cuvs::neighbors; -namespace raft::bench::neighbors { +namespace cuvs::bench::neighbors { using refine_uint8_int64 = RefineAnn; RAFT_BENCH_REGISTER(refine_uint8_int64, "", getInputs()); -} // namespace raft::bench::neighbors +} // namespace cuvs::bench::neighbors diff --git a/cpp/include/cuvs/cluster/detail/connectivities.cuh b/cpp/include/cuvs/cluster/detail/connectivities.cuh index 8121094c1..165058dbd 100644 --- a/cpp/include/cuvs/cluster/detail/connectivities.cuh +++ b/cpp/include/cuvs/cluster/detail/connectivities.cuh @@ -46,7 +46,7 @@ struct distance_graph_impl { const value_t* X, size_t m, size_t n, - raft::distance::DistanceType metric, + cuvs::distance::DistanceType metric, rmm::device_uvector& indptr, rmm::device_uvector& indices, rmm::device_uvector& data, @@ -59,12 +59,12 @@ struct distance_graph_impl { * @tparam value_t */ template -struct distance_graph_impl { +struct distance_graph_impl { void run(raft::resources const& handle, const value_t* X, size_t m, size_t n, - raft::distance::DistanceType metric, + cuvs::distance::DistanceType metric, rmm::device_uvector& indptr, rmm::device_uvector& indices, rmm::device_uvector& data, @@ -133,7 +133,7 @@ void pairwise_distances(const raft::resources& handle, const value_t* X, size_t m, size_t n, - raft::distance::DistanceType metric, + cuvs::distance::DistanceType metric, value_idx* indptr, value_idx* indices, value_t* data) @@ -176,12 +176,12 @@ void pairwise_distances(const raft::resources& handle, * @tparam value_t */ template -struct distance_graph_impl { +struct distance_graph_impl { void run(const raft::resources& handle, const value_t* X, size_t m, size_t n, - raft::distance::DistanceType metric, + cuvs::distance::DistanceType metric, rmm::device_uvector& indptr, rmm::device_uvector& indices, rmm::device_uvector& data, @@ -214,12 +214,12 @@ struct distance_graph_impl +template void get_distance_graph(raft::resources const& handle, const value_t* X, size_t m, size_t n, - raft::distance::DistanceType metric, + cuvs::distance::DistanceType metric, rmm::device_uvector& indptr, rmm::device_uvector& indices, rmm::device_uvector& data, diff --git a/cpp/include/cuvs/cluster/detail/kmeans.cuh b/cpp/include/cuvs/cluster/detail/kmeans.cuh index c48871245..c6b73fec9 100644 --- a/cpp/include/cuvs/cluster/detail/kmeans.cuh +++ b/cpp/include/cuvs/cluster/detail/kmeans.cuh @@ -137,8 +137,8 @@ void kmeansPlusPlus(raft::resources const& handle, // L2 norm of X: ||c||^2 auto L2NormX = raft::make_device_vector(handle, n_samples); - if (metric == raft::distance::DistanceType::L2Expanded || - metric == raft::distance::DistanceType::L2SqrtExpanded) { + if (metric == cuvs::distance::DistanceType::L2Expanded || + metric == cuvs::distance::DistanceType::L2SqrtExpanded) { raft::linalg::rowNorm(L2NormX.data_handle(), X.data_handle(), X.extent(1), @@ -400,8 +400,8 @@ void kmeans_fit_main(raft::resources const& handle, auto l2normx_view = raft::make_device_vector_view(L2NormX.data_handle(), n_samples); - if (metric == raft::distance::DistanceType::L2Expanded || - metric == raft::distance::DistanceType::L2SqrtExpanded) { + if (metric == cuvs::distance::DistanceType::L2Expanded || + metric == cuvs::distance::DistanceType::L2SqrtExpanded) { raft::linalg::rowNorm(L2NormX.data_handle(), X.data_handle(), X.extent(1), @@ -624,8 +624,8 @@ void initScalableKMeansPlusPlus(raft::resources const& handle, // L2 norm of X: ||x||^2 auto L2NormX = raft::make_device_vector(handle, n_samples); - if (metric == raft::distance::DistanceType::L2Expanded || - metric == raft::distance::DistanceType::L2SqrtExpanded) { + if (metric == cuvs::distance::DistanceType::L2Expanded || + metric == cuvs::distance::DistanceType::L2SqrtExpanded) { raft::linalg::rowNorm(L2NormX.data_handle(), X.data_handle(), X.extent(1), @@ -845,8 +845,8 @@ void kmeans_fit(raft::resources const& handle, // Display a message if the batch size is smaller than n_samples but will be ignored if (params.batch_samples < (int)n_samples && - (params.metric == raft::distance::DistanceType::L2Expanded || - params.metric == raft::distance::DistanceType::L2SqrtExpanded)) { + (params.metric == cuvs::distance::DistanceType::L2Expanded || + params.metric == cuvs::distance::DistanceType::L2SqrtExpanded)) { RAFT_LOG_DEBUG( "batch_samples=%d was passed, but batch_samples=%d will be used (reason: " "batch_samples has no impact on the memory footprint when FusedL2NN can be used)", @@ -855,8 +855,8 @@ void kmeans_fit(raft::resources const& handle, } // Display a message if batch_centroids is set and a fusedL2NN-compatible metric is used if (params.batch_centroids != 0 && params.batch_centroids != params.n_clusters && - (params.metric == raft::distance::DistanceType::L2Expanded || - params.metric == raft::distance::DistanceType::L2SqrtExpanded)) { + (params.metric == cuvs::distance::DistanceType::L2Expanded || + params.metric == cuvs::distance::DistanceType::L2SqrtExpanded)) { RAFT_LOG_DEBUG( "batch_centroids=%d was passed, but batch_centroids=%d will be used (reason: " "batch_centroids has no impact on the memory footprint when FusedL2NN can be used)", @@ -1033,8 +1033,8 @@ void kmeans_predict(raft::resources const& handle, // L2 norm of X: ||x||^2 auto L2NormX = raft::make_device_vector(handle, n_samples); - if (metric == raft::distance::DistanceType::L2Expanded || - metric == raft::distance::DistanceType::L2SqrtExpanded) { + if (metric == cuvs::distance::DistanceType::L2Expanded || + metric == cuvs::distance::DistanceType::L2SqrtExpanded) { raft::linalg::rowNorm(L2NormX.data_handle(), X.data_handle(), X.extent(1), diff --git a/cpp/include/cuvs/cluster/detail/kmeans_auto_find_k.cuh b/cpp/include/cuvs/cluster/detail/kmeans_auto_find_k.cuh index 8bc3cd95d..78566bb06 100644 --- a/cpp/include/cuvs/cluster/detail/kmeans_auto_find_k.cuh +++ b/cpp/include/cuvs/cluster/detail/kmeans_auto_find_k.cuh @@ -29,7 +29,7 @@ #include #include -namespace raft::cluster::detail { +namespace cuvs::cluster::detail { template void compute_dispersion(raft::resources const& handle, @@ -56,7 +56,7 @@ void compute_dispersion(raft::resources const& handle, params.n_clusters = val; - raft::cluster::detail::kmeans_fit_predict( + cuvs::cluster::detail::kmeans_fit_predict( handle, params, X, std::nullopt, std::make_optional(centroids_view), labels, residual, n_iter); detail::countLabels(handle, labels.data_handle(), clusterSizes.data_handle(), n, val, workspace); @@ -220,7 +220,7 @@ void find_k(raft::resources const& handle, raft::make_device_matrix_view(centroids.data_handle(), best_k[0], d); params.n_clusters = best_k[0]; - raft::cluster::detail::kmeans_fit_predict(handle, + cuvs::cluster::detail::kmeans_fit_predict(handle, params, X, std::nullopt, @@ -230,4 +230,4 @@ void find_k(raft::resources const& handle, n_iter); } } -} // namespace raft::cluster::detail \ No newline at end of file +} // namespace cuvs::cluster::detail \ No newline at end of file diff --git a/cpp/include/cuvs/cluster/detail/kmeans_balanced.cuh b/cpp/include/cuvs/cluster/detail/kmeans_balanced.cuh index 016bcafcb..b774a1b8f 100644 --- a/cpp/include/cuvs/cluster/detail/kmeans_balanced.cuh +++ b/cpp/include/cuvs/cluster/detail/kmeans_balanced.cuh @@ -96,8 +96,8 @@ inline std::enable_if_t> predict_core( { auto stream = resource::get_cuda_stream(handle); switch (params.metric) { - case raft::distance::DistanceType::L2Expanded: - case raft::distance::DistanceType::L2SqrtExpanded: { + case cuvs::distance::DistanceType::L2Expanded: + case cuvs::distance::DistanceType::L2SqrtExpanded: { auto workspace = raft::make_device_mdarray( handle, mr, make_extents((sizeof(int)) * n_rows)); @@ -114,7 +114,7 @@ inline std::enable_if_t> predict_core( raft::linalg::rowNorm( centroidsNorm.data_handle(), centers, dim, n_clusters, raft::linalg::L2Norm, true, stream); - raft::distance::fusedL2NNMinReduce, IdxT>( + cuvs::distance::fusedL2NNMinReduce, IdxT>( minClusterAndDistance.data_handle(), dataset, centers, @@ -124,7 +124,7 @@ inline std::enable_if_t> predict_core( n_clusters, dim, (void*)workspace.data_handle(), - (params.metric == raft::distance::DistanceType::L2Expanded) ? false : true, + (params.metric == cuvs::distance::DistanceType::L2Expanded) ? false : true, false, stream); @@ -137,7 +137,7 @@ inline std::enable_if_t> predict_core( raft::compose_op, raft::key_op>()); break; } - case raft::distance::DistanceType::InnerProduct: { + case cuvs::distance::DistanceType::InnerProduct: { // TODO: pass buffer rmm::device_uvector distances(n_rows * n_clusters, stream, mr); @@ -192,7 +192,7 @@ template constexpr auto calc_minibatch_size(IdxT n_clusters, IdxT n_rows, IdxT dim, - raft::distance::DistanceType metric, + cuvs::distance::DistanceType metric, bool needs_conversion) -> std::tuple { n_clusters = std::max(1, n_clusters); @@ -297,7 +297,7 @@ void calc_centers_and_sizes(const raft::resources& handle, } // Compute weight of each cluster - raft::cluster::detail::countLabels(handle, labels, temp_sizes, n_rows, n_clusters, workspace); + cuvs::cluster::detail::countLabels(handle, labels, temp_sizes, n_rows, n_clusters, workspace); // Add previous sizes if necessary if (!reset_counters) { @@ -389,8 +389,8 @@ void predict(const raft::resources& handle, rmm::device_uvector cur_dataset( std::is_same_v ? 0 : max_minibatch_size * dim, stream, mr); bool need_compute_norm = - dataset_norm == nullptr && (params.metric == raft::distance::DistanceType::L2Expanded || - params.metric == raft::distance::DistanceType::L2SqrtExpanded); + dataset_norm == nullptr && (params.metric == cuvs::distance::DistanceType::L2Expanded || + params.metric == cuvs::distance::DistanceType::L2SqrtExpanded); rmm::device_uvector cur_dataset_norm( need_compute_norm ? max_minibatch_size : 0, stream, mr); const MathT* dataset_norm_ptr = nullptr; @@ -655,9 +655,9 @@ void balancing_em_iters(const raft::resources& handle, switch (params.metric) { // For some metrics, cluster calculation and adjustment tends to favor zero center vectors. // To avoid converging to zero, we normalize the center vectors on every iteration. - case raft::distance::DistanceType::InnerProduct: - case raft::distance::DistanceType::CosineExpanded: - case raft::distance::DistanceType::CorrelationExpanded: { + case cuvs::distance::DistanceType::InnerProduct: + case cuvs::distance::DistanceType::CosineExpanded: + case cuvs::distance::DistanceType::CorrelationExpanded: { auto clusters_in_view = raft::make_device_matrix_view( cluster_centers, n_clusters, dim); auto clusters_out_view = raft::make_device_matrix_view( @@ -899,8 +899,8 @@ auto build_fine_clusters(const raft::resources& handle, cub::TransformInputIterator mapping_itr(dataset_mptr, mapping_op); raft::matrix::gather(mapping_itr, dim, n_rows, mc_trainset_ids, k, mc_trainset, stream); - if (params.metric == raft::distance::DistanceType::L2Expanded || - params.metric == raft::distance::DistanceType::L2SqrtExpanded) { + if (params.metric == cuvs::distance::DistanceType::L2Expanded || + params.metric == cuvs::distance::DistanceType::L2SqrtExpanded) { thrust::gather(resource::get_thrust_policy(handle), mc_trainset_ids, mc_trainset_ids + k, @@ -984,8 +984,8 @@ void build_hierarchical(const raft::resources& handle, // Precompute the L2 norm of the dataset if relevant. const MathT* dataset_norm = nullptr; rmm::device_uvector dataset_norm_buf(0, stream, device_memory); - if (params.metric == raft::distance::DistanceType::L2Expanded || - params.metric == raft::distance::DistanceType::L2SqrtExpanded) { + if (params.metric == cuvs::distance::DistanceType::L2Expanded || + params.metric == cuvs::distance::DistanceType::L2SqrtExpanded) { dataset_norm_buf.resize(n_rows, stream); for (IdxT offset = 0; offset < n_rows; offset += max_minibatch_size) { IdxT minibatch_size = std::min(max_minibatch_size, n_rows - offset); diff --git a/cpp/include/cuvs/cluster/detail/kmeans_common.cuh b/cpp/include/cuvs/cluster/detail/kmeans_common.cuh index 2cc5cfd89..965d8a549 100644 --- a/cpp/include/cuvs/cluster/detail/kmeans_common.cuh +++ b/cpp/include/cuvs/cluster/detail/kmeans_common.cuh @@ -294,7 +294,7 @@ void pairwise_distance_kmeans(raft::resources const& handle, raft::device_matrix_view centroids, raft::device_matrix_view pairwiseDistance, rmm::device_uvector& workspace, - raft::distance::DistanceType metric) + cuvs::distance::DistanceType metric) { auto n_samples = X.extent(0); auto n_features = X.extent(1); @@ -303,7 +303,7 @@ void pairwise_distance_kmeans(raft::resources const& handle, ASSERT(X.extent(1) == centroids.extent(1), "# features in dataset and centroids are different (must be same)"); - raft::distance::pairwise_distance(handle, + cuvs::distance::pairwise_distance(handle, X.data_handle(), centroids.data_handle(), pairwiseDistance.data_handle(), @@ -358,7 +358,7 @@ void minClusterAndDistanceCompute( raft::device_vector_view, IndexT> minClusterAndDistance, raft::device_vector_view L2NormX, rmm::device_uvector& L2NormBuf_OR_DistBuf, - raft::distance::DistanceType metric, + cuvs::distance::DistanceType metric, int batch_samples, int batch_centroids, rmm::device_uvector& workspace) @@ -368,8 +368,8 @@ void minClusterAndDistanceCompute( auto n_features = X.extent(1); auto n_clusters = centroids.extent(0); // todo(lsugy): change batch size computation when using fusedL2NN! - bool is_fused = metric == raft::distance::DistanceType::L2Expanded || - metric == raft::distance::DistanceType::L2SqrtExpanded; + bool is_fused = metric == cuvs::distance::DistanceType::L2Expanded || + metric == cuvs::distance::DistanceType::L2SqrtExpanded; auto dataBatchSize = is_fused ? (IndexT)n_samples : getDataBatchSize(batch_samples, n_samples); auto centroidsBatchSize = getCentroidsBatchSize(batch_centroids, n_clusters); @@ -426,7 +426,7 @@ void minClusterAndDistanceCompute( workspace.resize((sizeof(int)) * ns, stream); // todo(lsugy): remove cIdx - raft::distance::fusedL2NNMinReduce, IndexT>( + cuvs::distance::fusedL2NNMinReduce, IndexT>( minClusterAndDistanceView.data_handle(), datasetView.data_handle(), centroids.data_handle(), @@ -436,7 +436,7 @@ void minClusterAndDistanceCompute( n_clusters, n_features, (void*)workspace.data(), - metric != raft::distance::DistanceType::L2Expanded, + metric != cuvs::distance::DistanceType::L2Expanded, false, stream); } else { @@ -491,7 +491,7 @@ void minClusterDistanceCompute(raft::resources const& handle, raft::device_vector_view minClusterDistance, raft::device_vector_view L2NormX, rmm::device_uvector& L2NormBuf_OR_DistBuf, - raft::distance::DistanceType metric, + cuvs::distance::DistanceType metric, int batch_samples, int batch_centroids, rmm::device_uvector& workspace) @@ -501,8 +501,8 @@ void minClusterDistanceCompute(raft::resources const& handle, auto n_features = X.extent(1); auto n_clusters = centroids.extent(0); - bool is_fused = metric == raft::distance::DistanceType::L2Expanded || - metric == raft::distance::DistanceType::L2SqrtExpanded; + bool is_fused = metric == cuvs::distance::DistanceType::L2Expanded || + metric == cuvs::distance::DistanceType::L2SqrtExpanded; auto dataBatchSize = is_fused ? (IndexT)n_samples : getDataBatchSize(batch_samples, n_samples); auto centroidsBatchSize = getCentroidsBatchSize(batch_centroids, n_clusters); @@ -553,7 +553,7 @@ void minClusterDistanceCompute(raft::resources const& handle, if (is_fused) { workspace.resize((sizeof(IndexT)) * ns, stream); - raft::distance::fusedL2NNMinReduce( + cuvs::distance::fusedL2NNMinReduce( minClusterDistanceView.data_handle(), datasetView.data_handle(), centroids.data_handle(), @@ -563,7 +563,7 @@ void minClusterDistanceCompute(raft::resources const& handle, n_clusters, n_features, (void*)workspace.data(), - metric != raft::distance::DistanceType::L2Expanded, + metric != cuvs::distance::DistanceType::L2Expanded, false, stream); } else { diff --git a/cpp/include/cuvs/cluster/detail/mst.cuh b/cpp/include/cuvs/cluster/detail/mst.cuh index 768f57580..6d304d64c 100644 --- a/cpp/include/cuvs/cluster/detail/mst.cuh +++ b/cpp/include/cuvs/cluster/detail/mst.cuh @@ -75,7 +75,7 @@ void connect_knn_graph( size_t n, value_idx* color, red_op reduction_op, - raft::distance::DistanceType metric = raft::distance::DistanceType::L2SqrtExpanded) + cuvs::distance::DistanceType metric = cuvs::distance::DistanceType::L2SqrtExpanded) { auto stream = resource::get_cuda_stream(handle); @@ -156,7 +156,7 @@ void build_sorted_mst( value_idx* color, size_t nnz, red_op reduction_op, - raft::distance::DistanceType metric = raft::distance::DistanceType::L2SqrtExpanded, + cuvs::distance::DistanceType metric = cuvs::distance::DistanceType::L2SqrtExpanded, int max_iter = 10) { auto stream = resource::get_cuda_stream(handle); diff --git a/cpp/include/cuvs/cluster/detail/single_linkage.cuh b/cpp/include/cuvs/cluster/detail/single_linkage.cuh index 68ef82b94..5eb5ffb61 100644 --- a/cpp/include/cuvs/cluster/detail/single_linkage.cuh +++ b/cpp/include/cuvs/cluster/detail/single_linkage.cuh @@ -54,7 +54,7 @@ void single_linkage(raft::resources const& handle, const value_t* X, size_t m, size_t n, - raft::distance::DistanceType metric, + cuvs::distance::DistanceType metric, linkage_output* out, int c, size_t n_clusters) diff --git a/cpp/include/cuvs/cluster/kmeans.cuh b/cpp/include/cuvs/cluster/kmeans.cuh index df7d18d22..e773a09ea 100644 --- a/cpp/include/cuvs/cluster/kmeans.cuh +++ b/cpp/include/cuvs/cluster/kmeans.cuh @@ -47,12 +47,12 @@ using KeyValueIndexOp = detail::KeyValueIndexOp; * * @code{.cpp} * #include - * #include - * #include - * using namespace raft::cluster; + * #include + * #include + * using namespace cuvs::cluster; * ... * raft::raft::resources handle; - * raft::cluster::KMeansParams params; + * cuvs::cluster::KMeansParams params; * int n_features = 15, inertia, n_iter; * auto centroids = raft::make_device_matrix(handle, params.n_clusters, n_features); * @@ -101,12 +101,12 @@ void fit(raft::resources const& handle, * * @code{.cpp} * #include - * #include - * #include - * using namespace raft::cluster; + * #include + * #include + * using namespace cuvs::cluster; * ... * raft::raft::resources handle; - * raft::cluster::KMeansParams params; + * cuvs::cluster::KMeansParams params; * int n_features = 15, inertia, n_iter; * auto centroids = raft::make_device_matrix(handle, params.n_clusters, n_features); * @@ -168,12 +168,12 @@ void predict(raft::resources const& handle, * * @code{.cpp} * #include - * #include - * #include - * using namespace raft::cluster; + * #include + * #include + * using namespace cuvs::cluster; * ... * raft::raft::resources handle; - * raft::cluster::KMeansParams params; + * cuvs::cluster::KMeansParams params; * int n_features = 15, inertia, n_iter; * auto centroids = raft::make_device_matrix(handle, params.n_clusters, n_features); * auto labels = raft::make_device_vector(handle, X.extent(0)); @@ -269,12 +269,12 @@ void transform(raft::resources const& handle, * * @code{.cpp} * #include - * #include - * #include + * #include + * #include * * #include * - * using namespace raft::cluster; + * using namespace cuvs::cluster; * * raft::handle_t handle; * int n_samples = 100, n_features = 15, n_clusters = 10; @@ -437,7 +437,7 @@ void min_cluster_distance(raft::resources const& handle, raft::device_vector_view minClusterDistance, raft::device_vector_view L2NormX, rmm::device_uvector& L2NormBuf_OR_DistBuf, - raft::distance::DistanceType metric, + cuvs::distance::DistanceType metric, int batch_samples, int batch_centroids, rmm::device_uvector& workspace) @@ -488,7 +488,7 @@ void min_cluster_and_distance( raft::device_vector_view, IndexT> minClusterAndDistance, raft::device_vector_view L2NormX, rmm::device_uvector& L2NormBuf_OR_DistBuf, - raft::distance::DistanceType metric, + cuvs::distance::DistanceType metric, int batch_samples, int batch_centroids, rmm::device_uvector& workspace) @@ -629,11 +629,11 @@ void fit_main(raft::resources const& handle, }; // namespace cuvs::cluster::kmeans -namespace raft::cluster { +namespace cuvs::cluster { /** - * Note: All of the functions below in raft::cluster are deprecated and will - * be removed in a future release. Please use raft::cluster::kmeans instead. + * Note: All of the functions below in cuvs::cluster are deprecated and will + * be removed in a future release. Please use cuvs::cluster::kmeans instead. */ /** @@ -1113,4 +1113,4 @@ void kmeans_fit_main(raft::resources const& handle, kmeans::fit_main( handle, params, X, weight, centroidsRawData, inertia, n_iter, workspace); } -}; // namespace raft::cluster +}; // namespace cuvs::cluster diff --git a/cpp/include/cuvs/cluster/kmeans_balanced.cuh b/cpp/include/cuvs/cluster/kmeans_balanced.cuh index d44e70296..7735587e7 100644 --- a/cpp/include/cuvs/cluster/kmeans_balanced.cuh +++ b/cpp/include/cuvs/cluster/kmeans_balanced.cuh @@ -51,13 +51,13 @@ namespace cuvs::cluster::kmeans_balanced { * * @code{.cpp} * #include - * #include - * #include + * #include + * #include * ... * raft::handle_t handle; - * raft::cluster::kmeans_balanced_params params; + * cuvs::cluster::kmeans_balanced_params params; * auto centroids = raft::make_device_matrix(handle, n_clusters, n_features); - * raft::cluster::kmeans_balanced::fit(handle, params, X, centroids.view()); + * cuvs::cluster::kmeans_balanced::fit(handle, params, X, centroids.view()); * @endcode * * @tparam DataT Type of the input data. @@ -103,13 +103,13 @@ void fit(const raft::resources& handle, * * @code{.cpp} * #include - * #include - * #include + * #include + * #include * ... * raft::handle_t handle; - * raft::cluster::kmeans_balanced_params params; + * cuvs::cluster::kmeans_balanced_params params; * auto labels = raft::make_device_vector(handle, n_rows); - * raft::cluster::kmeans_balanced::predict(handle, params, X, centroids, labels); + * cuvs::cluster::kmeans_balanced::predict(handle, params, X, centroids, labels); * @endcode * * @tparam DataT Type of the input data. @@ -166,14 +166,14 @@ void predict(const raft::resources& handle, * * @code{.cpp} * #include - * #include - * #include + * #include + * #include * ... * raft::handle_t handle; - * raft::cluster::kmeans_balanced_params params; + * cuvs::cluster::kmeans_balanced_params params; * auto centroids = raft::make_device_matrix(handle, n_clusters, n_features); * auto labels = raft::make_device_vector(handle, n_rows); - * raft::cluster::kmeans_balanced::fit_predict( + * cuvs::cluster::kmeans_balanced::fit_predict( * handle, params, X, centroids.view(), labels.view()); * @endcode * @@ -205,8 +205,8 @@ void fit_predict(const raft::resources& handle, { auto centroids_const = raft::make_device_matrix_view( centroids.data_handle(), centroids.extent(0), centroids.extent(1)); - raft::cluster::kmeans_balanced::fit(handle, params, X, centroids, mapping_op); - raft::cluster::kmeans_balanced::predict(handle, params, X, centroids_const, labels, mapping_op); + cuvs::cluster::kmeans_balanced::fit(handle, params, X, centroids, mapping_op); + cuvs::cluster::kmeans_balanced::predict(handle, params, X, centroids_const, labels, mapping_op); } namespace helpers { @@ -220,15 +220,15 @@ namespace helpers { * * @code{.cpp} * #include - * #include - * #include + * #include + * #include * ... * raft::handle_t handle; - * raft::cluster::kmeans_balanced_params params; + * cuvs::cluster::kmeans_balanced_params params; * auto centroids = raft::make_device_matrix(handle, n_clusters, n_features); * auto labels = raft::make_device_vector(handle, n_samples); * auto sizes = raft::make_device_vector(handle, n_clusters); - * raft::cluster::kmeans_balanced::build_clusters( + * cuvs::cluster::kmeans_balanced::build_clusters( * handle, params, X, centroids.view(), labels.view(), sizes.view()); * @endcode * @@ -301,12 +301,12 @@ void build_clusters(const raft::resources& handle, * * @code{.cpp} * #include - * #include + * #include * ... * raft::handle_t handle; * auto centroids = raft::make_device_matrix(handle, n_clusters, n_features); * auto sizes = raft::make_device_vector(handle, n_clusters); - * raft::cluster::kmeans_balanced::calc_centers_and_sizes( + * cuvs::cluster::kmeans_balanced::calc_centers_and_sizes( * handle, X, labels, centroids.view(), sizes.view(), true); * @endcode * diff --git a/cpp/include/cuvs/cluster/kmeans_balanced_types.hpp b/cpp/include/cuvs/cluster/kmeans_balanced_types.hpp index c94b578b2..5a4793fbe 100644 --- a/cpp/include/cuvs/cluster/kmeans_balanced_types.hpp +++ b/cpp/include/cuvs/cluster/kmeans_balanced_types.hpp @@ -40,8 +40,8 @@ struct kmeans_balanced_params : kmeans_base_params { } // namespace cuvs::cluster::kmeans_balanced -namespace raft::cluster { +namespace cuvs::cluster { using kmeans_balanced::kmeans_balanced_params; -} // namespace raft::cluster +} // namespace cuvs::cluster diff --git a/cpp/include/cuvs/cluster/kmeans_types.hpp b/cpp/include/cuvs/cluster/kmeans_types.hpp index 3e40bb09a..c9090166d 100644 --- a/cpp/include/cuvs/cluster/kmeans_types.hpp +++ b/cpp/include/cuvs/cluster/kmeans_types.hpp @@ -18,19 +18,19 @@ #include #include -namespace raft::cluster { +namespace cuvs::cluster { /** Base structure for parameters that are common to all k-means algorithms */ struct kmeans_base_params { /** * Metric to use for distance computation. The supported metrics can vary per algorithm. */ - raft::distance::DistanceType metric = raft::distance::DistanceType::L2Expanded; + cuvs::distance::DistanceType metric = cuvs::distance::DistanceType::L2Expanded; }; -} // namespace raft::cluster +} // namespace cuvs::cluster -namespace raft::cluster::kmeans { +namespace cuvs::cluster::kmeans { /** * Simple object to specify hyper-parameters to the kmeans algorithm. @@ -113,10 +113,10 @@ struct KMeansParams : kmeans_base_params { bool inertia_check = false; }; -} // namespace raft::cluster::kmeans +} // namespace cuvs::cluster::kmeans -namespace raft::cluster { +namespace cuvs::cluster { using kmeans::KMeansParams; -} // namespace raft::cluster +} // namespace cuvs::cluster diff --git a/cpp/include/cuvs/cluster/single_linkage.cuh b/cpp/include/cuvs/cluster/single_linkage.cuh index 224c025bf..88c964678 100644 --- a/cpp/include/cuvs/cluster/single_linkage.cuh +++ b/cpp/include/cuvs/cluster/single_linkage.cuh @@ -19,11 +19,11 @@ #include #include -namespace raft::cluster { +namespace cuvs::cluster { /** - * Note: All of the functions below in the raft::cluster namespace are deprecated - * and will be removed in a future release. Please use raft::cluster::hierarchy + * Note: All of the functions below in the cuvs::cluster namespace are deprecated + * and will be removed in a future release. Please use cuvs::cluster::hierarchy * instead. */ @@ -54,7 +54,7 @@ void single_linkage(raft::resources const& handle, const value_t* X, size_t m, size_t n, - raft::distance::DistanceType metric, + cuvs::distance::DistanceType metric, linkage_output* out, int c, size_t n_clusters) @@ -62,9 +62,9 @@ void single_linkage(raft::resources const& handle, detail::single_linkage( handle, X, m, n, metric, out, c, n_clusters); } -}; // namespace raft::cluster +}; // namespace cuvs::cluster -namespace raft::cluster::hierarchy { +namespace cuvs::cluster::hierarchy { constexpr int DEFAULT_CONST_C = 15; @@ -91,7 +91,7 @@ void single_linkage(raft::resources const& handle, raft::device_matrix_view X, raft::device_matrix_view dendrogram, raft::device_vector_view labels, - raft::distance::DistanceType metric, + cuvs::distance::DistanceType metric, size_t n_clusters, std::optional c = std::make_optional(DEFAULT_CONST_C)) { @@ -99,7 +99,7 @@ void single_linkage(raft::resources const& handle, out_arrs.children = dendrogram.data_handle(); out_arrs.labels = labels.data_handle(); - raft::cluster::single_linkage( + cuvs::cluster::single_linkage( handle, X.data_handle(), static_cast(X.extent(0)), @@ -109,4 +109,4 @@ void single_linkage(raft::resources const& handle, c.has_value() ? c.value() : DEFAULT_CONST_C, n_clusters); } -}; // namespace raft::cluster::hierarchy +}; // namespace cuvs::cluster::hierarchy diff --git a/cpp/include/cuvs/distance/detail/compress_to_bits.cuh b/cpp/include/cuvs/distance/detail/compress_to_bits.cuh index 5ffb717c4..9ce47774a 100644 --- a/cpp/include/cuvs/distance/detail/compress_to_bits.cuh +++ b/cpp/include/cuvs/distance/detail/compress_to_bits.cuh @@ -20,7 +20,7 @@ #include #include -namespace raft::distance::detail { +namespace cuvs::distance::detail { /** * @brief Compress 2D boolean matrix to bitfield @@ -120,4 +120,4 @@ void compress_to_bits(raft::resources const& handle, RAFT_CUDA_TRY(cudaGetLastError()); } -}; // namespace raft::distance::detail +}; // namespace cuvs::distance::detail diff --git a/cpp/include/cuvs/distance/detail/distance.cuh b/cpp/include/cuvs/distance/detail/distance.cuh index b6885808c..014ce25dc 100644 --- a/cpp/include/cuvs/distance/detail/distance.cuh +++ b/cpp/include/cuvs/distance/detail/distance.cuh @@ -16,13 +16,13 @@ #pragma once +#include +#include +#include +#include +#include #include #include -#include -#include -#include -#include -#include #include #include #include @@ -688,7 +688,7 @@ void distance_impl(raft::resources const& handle, * as follows: 
OutType fin_op(AccType in, int g_idx);
. If one needs * any other parameters, feel free to pass them via closure. */ -template +#include // The distance operations: -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include diff --git a/cpp/include/cuvs/distance/detail/distance_ops/canberra.cuh b/cpp/include/cuvs/distance/detail/distance_ops/canberra.cuh index eaf37b7e9..8bbdc9945 100644 --- a/cpp/include/cuvs/distance/detail/distance_ops/canberra.cuh +++ b/cpp/include/cuvs/distance/detail/distance_ops/canberra.cuh @@ -19,7 +19,7 @@ #include // raft::abs #include // DI -namespace raft::distance::detail::ops { +namespace cuvs::distance::detail::ops { /** * @brief The canberra distance matrix calculation @@ -68,4 +68,4 @@ struct canberra_distance_op { } }; -} // namespace raft::distance::detail::ops +} // namespace cuvs::distance::detail::ops diff --git a/cpp/include/cuvs/distance/detail/distance_ops/correlation.cuh b/cpp/include/cuvs/distance/detail/distance_ops/correlation.cuh index 4fc4bb829..f033f3dfa 100644 --- a/cpp/include/cuvs/distance/detail/distance_ops/correlation.cuh +++ b/cpp/include/cuvs/distance/detail/distance_ops/correlation.cuh @@ -18,7 +18,7 @@ #include // DI -namespace raft::distance::detail::ops { +namespace cuvs::distance::detail::ops { /** @brief The correlation distance * @@ -123,4 +123,4 @@ struct correlation_distance_op { } }; -} // namespace raft::distance::detail::ops +} // namespace cuvs::distance::detail::ops diff --git a/cpp/include/cuvs/distance/detail/distance_ops/cosine.cuh b/cpp/include/cuvs/distance/detail/distance_ops/cosine.cuh index 0883136c9..d48731651 100644 --- a/cpp/include/cuvs/distance/detail/distance_ops/cosine.cuh +++ b/cpp/include/cuvs/distance/detail/distance_ops/cosine.cuh @@ -18,7 +18,7 @@ #include // DI -namespace raft::distance::detail::ops { +namespace cuvs::distance::detail::ops { // Epilogue operator for CUTLASS based kernel template @@ -82,4 +82,4 @@ struct cosine_distance_op { } }; -} // namespace raft::distance::detail::ops +} // namespace cuvs::distance::detail::ops diff --git a/cpp/include/cuvs/distance/detail/distance_ops/cutlass.cuh b/cpp/include/cuvs/distance/detail/distance_ops/cutlass.cuh index 68e843c6f..6d928314d 100644 --- a/cpp/include/cuvs/distance/detail/distance_ops/cutlass.cuh +++ b/cpp/include/cuvs/distance/detail/distance_ops/cutlass.cuh @@ -19,7 +19,7 @@ #include // std::false_type #include // std::declval -namespace raft::distance::detail::ops { +namespace cuvs::distance::detail::ops { // This file defines the named requirement "has_cutlass_op" that can be used to // determine if a distance operation has a CUTLASS op that can be used to pass @@ -37,4 +37,4 @@ template struct has_cutlass_op().get_cutlass_op())>> : std::true_type {}; -} // namespace raft::distance::detail::ops +} // namespace cuvs::distance::detail::ops diff --git a/cpp/include/cuvs/distance/detail/distance_ops/hamming.cuh b/cpp/include/cuvs/distance/detail/distance_ops/hamming.cuh index 475b8892e..7c6553f38 100644 --- a/cpp/include/cuvs/distance/detail/distance_ops/hamming.cuh +++ b/cpp/include/cuvs/distance/detail/distance_ops/hamming.cuh @@ -18,7 +18,7 @@ #include // DI -namespace raft::distance::detail::ops { +namespace cuvs::distance::detail::ops { /** * @brief the Hamming Unexpanded distance matrix calculation @@ -70,4 +70,4 @@ struct hamming_distance_op { } }; -} // namespace raft::distance::detail::ops +} // namespace cuvs::distance::detail::ops diff --git a/cpp/include/cuvs/distance/detail/distance_ops/hellinger.cuh b/cpp/include/cuvs/distance/detail/distance_ops/hellinger.cuh index 0489b4585..ad5ca3156 100644 --- a/cpp/include/cuvs/distance/detail/distance_ops/hellinger.cuh +++ b/cpp/include/cuvs/distance/detail/distance_ops/hellinger.cuh @@ -17,7 +17,7 @@ #pragma once #include // DI -namespace raft::distance::detail::ops { +namespace cuvs::distance::detail::ops { /** * @brief the Hellinger distance matrix calculation @@ -74,4 +74,4 @@ struct hellinger_distance_op { } }; -} // namespace raft::distance::detail::ops +} // namespace cuvs::distance::detail::ops diff --git a/cpp/include/cuvs/distance/detail/distance_ops/jensen_shannon.cuh b/cpp/include/cuvs/distance/detail/distance_ops/jensen_shannon.cuh index e46c63734..216639494 100644 --- a/cpp/include/cuvs/distance/detail/distance_ops/jensen_shannon.cuh +++ b/cpp/include/cuvs/distance/detail/distance_ops/jensen_shannon.cuh @@ -18,7 +18,7 @@ #include // raft::log #include // DI -namespace raft::distance::detail::ops { +namespace cuvs::distance::detail::ops { // Describes the computation the jensen_shannon distance @@ -78,4 +78,4 @@ struct jensen_shannon_distance_op { } }; -} // namespace raft::distance::detail::ops +} // namespace cuvs::distance::detail::ops diff --git a/cpp/include/cuvs/distance/detail/distance_ops/kl_divergence.cuh b/cpp/include/cuvs/distance/detail/distance_ops/kl_divergence.cuh index d083c5ddc..929c3a559 100644 --- a/cpp/include/cuvs/distance/detail/distance_ops/kl_divergence.cuh +++ b/cpp/include/cuvs/distance/detail/distance_ops/kl_divergence.cuh @@ -18,7 +18,7 @@ #include // raft::log #include // DI -namespace raft::distance::detail::ops { +namespace cuvs::distance::detail::ops { /** * @brief the KL Divergence distance matrix calculation @@ -96,4 +96,4 @@ struct kl_divergence_op { } } }; -} // namespace raft::distance::detail::ops +} // namespace cuvs::distance::detail::ops diff --git a/cpp/include/cuvs/distance/detail/distance_ops/l1.cuh b/cpp/include/cuvs/distance/detail/distance_ops/l1.cuh index 7e86fd360..76eaffaf3 100644 --- a/cpp/include/cuvs/distance/detail/distance_ops/l1.cuh +++ b/cpp/include/cuvs/distance/detail/distance_ops/l1.cuh @@ -17,7 +17,7 @@ #pragma once #include // DI -namespace raft::distance::detail::ops { +namespace cuvs::distance::detail::ops { /** * @brief the L1 distance matrix calculation @@ -59,4 +59,4 @@ struct l1_distance_op { }; }; -} // namespace raft::distance::detail::ops +} // namespace cuvs::distance::detail::ops diff --git a/cpp/include/cuvs/distance/detail/distance_ops/l2_exp.cuh b/cpp/include/cuvs/distance/detail/distance_ops/l2_exp.cuh index a218c85a0..f45c41206 100644 --- a/cpp/include/cuvs/distance/detail/distance_ops/l2_exp.cuh +++ b/cpp/include/cuvs/distance/detail/distance_ops/l2_exp.cuh @@ -19,7 +19,7 @@ #include #include // DI -namespace raft::distance::detail::ops { +namespace cuvs::distance::detail::ops { /** * Reserve 1 digit of precision from each floating-point type @@ -133,4 +133,4 @@ struct l2_exp_distance_op { } }; -} // namespace raft::distance::detail::ops +} // namespace cuvs::distance::detail::ops diff --git a/cpp/include/cuvs/distance/detail/distance_ops/l2_unexp.cuh b/cpp/include/cuvs/distance/detail/distance_ops/l2_unexp.cuh index 62c212ee8..aa6cc27f3 100644 --- a/cpp/include/cuvs/distance/detail/distance_ops/l2_unexp.cuh +++ b/cpp/include/cuvs/distance/detail/distance_ops/l2_unexp.cuh @@ -18,7 +18,7 @@ #include // DI -namespace raft::distance::detail::ops { +namespace cuvs::distance::detail::ops { /** * @brief the unexpanded euclidean distance matrix calculation @@ -76,4 +76,4 @@ struct l2_unexp_distance_op { }; }; -} // namespace raft::distance::detail::ops +} // namespace cuvs::distance::detail::ops diff --git a/cpp/include/cuvs/distance/detail/distance_ops/l_inf.cuh b/cpp/include/cuvs/distance/detail/distance_ops/l_inf.cuh index 88853a308..d8f9384d7 100644 --- a/cpp/include/cuvs/distance/detail/distance_ops/l_inf.cuh +++ b/cpp/include/cuvs/distance/detail/distance_ops/l_inf.cuh @@ -18,7 +18,7 @@ #include // DI -namespace raft::distance::detail::ops { +namespace cuvs::distance::detail::ops { /** * @brief the L_inf (Chebyshev) distance matrix calculation @@ -64,4 +64,4 @@ struct l_inf_distance_op { } }; -} // namespace raft::distance::detail::ops +} // namespace cuvs::distance::detail::ops diff --git a/cpp/include/cuvs/distance/detail/distance_ops/lp_unexp.cuh b/cpp/include/cuvs/distance/detail/distance_ops/lp_unexp.cuh index 290f4af1b..6136f9f3e 100644 --- a/cpp/include/cuvs/distance/detail/distance_ops/lp_unexp.cuh +++ b/cpp/include/cuvs/distance/detail/distance_ops/lp_unexp.cuh @@ -18,7 +18,7 @@ #include // raft::pow, raft::abs #include // DI -namespace raft::distance::detail::ops { +namespace cuvs::distance::detail::ops { /** * @brief the unexpanded Lp (Minkowski) distance matrix calculation @@ -75,4 +75,4 @@ struct lp_unexp_distance_op { } }; -} // namespace raft::distance::detail::ops +} // namespace cuvs::distance::detail::ops diff --git a/cpp/include/cuvs/distance/detail/distance_ops/russel_rao.cuh b/cpp/include/cuvs/distance/detail/distance_ops/russel_rao.cuh index 63dbf350d..5dffdcdb8 100644 --- a/cpp/include/cuvs/distance/detail/distance_ops/russel_rao.cuh +++ b/cpp/include/cuvs/distance/detail/distance_ops/russel_rao.cuh @@ -18,7 +18,7 @@ #include // DI -namespace raft::distance::detail::ops { +namespace cuvs::distance::detail::ops { /** * @brief the Russell Rao distance matrix calculation @@ -71,4 +71,4 @@ struct russel_rao_distance_op { } }; -} // namespace raft::distance::detail::ops +} // namespace cuvs::distance::detail::ops diff --git a/cpp/include/cuvs/distance/detail/distance_ops/template.cuh b/cpp/include/cuvs/distance/detail/distance_ops/template.cuh index 432006836..bdb933237 100644 --- a/cpp/include/cuvs/distance/detail/distance_ops/template.cuh +++ b/cpp/include/cuvs/distance/detail/distance_ops/template.cuh @@ -18,7 +18,7 @@ #include // DI -namespace raft::distance::detail::ops { +namespace cuvs::distance::detail::ops { // Describes the computation the template distance // @@ -65,4 +65,4 @@ struct template_distance_op { constexpr l2_exp_cutlass_op get_cutlass_op() const { TODO; } }; -} // namespace raft::distance::detail::ops +} // namespace cuvs::distance::detail::ops diff --git a/cpp/include/cuvs/distance/detail/fused_distance_nn/cutlass_base.cuh b/cpp/include/cuvs/distance/detail/fused_distance_nn/cutlass_base.cuh index a1cf1a9b1..da46dc69a 100644 --- a/cpp/include/cuvs/distance/detail/fused_distance_nn/cutlass_base.cuh +++ b/cpp/include/cuvs/distance/detail/fused_distance_nn/cutlass_base.cuh @@ -37,8 +37,8 @@ #include #include -#include // FusedDistanceNNEpilogueElementwise -#include // FusedDistanceNNGemm +#include // FusedDistanceNNEpilogueElementwise +#include // FusedDistanceNNGemm #include // getMultiProcessorCount #include // RAFT_CUTLASS_TRY diff --git a/cpp/include/cuvs/distance/detail/fused_distance_nn/epilogue.cuh b/cpp/include/cuvs/distance/detail/fused_distance_nn/epilogue.cuh index 8a0bea346..7053f2702 100644 --- a/cpp/include/cuvs/distance/detail/fused_distance_nn/epilogue.cuh +++ b/cpp/include/cuvs/distance/detail/fused_distance_nn/epilogue.cuh @@ -69,10 +69,10 @@ operation. #include #include #include -#include +#include -#include -#include +#include +#include //////////////////////////////////////////////////////////////////////////////// diff --git a/cpp/include/cuvs/distance/detail/fused_distance_nn/gemm.h b/cpp/include/cuvs/distance/detail/fused_distance_nn/gemm.h index 3da8b3ee3..fd5956a57 100644 --- a/cpp/include/cuvs/distance/detail/fused_distance_nn/gemm.h +++ b/cpp/include/cuvs/distance/detail/fused_distance_nn/gemm.h @@ -52,8 +52,8 @@ #include #include -#include -#include +#include +#include ///////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/cpp/include/cuvs/distance/detail/fused_l2_nn.cuh b/cpp/include/cuvs/distance/detail/fused_l2_nn.cuh index 2468dcd74..c05425028 100644 --- a/cpp/include/cuvs/distance/detail/fused_l2_nn.cuh +++ b/cpp/include/cuvs/distance/detail/fused_l2_nn.cuh @@ -17,12 +17,12 @@ #pragma once #include // size_t -#include // std::numeric_limits -#include // raft::KeyValuePair -#include // raft::identity_op -#include // ops::l2_exp_distance_op -#include -#include // PairwiseDistances +#include // ops::l2_exp_distance_op +#include +#include // PairwiseDistances +#include // std::numeric_limits +#include // raft::KeyValuePair +#include // raft::identity_op #include // Policy #include // raft::util::arch::SM_* #include // raft::ceildiv, raft::shfl @@ -336,7 +336,7 @@ void fusedL2NNImpl(OutT* min, if (cutlass_range.contains(runtime_arch)) { // If device is SM_80 or later, use CUTLASS-based kernel. - using L2Op = raft::distance::detail::ops::l2_exp_cutlass_op; + using L2Op = cuvs::distance::detail::ops::l2_exp_cutlass_op; using kvp_cg_min_reduce_op_ = kvp_cg_min_reduce_op; kvp_cg_min_reduce_op_ cg_reduce_op; L2Op L2_dist_op(sqrt); diff --git a/cpp/include/cuvs/distance/detail/kernels/gram_matrix.cuh b/cpp/include/cuvs/distance/detail/kernels/gram_matrix.cuh index e121c1be9..1f4424ea9 100644 --- a/cpp/include/cuvs/distance/detail/kernels/gram_matrix.cuh +++ b/cpp/include/cuvs/distance/detail/kernels/gram_matrix.cuh @@ -16,11 +16,11 @@ #pragma once +#include +#include #include #include #include -#include -#include // #include #include #include @@ -28,7 +28,7 @@ #include #include -namespace raft::distance::kernels::detail { +namespace cuvs::distance::kernels::detail { template using dense_input_matrix_view_t = raft::device_matrix_view; @@ -476,14 +476,14 @@ class GramMatrixBase { auto out_row_major = raft::make_device_matrix_view( out.data_handle(), out.extent(1), out.extent(0)); raft::sparse::distance::pairwise_distance( - handle, x2, x1, out_row_major, raft::distance::DistanceType::InnerProduct, 0.0); + handle, x2, x1, out_row_major, cuvs::distance::DistanceType::InnerProduct, 0.0); } else { auto out_row_major = raft::make_device_matrix_view( out.data_handle(), out.extent(0), out.extent(1)); raft::sparse::distance::pairwise_distance( - handle, x1, x2, out_row_major, raft::distance::DistanceType::InnerProduct, 0.0); + handle, x1, x2, out_row_major, cuvs::distance::DistanceType::InnerProduct, 0.0); } } }; -}; // end namespace raft::distance::kernels::detail +}; // end namespace cuvs::distance::kernels::detail diff --git a/cpp/include/cuvs/distance/detail/kernels/kernel_factory.cuh b/cpp/include/cuvs/distance/detail/kernels/kernel_factory.cuh index bb3ff1c2f..d0f1f5569 100644 --- a/cpp/include/cuvs/distance/detail/kernels/kernel_factory.cuh +++ b/cpp/include/cuvs/distance/detail/kernels/kernel_factory.cuh @@ -18,10 +18,10 @@ #include "gram_matrix.cuh" #include "kernel_matrices.cuh" -#include +#include #include -namespace raft::distance::kernels::detail { +namespace cuvs::distance::kernels::detail { template class KernelFactory { @@ -61,4 +61,4 @@ class KernelFactory { } }; -}; // end namespace raft::distance::kernels::detail +}; // end namespace cuvs::distance::kernels::detail diff --git a/cpp/include/cuvs/distance/detail/kernels/kernel_matrices.cuh b/cpp/include/cuvs/distance/detail/kernels/kernel_matrices.cuh index 8d5b2c766..1f9db896e 100644 --- a/cpp/include/cuvs/distance/detail/kernels/kernel_matrices.cuh +++ b/cpp/include/cuvs/distance/detail/kernels/kernel_matrices.cuh @@ -19,13 +19,13 @@ #include "gram_matrix.cuh" #include -#include -#include +#include +#include #include #include #include -namespace raft::distance::kernels::detail { +namespace cuvs::distance::kernels::detail { /** Epiloge function for polynomial kernel without padding. * Calculates output = (gain*in + offset)^exponent @@ -755,7 +755,7 @@ class RBFKernel : public GramMatrixBase { raft::resources handle; resource::set_cuda_stream(handle, stream); - raft::distance::distance { } }; -}; // end namespace raft::distance::kernels::detail +}; // end namespace cuvs::distance::kernels::detail diff --git a/cpp/include/cuvs/distance/detail/kernels/rbf_fin_op.cuh b/cpp/include/cuvs/distance/detail/kernels/rbf_fin_op.cuh index cd1967547..73588baea 100644 --- a/cpp/include/cuvs/distance/detail/kernels/rbf_fin_op.cuh +++ b/cpp/include/cuvs/distance/detail/kernels/rbf_fin_op.cuh @@ -20,7 +20,7 @@ * This file defines rbf_fin_op, which is used in GramMatrixBase. * * This struct has been moved to a separate file, so that it is cheap to include - * in distance/distance-ext.cuh, where an instance of raft::distance::distance + * in distance/distance-ext.cuh, where an instance of cuvs::distance::distance * with the rbf_fin_op is instantiated. * */ @@ -28,7 +28,7 @@ #include // raft::exp #include // HD -namespace raft::distance::kernels::detail { +namespace cuvs::distance::kernels::detail { /** @brief: Final op for Gram matrix with RBF kernel. * @@ -48,4 +48,4 @@ struct rbf_fin_op { } }; // struct rbf_fin_op -} // namespace raft::distance::kernels::detail +} // namespace cuvs::distance::kernels::detail diff --git a/cpp/include/cuvs/distance/detail/masked_distance_base.cuh b/cpp/include/cuvs/distance/detail/masked_distance_base.cuh index 55da63414..be95abe04 100644 --- a/cpp/include/cuvs/distance/detail/masked_distance_base.cuh +++ b/cpp/include/cuvs/distance/detail/masked_distance_base.cuh @@ -14,7 +14,7 @@ * limitations under the License. */ #pragma once -#include +#include #include #include diff --git a/cpp/include/cuvs/distance/detail/masked_nn.cuh b/cpp/include/cuvs/distance/detail/masked_nn.cuh index 4de9f4764..69f98bb49 100644 --- a/cpp/include/cuvs/distance/detail/masked_nn.cuh +++ b/cpp/include/cuvs/distance/detail/masked_nn.cuh @@ -21,9 +21,9 @@ #include #include -#include -#include -#include +#include +#include +#include #include #include #include diff --git a/cpp/include/cuvs/distance/detail/pairwise_distance_cutlass_base.cuh b/cpp/include/cuvs/distance/detail/pairwise_distance_cutlass_base.cuh index aeb862b06..240eb596a 100644 --- a/cpp/include/cuvs/distance/detail/pairwise_distance_cutlass_base.cuh +++ b/cpp/include/cuvs/distance/detail/pairwise_distance_cutlass_base.cuh @@ -38,7 +38,7 @@ #include #include -#include +#include #include #include "./pairwise_distance_epilogue_elementwise.h" diff --git a/cpp/include/cuvs/distance/detail/pairwise_matrix/dispatch-ext.cuh b/cpp/include/cuvs/distance/detail/pairwise_matrix/dispatch-ext.cuh index e1dc6f9b3..efaebb379 100644 --- a/cpp/include/cuvs/distance/detail/pairwise_matrix/dispatch-ext.cuh +++ b/cpp/include/cuvs/distance/detail/pairwise_matrix/dispatch-ext.cuh @@ -15,16 +15,16 @@ */ #pragma once +#include // ops::* +#include // ops::has_cutlass_op +#include // rbf_fin_op +#include // pairwise_matrix_params #include // raft::identity_op -#include // ops::* -#include // ops::has_cutlass_op -#include // rbf_fin_op -#include // pairwise_matrix_params #include // RAFT_EXPLICIT #ifdef RAFT_EXPLICIT_INSTANTIATE_ONLY -namespace raft::distance::detail { +namespace cuvs::distance::detail { template , DataT, AccT, OutT, FinOpT, IdxT>( \ OpT distance_op, \ IdxT m, \ @@ -70,7 +70,7 @@ void pairwise_matrix_dispatch(OpT distance_op, * Hierarchy of instantiations: * * This file defines extern template instantiations of the distance kernels. The - * instantiation of the public API is handled in raft/distance/distance-ext.cuh. + * instantiation of the public API is handled in cuvs/distance/distance-ext.cuh. * * After adding an instance here, make sure to also add the instance there. */ @@ -78,113 +78,113 @@ void pairwise_matrix_dispatch(OpT distance_op, // The following two instances are used in the RBF kernel object. Note the use of int64_t for the // index type. instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::l2_unexp_distance_op, + cuvs::distance::detail::ops::l2_unexp_distance_op, float, float, float, - raft::distance::kernels::detail::rbf_fin_op, + cuvs::distance::kernels::detail::rbf_fin_op, int64_t); instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::l2_unexp_distance_op, + cuvs::distance::detail::ops::l2_unexp_distance_op, double, double, double, - raft::distance::kernels::detail::rbf_fin_op, + cuvs::distance::kernels::detail::rbf_fin_op, int64_t); // Rest of instances instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::canberra_distance_op, float, float, float, raft::identity_op, int); + cuvs::distance::detail::ops::canberra_distance_op, float, float, float, raft::identity_op, int); instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::canberra_distance_op, + cuvs::distance::detail::ops::canberra_distance_op, double, double, double, raft::identity_op, int); instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::correlation_distance_op, + cuvs::distance::detail::ops::correlation_distance_op, float, float, float, raft::identity_op, int); instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::correlation_distance_op, + cuvs::distance::detail::ops::correlation_distance_op, double, double, double, raft::identity_op, int); instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::cosine_distance_op, float, float, float, raft::identity_op, int); + cuvs::distance::detail::ops::cosine_distance_op, float, float, float, raft::identity_op, int); instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::cosine_distance_op, double, double, double, raft::identity_op, int); + cuvs::distance::detail::ops::cosine_distance_op, double, double, double, raft::identity_op, int); instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::hamming_distance_op, float, float, float, raft::identity_op, int); + cuvs::distance::detail::ops::hamming_distance_op, float, float, float, raft::identity_op, int); instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::hamming_distance_op, double, double, double, raft::identity_op, int); + cuvs::distance::detail::ops::hamming_distance_op, double, double, double, raft::identity_op, int); instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::hellinger_distance_op, float, float, float, raft::identity_op, int); + cuvs::distance::detail::ops::hellinger_distance_op, float, float, float, raft::identity_op, int); instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::hellinger_distance_op, + cuvs::distance::detail::ops::hellinger_distance_op, double, double, double, raft::identity_op, int); instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::jensen_shannon_distance_op, + cuvs::distance::detail::ops::jensen_shannon_distance_op, float, float, float, raft::identity_op, int); instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::jensen_shannon_distance_op, + cuvs::distance::detail::ops::jensen_shannon_distance_op, double, double, double, raft::identity_op, int); instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::kl_divergence_op, float, float, float, raft::identity_op, int); + cuvs::distance::detail::ops::kl_divergence_op, float, float, float, raft::identity_op, int); instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::kl_divergence_op, double, double, double, raft::identity_op, int); + cuvs::distance::detail::ops::kl_divergence_op, double, double, double, raft::identity_op, int); instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::l1_distance_op, float, float, float, raft::identity_op, int); + cuvs::distance::detail::ops::l1_distance_op, float, float, float, raft::identity_op, int); instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::l1_distance_op, double, double, double, raft::identity_op, int); + cuvs::distance::detail::ops::l1_distance_op, double, double, double, raft::identity_op, int); instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::l2_exp_distance_op, float, float, float, raft::identity_op, int); + cuvs::distance::detail::ops::l2_exp_distance_op, float, float, float, raft::identity_op, int); instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::l2_exp_distance_op, double, double, double, raft::identity_op, int); + cuvs::distance::detail::ops::l2_exp_distance_op, double, double, double, raft::identity_op, int); instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::l2_unexp_distance_op, float, float, float, raft::identity_op, int); + cuvs::distance::detail::ops::l2_unexp_distance_op, float, float, float, raft::identity_op, int); instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::l2_unexp_distance_op, + cuvs::distance::detail::ops::l2_unexp_distance_op, double, double, double, raft::identity_op, int); instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::l_inf_distance_op, float, float, float, raft::identity_op, int); + cuvs::distance::detail::ops::l_inf_distance_op, float, float, float, raft::identity_op, int); instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::l_inf_distance_op, double, double, double, raft::identity_op, int); + cuvs::distance::detail::ops::l_inf_distance_op, double, double, double, raft::identity_op, int); instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::lp_unexp_distance_op, float, float, float, raft::identity_op, int); + cuvs::distance::detail::ops::lp_unexp_distance_op, float, float, float, raft::identity_op, int); instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::lp_unexp_distance_op, + cuvs::distance::detail::ops::lp_unexp_distance_op, double, double, double, raft::identity_op, int); instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::russel_rao_distance_op, float, float, float, raft::identity_op, int); + cuvs::distance::detail::ops::russel_rao_distance_op, float, float, float, raft::identity_op, int); instantiate_raft_distance_detail_pairwise_matrix_dispatch( - raft::distance::detail::ops::russel_rao_distance_op, + cuvs::distance::detail::ops::russel_rao_distance_op, double, double, double, diff --git a/cpp/include/cuvs/distance/detail/pairwise_matrix/dispatch-inl.cuh b/cpp/include/cuvs/distance/detail/pairwise_matrix/dispatch-inl.cuh index fd9d44466..ca011731e 100644 --- a/cpp/include/cuvs/distance/detail/pairwise_matrix/dispatch-inl.cuh +++ b/cpp/include/cuvs/distance/detail/pairwise_matrix/dispatch-inl.cuh @@ -25,24 +25,24 @@ * * 2. Provide concise function templates that can be instantiated in * src/distance/detail/pairwise_matrix/. Previously, - * raft::distance::detail::distance was instantiated. The function + * cuvs::distance::detail::distance was instantiated. The function * necessarily required a large set of include files, which slowed down the - * build. The raft::distance::detail::pairwise_matrix_arch_dispatch functions + * build. The cuvs::distance::detail::pairwise_matrix_arch_dispatch functions * do not require as large an include files set, which speeds up the build. */ -#include // ops::has_cutlass_op -#include // dispatch_sm60 -#include // pairwise_matrix_params +#include // ops::has_cutlass_op +#include // dispatch_sm60 +#include // pairwise_matrix_params #include // raft::util::arch::SM_* // NOTE: to minimize compile times, we do not include dispatch_sm80.cuh. // Including dispatch_sm80.cuh can slow down compile times (due to CUTLASS). // Therefore, it is the including file's responsibility to include the correct -// dispatch_smXX.cuh headers, as is done in raft/distance/detail/distance.cuh +// dispatch_smXX.cuh headers, as is done in cuvs/distance/detail/distance.cuh // and src/distance/detail/pairwise_matrix/dispatch_*.cu. -namespace raft::distance::detail { +namespace cuvs::distance::detail { // This forward-declaration ensures that we do not need to include // dispatch_sm80.cuh if we are not calling it in practice. This makes compiling @@ -124,4 +124,4 @@ void pairwise_matrix_dispatch(OpT distance_op, } } -}; // namespace raft::distance::detail +}; // namespace cuvs::distance::detail diff --git a/cpp/include/cuvs/distance/detail/pairwise_matrix/dispatch_layout.cuh b/cpp/include/cuvs/distance/detail/pairwise_matrix/dispatch_layout.cuh index f2b0e5982..2e9004b56 100644 --- a/cpp/include/cuvs/distance/detail/pairwise_matrix/dispatch_layout.cuh +++ b/cpp/include/cuvs/distance/detail/pairwise_matrix/dispatch_layout.cuh @@ -17,10 +17,10 @@ #include // std::min #include // size_t +#include // pairwise_matrix_params #include // RAFT_EXPECTS -#include // pairwise_matrix_params #include // std::integral_constant -namespace raft::distance::detail { +namespace cuvs::distance::detail { /** * @brief: Computes minimal common alignment of the rows in a 2D array in bytes @@ -113,4 +113,4 @@ auto dispatch_layout(bool row_major, int vec_len, F&& f) } } -}; // namespace raft::distance::detail +}; // namespace cuvs::distance::detail diff --git a/cpp/include/cuvs/distance/detail/pairwise_matrix/dispatch_sm60.cuh b/cpp/include/cuvs/distance/detail/pairwise_matrix/dispatch_sm60.cuh index 2080fbe9c..9f9ed1cad 100644 --- a/cpp/include/cuvs/distance/detail/pairwise_matrix/dispatch_sm60.cuh +++ b/cpp/include/cuvs/distance/detail/pairwise_matrix/dispatch_sm60.cuh @@ -16,11 +16,11 @@ #pragma once #include // std::min -#include // dispatch_layout -#include // pairwise_matrix_sm60_wrapper +#include // dispatch_layout +#include // pairwise_matrix_sm60_wrapper #include // raft::linalg::Policy4x4 -namespace raft::distance::detail { +namespace cuvs::distance::detail { template // std::min -#include // cutlassDistanceKernel -#include // dispatch_layout +#include // cutlassDistanceKernel +#include // dispatch_layout -namespace raft::distance::detail { +namespace cuvs::distance::detail { template // assert +#include // PairwiseDistances +#include // pairwise_matrix_params #include // raft::void_op -#include // PairwiseDistances -#include // pairwise_matrix_params #include // raft::util::arch::SM_compute_arch -namespace raft::distance::detail { +namespace cuvs::distance::detail { template make_pairwise_matri grid, block, smem_size, kernel}; } -}; // namespace raft::distance::detail +}; // namespace cuvs::distance::detail diff --git a/cpp/include/cuvs/distance/detail/pairwise_matrix/params.cuh b/cpp/include/cuvs/distance/detail/pairwise_matrix/params.cuh index 005b95afe..aa419aca0 100644 --- a/cpp/include/cuvs/distance/detail/pairwise_matrix/params.cuh +++ b/cpp/include/cuvs/distance/detail/pairwise_matrix/params.cuh @@ -15,7 +15,7 @@ */ #pragma once -namespace raft::distance::detail { +namespace cuvs::distance::detail { template struct pairwise_matrix_params { @@ -44,4 +44,4 @@ struct pairwise_matrix_params { } }; -} // namespace raft::distance::detail +} // namespace cuvs::distance::detail diff --git a/cpp/include/cuvs/distance/distance-ext.cuh b/cpp/include/cuvs/distance/distance-ext.cuh index 7171ba605..31305f550 100644 --- a/cpp/include/cuvs/distance/distance-ext.cuh +++ b/cpp/include/cuvs/distance/distance-ext.cuh @@ -15,11 +15,11 @@ */ #pragma once +#include // rbf_fin_op +#include // cuvs::distance::DistanceType #include // raft::device_matrix_view #include // raft::identity_op #include // raft::resources -#include // rbf_fin_op -#include // raft::distance::DistanceType #include // RAFT_EXPLICIT #include // rmm::device_uvector @@ -28,7 +28,7 @@ namespace raft { namespace distance { -template size_t getWorkspaceSize(const DataT* x, const DataT* y, IdxT m, IdxT n, IdxT k) RAFT_EXPLICIT; -template const& x, raft::device_matrix_view const& y) RAFT_EXPLICIT; -template & workspace, - raft::distance::DistanceType metric, + cuvs::distance::DistanceType metric, bool isRowMajor = true, Type metric_arg = 2.0f) RAFT_EXPLICIT; @@ -116,11 +116,11 @@ void pairwise_distance(raft::resources const& handle, IdxT m, IdxT n, IdxT k, - raft::distance::DistanceType metric, + cuvs::distance::DistanceType metric, bool isRowMajor = true, Type metric_arg = 2.0f) RAFT_EXPLICIT; -template const x, device_matrix_view const y, device_matrix_view dist, - raft::distance::DistanceType metric, + cuvs::distance::DistanceType metric, Type metric_arg = 2.0f) RAFT_EXPLICIT; }; // namespace distance @@ -149,7 +149,7 @@ void pairwise_distance(raft::resources const& handle, * Hierarchy of instantiations: * * This file defines the extern template instantiations for the public API of - * raft::distance. To improve compile times, the extern template instantiation + * cuvs::distance. To improve compile times, the extern template instantiation * of the distance kernels is handled in * distance/detail/pairwise_matrix/dispatch-ext.cuh. * @@ -158,7 +158,7 @@ void pairwise_distance(raft::resources const& handle, */ #define instantiate_raft_distance_distance(DT, DataT, AccT, OutT, FinalLambda, IdxT) \ - extern template void raft::distance::distance( \ + extern template void cuvs::distance::distance( \ raft::resources const& handle, \ const DataT* x, \ const DataT* y, \ @@ -174,93 +174,93 @@ void pairwise_distance(raft::resources const& handle, // The following two instances are used in test/distance/gram.cu. Note the use // of int64_t for the index type. -instantiate_raft_distance_distance(raft::distance::DistanceType::L2Unexpanded, +instantiate_raft_distance_distance(cuvs::distance::DistanceType::L2Unexpanded, float, float, float, - raft::distance::kernels::detail::rbf_fin_op, + cuvs::distance::kernels::detail::rbf_fin_op, int64_t); -instantiate_raft_distance_distance(raft::distance::DistanceType::L2Unexpanded, +instantiate_raft_distance_distance(cuvs::distance::DistanceType::L2Unexpanded, double, double, double, - raft::distance::kernels::detail::rbf_fin_op, + cuvs::distance::kernels::detail::rbf_fin_op, int64_t); instantiate_raft_distance_distance( - raft::distance::DistanceType::Canberra, float, float, float, raft::identity_op, int); + cuvs::distance::DistanceType::Canberra, float, float, float, raft::identity_op, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::Canberra, double, double, double, raft::identity_op, int); + cuvs::distance::DistanceType::Canberra, double, double, double, raft::identity_op, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::CorrelationExpanded, float, float, float, raft::identity_op, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::CorrelationExpanded, + cuvs::distance::DistanceType::CorrelationExpanded, float, float, float, raft::identity_op, int); +instantiate_raft_distance_distance(cuvs::distance::DistanceType::CorrelationExpanded, double, double, double, raft::identity_op, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::CosineExpanded, float, float, float, raft::identity_op, int); + cuvs::distance::DistanceType::CosineExpanded, float, float, float, raft::identity_op, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::CosineExpanded, double, double, double, raft::identity_op, int); + cuvs::distance::DistanceType::CosineExpanded, double, double, double, raft::identity_op, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::HammingUnexpanded, float, float, float, raft::identity_op, int); + cuvs::distance::DistanceType::HammingUnexpanded, float, float, float, raft::identity_op, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::HammingUnexpanded, double, double, double, raft::identity_op, int); + cuvs::distance::DistanceType::HammingUnexpanded, double, double, double, raft::identity_op, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::HellingerExpanded, float, float, float, raft::identity_op, int); + cuvs::distance::DistanceType::HellingerExpanded, float, float, float, raft::identity_op, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::HellingerExpanded, double, double, double, raft::identity_op, int); + cuvs::distance::DistanceType::HellingerExpanded, double, double, double, raft::identity_op, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::InnerProduct, float, float, float, raft::identity_op, int); + cuvs::distance::DistanceType::InnerProduct, float, float, float, raft::identity_op, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::InnerProduct, double, double, double, raft::identity_op, int); + cuvs::distance::DistanceType::InnerProduct, double, double, double, raft::identity_op, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::JensenShannon, float, float, float, raft::identity_op, int); + cuvs::distance::DistanceType::JensenShannon, float, float, float, raft::identity_op, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::JensenShannon, double, double, double, raft::identity_op, int); + cuvs::distance::DistanceType::JensenShannon, double, double, double, raft::identity_op, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::KLDivergence, float, float, float, raft::identity_op, int); + cuvs::distance::DistanceType::KLDivergence, float, float, float, raft::identity_op, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::KLDivergence, double, double, double, raft::identity_op, int); + cuvs::distance::DistanceType::KLDivergence, double, double, double, raft::identity_op, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::L1, float, float, float, raft::identity_op, int); + cuvs::distance::DistanceType::L1, float, float, float, raft::identity_op, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::L1, double, double, double, raft::identity_op, int); + cuvs::distance::DistanceType::L1, double, double, double, raft::identity_op, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Expanded, float, float, float, raft::identity_op, int); + cuvs::distance::DistanceType::L2Expanded, float, float, float, raft::identity_op, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Expanded, double, double, double, raft::identity_op, int); + cuvs::distance::DistanceType::L2Expanded, double, double, double, raft::identity_op, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::L2SqrtExpanded, float, float, float, raft::identity_op, int); + cuvs::distance::DistanceType::L2SqrtExpanded, float, float, float, raft::identity_op, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::L2SqrtExpanded, double, double, double, raft::identity_op, int); + cuvs::distance::DistanceType::L2SqrtExpanded, double, double, double, raft::identity_op, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::L2SqrtUnexpanded, float, float, float, raft::identity_op, int); + cuvs::distance::DistanceType::L2SqrtUnexpanded, float, float, float, raft::identity_op, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::L2SqrtUnexpanded, double, double, double, raft::identity_op, int); + cuvs::distance::DistanceType::L2SqrtUnexpanded, double, double, double, raft::identity_op, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Unexpanded, float, float, float, raft::identity_op, int); + cuvs::distance::DistanceType::L2Unexpanded, float, float, float, raft::identity_op, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Unexpanded, double, double, double, raft::identity_op, int); + cuvs::distance::DistanceType::L2Unexpanded, double, double, double, raft::identity_op, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::Linf, float, float, float, raft::identity_op, int); + cuvs::distance::DistanceType::Linf, float, float, float, raft::identity_op, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::Linf, double, double, double, raft::identity_op, int); + cuvs::distance::DistanceType::Linf, double, double, double, raft::identity_op, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::LpUnexpanded, float, float, float, raft::identity_op, int); + cuvs::distance::DistanceType::LpUnexpanded, float, float, float, raft::identity_op, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::LpUnexpanded, double, double, double, raft::identity_op, int); + cuvs::distance::DistanceType::LpUnexpanded, double, double, double, raft::identity_op, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::RusselRaoExpanded, float, float, float, raft::identity_op, int); + cuvs::distance::DistanceType::RusselRaoExpanded, float, float, float, raft::identity_op, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::RusselRaoExpanded, double, double, double, raft::identity_op, int); + cuvs::distance::DistanceType::RusselRaoExpanded, double, double, double, raft::identity_op, int); #undef instantiate_raft_distance_distance // Same, but without raft::identity_op #define instantiate_raft_distance_distance(DT, DataT, AccT, OutT, IdxT) \ - extern template void raft::distance::distance( \ + extern template void cuvs::distance::distance( \ raft::resources const& handle, \ const DataT* x, \ const DataT* y, \ @@ -274,71 +274,71 @@ instantiate_raft_distance_distance( DataT metric_arg) instantiate_raft_distance_distance( - raft::distance::DistanceType::Canberra, float, float, float, int); + cuvs::distance::DistanceType::Canberra, float, float, float, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::Canberra, double, double, double, int); + cuvs::distance::DistanceType::Canberra, double, double, double, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::CorrelationExpanded, float, float, float, int); + cuvs::distance::DistanceType::CorrelationExpanded, float, float, float, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::CorrelationExpanded, double, double, double, int); + cuvs::distance::DistanceType::CorrelationExpanded, double, double, double, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::CosineExpanded, float, float, float, int); + cuvs::distance::DistanceType::CosineExpanded, float, float, float, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::CosineExpanded, double, double, double, int); + cuvs::distance::DistanceType::CosineExpanded, double, double, double, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::HammingUnexpanded, float, float, float, int); + cuvs::distance::DistanceType::HammingUnexpanded, float, float, float, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::HammingUnexpanded, double, double, double, int); + cuvs::distance::DistanceType::HammingUnexpanded, double, double, double, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::HellingerExpanded, float, float, float, int); + cuvs::distance::DistanceType::HellingerExpanded, float, float, float, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::HellingerExpanded, double, double, double, int); + cuvs::distance::DistanceType::HellingerExpanded, double, double, double, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::InnerProduct, float, float, float, int); + cuvs::distance::DistanceType::InnerProduct, float, float, float, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::InnerProduct, double, double, double, int); + cuvs::distance::DistanceType::InnerProduct, double, double, double, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::JensenShannon, float, float, float, int); + cuvs::distance::DistanceType::JensenShannon, float, float, float, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::JensenShannon, double, double, double, int); + cuvs::distance::DistanceType::JensenShannon, double, double, double, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::KLDivergence, float, float, float, int); + cuvs::distance::DistanceType::KLDivergence, float, float, float, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::KLDivergence, double, double, double, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L1, float, float, float, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L1, double, double, double, int); + cuvs::distance::DistanceType::KLDivergence, double, double, double, int); +instantiate_raft_distance_distance(cuvs::distance::DistanceType::L1, float, float, float, int); +instantiate_raft_distance_distance(cuvs::distance::DistanceType::L1, double, double, double, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Expanded, float, float, float, int); + cuvs::distance::DistanceType::L2Expanded, float, float, float, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Expanded, double, double, double, int); + cuvs::distance::DistanceType::L2Expanded, double, double, double, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::L2SqrtExpanded, float, float, float, int); + cuvs::distance::DistanceType::L2SqrtExpanded, float, float, float, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::L2SqrtExpanded, double, double, double, int); + cuvs::distance::DistanceType::L2SqrtExpanded, double, double, double, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::L2SqrtUnexpanded, float, float, float, int); + cuvs::distance::DistanceType::L2SqrtUnexpanded, float, float, float, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::L2SqrtUnexpanded, double, double, double, int); + cuvs::distance::DistanceType::L2SqrtUnexpanded, double, double, double, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Unexpanded, float, float, float, int); + cuvs::distance::DistanceType::L2Unexpanded, float, float, float, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Unexpanded, double, double, double, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::Linf, float, float, float, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::Linf, double, double, double, int); + cuvs::distance::DistanceType::L2Unexpanded, double, double, double, int); +instantiate_raft_distance_distance(cuvs::distance::DistanceType::Linf, float, float, float, int); +instantiate_raft_distance_distance(cuvs::distance::DistanceType::Linf, double, double, double, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::LpUnexpanded, float, float, float, int); + cuvs::distance::DistanceType::LpUnexpanded, float, float, float, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::LpUnexpanded, double, double, double, int); + cuvs::distance::DistanceType::LpUnexpanded, double, double, double, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::RusselRaoExpanded, float, float, float, int); + cuvs::distance::DistanceType::RusselRaoExpanded, float, float, float, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::RusselRaoExpanded, double, double, double, int); + cuvs::distance::DistanceType::RusselRaoExpanded, double, double, double, int); #undef instantiate_raft_distance_distance // Same, but without workspace #define instantiate_raft_distance_distance(DT, DataT, AccT, OutT, IdxT) \ - extern template void raft::distance::distance( \ + extern template void cuvs::distance::distance( \ raft::resources const& handle, \ const DataT* x, \ const DataT* y, \ @@ -350,377 +350,377 @@ instantiate_raft_distance_distance( DataT metric_arg) instantiate_raft_distance_distance( - raft::distance::DistanceType::Canberra, float, float, float, int); + cuvs::distance::DistanceType::Canberra, float, float, float, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::Canberra, double, double, double, int); + cuvs::distance::DistanceType::Canberra, double, double, double, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::CorrelationExpanded, float, float, float, int); + cuvs::distance::DistanceType::CorrelationExpanded, float, float, float, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::CorrelationExpanded, double, double, double, int); + cuvs::distance::DistanceType::CorrelationExpanded, double, double, double, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::CosineExpanded, float, float, float, int); + cuvs::distance::DistanceType::CosineExpanded, float, float, float, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::CosineExpanded, double, double, double, int); + cuvs::distance::DistanceType::CosineExpanded, double, double, double, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::HammingUnexpanded, float, float, float, int); + cuvs::distance::DistanceType::HammingUnexpanded, float, float, float, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::HammingUnexpanded, double, double, double, int); + cuvs::distance::DistanceType::HammingUnexpanded, double, double, double, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::HellingerExpanded, float, float, float, int); + cuvs::distance::DistanceType::HellingerExpanded, float, float, float, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::HellingerExpanded, double, double, double, int); + cuvs::distance::DistanceType::HellingerExpanded, double, double, double, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::InnerProduct, float, float, float, int); + cuvs::distance::DistanceType::InnerProduct, float, float, float, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::InnerProduct, double, double, double, int); + cuvs::distance::DistanceType::InnerProduct, double, double, double, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::JensenShannon, float, float, float, int); + cuvs::distance::DistanceType::JensenShannon, float, float, float, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::JensenShannon, double, double, double, int); + cuvs::distance::DistanceType::JensenShannon, double, double, double, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::KLDivergence, float, float, float, int); + cuvs::distance::DistanceType::KLDivergence, float, float, float, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::KLDivergence, double, double, double, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L1, float, float, float, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L1, double, double, double, int); + cuvs::distance::DistanceType::KLDivergence, double, double, double, int); +instantiate_raft_distance_distance(cuvs::distance::DistanceType::L1, float, float, float, int); +instantiate_raft_distance_distance(cuvs::distance::DistanceType::L1, double, double, double, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Expanded, float, float, float, int); + cuvs::distance::DistanceType::L2Expanded, float, float, float, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Expanded, double, double, double, int); + cuvs::distance::DistanceType::L2Expanded, double, double, double, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::L2SqrtExpanded, float, float, float, int); + cuvs::distance::DistanceType::L2SqrtExpanded, float, float, float, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::L2SqrtExpanded, double, double, double, int); + cuvs::distance::DistanceType::L2SqrtExpanded, double, double, double, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::L2SqrtUnexpanded, float, float, float, int); + cuvs::distance::DistanceType::L2SqrtUnexpanded, float, float, float, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::L2SqrtUnexpanded, double, double, double, int); + cuvs::distance::DistanceType::L2SqrtUnexpanded, double, double, double, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Unexpanded, float, float, float, int); + cuvs::distance::DistanceType::L2Unexpanded, float, float, float, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Unexpanded, double, double, double, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::Linf, float, float, float, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::Linf, double, double, double, int); + cuvs::distance::DistanceType::L2Unexpanded, double, double, double, int); +instantiate_raft_distance_distance(cuvs::distance::DistanceType::Linf, float, float, float, int); +instantiate_raft_distance_distance(cuvs::distance::DistanceType::Linf, double, double, double, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::LpUnexpanded, float, float, float, int); + cuvs::distance::DistanceType::LpUnexpanded, float, float, float, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::LpUnexpanded, double, double, double, int); + cuvs::distance::DistanceType::LpUnexpanded, double, double, double, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::RusselRaoExpanded, float, float, float, int); + cuvs::distance::DistanceType::RusselRaoExpanded, float, float, float, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::RusselRaoExpanded, double, double, double, int); + cuvs::distance::DistanceType::RusselRaoExpanded, double, double, double, int); #undef instantiate_raft_distance_distance #define instantiate_raft_distance_getWorkspaceSize(DistT, DataT, AccT, OutT, IdxT) \ - extern template size_t raft::distance::getWorkspaceSize( \ + extern template size_t cuvs::distance::getWorkspaceSize( \ const DataT* x, const DataT* y, IdxT m, IdxT n, IdxT k) instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::Canberra, float, float, float, int); + cuvs::distance::DistanceType::Canberra, float, float, float, int); instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::Canberra, double, double, double, int); + cuvs::distance::DistanceType::Canberra, double, double, double, int); instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::CorrelationExpanded, float, float, float, int); + cuvs::distance::DistanceType::CorrelationExpanded, float, float, float, int); instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::CorrelationExpanded, double, double, double, int); + cuvs::distance::DistanceType::CorrelationExpanded, double, double, double, int); instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::CosineExpanded, float, float, float, int); + cuvs::distance::DistanceType::CosineExpanded, float, float, float, int); instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::CosineExpanded, double, double, double, int); + cuvs::distance::DistanceType::CosineExpanded, double, double, double, int); instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::HammingUnexpanded, float, float, float, int); + cuvs::distance::DistanceType::HammingUnexpanded, float, float, float, int); instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::HammingUnexpanded, double, double, double, int); + cuvs::distance::DistanceType::HammingUnexpanded, double, double, double, int); instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::HellingerExpanded, float, float, float, int); + cuvs::distance::DistanceType::HellingerExpanded, float, float, float, int); instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::HellingerExpanded, double, double, double, int); + cuvs::distance::DistanceType::HellingerExpanded, double, double, double, int); instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::InnerProduct, float, float, float, int); + cuvs::distance::DistanceType::InnerProduct, float, float, float, int); instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::InnerProduct, double, double, double, int); + cuvs::distance::DistanceType::InnerProduct, double, double, double, int); instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::JensenShannon, float, float, float, int); + cuvs::distance::DistanceType::JensenShannon, float, float, float, int); instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::JensenShannon, double, double, double, int); + cuvs::distance::DistanceType::JensenShannon, double, double, double, int); instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::KLDivergence, float, float, float, int); + cuvs::distance::DistanceType::KLDivergence, float, float, float, int); instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::KLDivergence, double, double, double, int); + cuvs::distance::DistanceType::KLDivergence, double, double, double, int); instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L1, float, float, float, int); + cuvs::distance::DistanceType::L1, float, float, float, int); instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L1, double, double, double, int); + cuvs::distance::DistanceType::L1, double, double, double, int); instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2Expanded, float, float, float, int); + cuvs::distance::DistanceType::L2Expanded, float, float, float, int); instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2Expanded, double, double, double, int); + cuvs::distance::DistanceType::L2Expanded, double, double, double, int); instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2SqrtExpanded, float, float, float, int); + cuvs::distance::DistanceType::L2SqrtExpanded, float, float, float, int); instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2SqrtExpanded, double, double, double, int); + cuvs::distance::DistanceType::L2SqrtExpanded, double, double, double, int); instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2SqrtUnexpanded, float, float, float, int); + cuvs::distance::DistanceType::L2SqrtUnexpanded, float, float, float, int); instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2SqrtUnexpanded, double, double, double, int); + cuvs::distance::DistanceType::L2SqrtUnexpanded, double, double, double, int); instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2Unexpanded, float, float, float, int); + cuvs::distance::DistanceType::L2Unexpanded, float, float, float, int); instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2Unexpanded, double, double, double, int); + cuvs::distance::DistanceType::L2Unexpanded, double, double, double, int); instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::Linf, float, float, float, int); + cuvs::distance::DistanceType::Linf, float, float, float, int); instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::Linf, double, double, double, int); + cuvs::distance::DistanceType::Linf, double, double, double, int); instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::LpUnexpanded, float, float, float, int); + cuvs::distance::DistanceType::LpUnexpanded, float, float, float, int); instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::LpUnexpanded, double, double, double, int); + cuvs::distance::DistanceType::LpUnexpanded, double, double, double, int); instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::RusselRaoExpanded, float, float, float, int); + cuvs::distance::DistanceType::RusselRaoExpanded, float, float, float, int); instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::RusselRaoExpanded, double, double, double, int); + cuvs::distance::DistanceType::RusselRaoExpanded, double, double, double, int); #undef instantiate_raft_distance_getWorkspaceSize #define instantiate_raft_distance_getWorkspaceSize(DistT, DataT, AccT, OutT, IdxT, layout) \ - extern template size_t raft::distance::getWorkspaceSize( \ + extern template size_t cuvs::distance::getWorkspaceSize( \ raft::device_matrix_view const& x, \ raft::device_matrix_view const& y) // We could consider not taking template parameters for this function. The // number of instantiations seems a bit excessive.. instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::Canberra, float, float, float, int, raft::layout_c_contiguous); + cuvs::distance::DistanceType::Canberra, float, float, float, int, raft::layout_c_contiguous); instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::Canberra, double, double, double, int, raft::layout_c_contiguous); + cuvs::distance::DistanceType::Canberra, double, double, double, int, raft::layout_c_contiguous); instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::Canberra, float, float, float, int, raft::layout_f_contiguous); + cuvs::distance::DistanceType::Canberra, float, float, float, int, raft::layout_f_contiguous); instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::Canberra, double, double, double, int, raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::CorrelationExpanded, + cuvs::distance::DistanceType::Canberra, double, double, double, int, raft::layout_f_contiguous); +instantiate_raft_distance_getWorkspaceSize(cuvs::distance::DistanceType::CorrelationExpanded, float, float, float, int, raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::CorrelationExpanded, +instantiate_raft_distance_getWorkspaceSize(cuvs::distance::DistanceType::CorrelationExpanded, double, double, double, int, raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::CorrelationExpanded, +instantiate_raft_distance_getWorkspaceSize(cuvs::distance::DistanceType::CorrelationExpanded, float, float, float, int, raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::CorrelationExpanded, +instantiate_raft_distance_getWorkspaceSize(cuvs::distance::DistanceType::CorrelationExpanded, double, double, double, int, raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::CosineExpanded, +instantiate_raft_distance_getWorkspaceSize(cuvs::distance::DistanceType::CosineExpanded, float, float, float, int, raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::CosineExpanded, +instantiate_raft_distance_getWorkspaceSize(cuvs::distance::DistanceType::CosineExpanded, double, double, double, int, raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::CosineExpanded, +instantiate_raft_distance_getWorkspaceSize(cuvs::distance::DistanceType::CosineExpanded, float, float, float, int, raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::CosineExpanded, +instantiate_raft_distance_getWorkspaceSize(cuvs::distance::DistanceType::CosineExpanded, double, double, double, int, raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::HammingUnexpanded, +instantiate_raft_distance_getWorkspaceSize(cuvs::distance::DistanceType::HammingUnexpanded, float, float, float, int, raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::HammingUnexpanded, +instantiate_raft_distance_getWorkspaceSize(cuvs::distance::DistanceType::HammingUnexpanded, double, double, double, int, raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::HammingUnexpanded, +instantiate_raft_distance_getWorkspaceSize(cuvs::distance::DistanceType::HammingUnexpanded, float, float, float, int, raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::HammingUnexpanded, +instantiate_raft_distance_getWorkspaceSize(cuvs::distance::DistanceType::HammingUnexpanded, double, double, double, int, raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::HellingerExpanded, +instantiate_raft_distance_getWorkspaceSize(cuvs::distance::DistanceType::HellingerExpanded, float, float, float, int, raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::HellingerExpanded, +instantiate_raft_distance_getWorkspaceSize(cuvs::distance::DistanceType::HellingerExpanded, double, double, double, int, raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::HellingerExpanded, +instantiate_raft_distance_getWorkspaceSize(cuvs::distance::DistanceType::HellingerExpanded, float, float, float, int, raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::HellingerExpanded, +instantiate_raft_distance_getWorkspaceSize(cuvs::distance::DistanceType::HellingerExpanded, double, double, double, int, raft::layout_f_contiguous); instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::InnerProduct, float, float, float, int, raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::InnerProduct, + cuvs::distance::DistanceType::InnerProduct, float, float, float, int, raft::layout_c_contiguous); +instantiate_raft_distance_getWorkspaceSize(cuvs::distance::DistanceType::InnerProduct, double, double, double, int, raft::layout_c_contiguous); instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::InnerProduct, float, float, float, int, raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::InnerProduct, + cuvs::distance::DistanceType::InnerProduct, float, float, float, int, raft::layout_f_contiguous); +instantiate_raft_distance_getWorkspaceSize(cuvs::distance::DistanceType::InnerProduct, double, double, double, int, raft::layout_f_contiguous); instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::JensenShannon, float, float, float, int, raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::JensenShannon, + cuvs::distance::DistanceType::JensenShannon, float, float, float, int, raft::layout_c_contiguous); +instantiate_raft_distance_getWorkspaceSize(cuvs::distance::DistanceType::JensenShannon, double, double, double, int, raft::layout_c_contiguous); instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::JensenShannon, float, float, float, int, raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::JensenShannon, + cuvs::distance::DistanceType::JensenShannon, float, float, float, int, raft::layout_f_contiguous); +instantiate_raft_distance_getWorkspaceSize(cuvs::distance::DistanceType::JensenShannon, double, double, double, int, raft::layout_f_contiguous); instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::KLDivergence, float, float, float, int, raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::KLDivergence, + cuvs::distance::DistanceType::KLDivergence, float, float, float, int, raft::layout_c_contiguous); +instantiate_raft_distance_getWorkspaceSize(cuvs::distance::DistanceType::KLDivergence, double, double, double, int, raft::layout_c_contiguous); instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::KLDivergence, float, float, float, int, raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::KLDivergence, + cuvs::distance::DistanceType::KLDivergence, float, float, float, int, raft::layout_f_contiguous); +instantiate_raft_distance_getWorkspaceSize(cuvs::distance::DistanceType::KLDivergence, double, double, double, int, raft::layout_f_contiguous); instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L1, float, float, float, int, raft::layout_c_contiguous); + cuvs::distance::DistanceType::L1, float, float, float, int, raft::layout_c_contiguous); instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L1, double, double, double, int, raft::layout_c_contiguous); + cuvs::distance::DistanceType::L1, double, double, double, int, raft::layout_c_contiguous); instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L1, float, float, float, int, raft::layout_f_contiguous); + cuvs::distance::DistanceType::L1, float, float, float, int, raft::layout_f_contiguous); instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L1, double, double, double, int, raft::layout_f_contiguous); + cuvs::distance::DistanceType::L1, double, double, double, int, raft::layout_f_contiguous); instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2Expanded, float, float, float, int, raft::layout_c_contiguous); + cuvs::distance::DistanceType::L2Expanded, float, float, float, int, raft::layout_c_contiguous); instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2Expanded, double, double, double, int, raft::layout_c_contiguous); + cuvs::distance::DistanceType::L2Expanded, double, double, double, int, raft::layout_c_contiguous); instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2Expanded, float, float, float, int, raft::layout_f_contiguous); + cuvs::distance::DistanceType::L2Expanded, float, float, float, int, raft::layout_f_contiguous); instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2Expanded, double, double, double, int, raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::L2SqrtExpanded, + cuvs::distance::DistanceType::L2Expanded, double, double, double, int, raft::layout_f_contiguous); +instantiate_raft_distance_getWorkspaceSize(cuvs::distance::DistanceType::L2SqrtExpanded, float, float, float, int, raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::L2SqrtExpanded, +instantiate_raft_distance_getWorkspaceSize(cuvs::distance::DistanceType::L2SqrtExpanded, double, double, double, int, raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::L2SqrtExpanded, +instantiate_raft_distance_getWorkspaceSize(cuvs::distance::DistanceType::L2SqrtExpanded, float, float, float, int, raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::L2SqrtExpanded, +instantiate_raft_distance_getWorkspaceSize(cuvs::distance::DistanceType::L2SqrtExpanded, double, double, double, int, raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::L2SqrtUnexpanded, +instantiate_raft_distance_getWorkspaceSize(cuvs::distance::DistanceType::L2SqrtUnexpanded, float, float, float, int, raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::L2SqrtUnexpanded, +instantiate_raft_distance_getWorkspaceSize(cuvs::distance::DistanceType::L2SqrtUnexpanded, double, double, double, int, raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::L2SqrtUnexpanded, +instantiate_raft_distance_getWorkspaceSize(cuvs::distance::DistanceType::L2SqrtUnexpanded, float, float, float, int, raft::layout_f_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::L2SqrtUnexpanded, +instantiate_raft_distance_getWorkspaceSize(cuvs::distance::DistanceType::L2SqrtUnexpanded, double, double, double, int, raft::layout_f_contiguous); instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2Unexpanded, float, float, float, int, raft::layout_c_contiguous); -instantiate_raft_distance_getWorkspaceSize(raft::distance::DistanceType::L2Unexpanded, + cuvs::distance::DistanceType::L2Unexpanded, float, float, float, int, raft::layout_c_contiguous); +instantiate_raft_distance_getWorkspaceSize(cuvs::distance::DistanceType::L2Unexpanded, double, double, double, int, raft::layout_c_contiguous); instantiate_raft_distance_getWorkspaceSize( - raft::distance::DistanceType::L2Unexpanded, float, float, float, int, raft::layout_f_contiguous); + cuvs::distance::DistanceType::L2Unexpanded, float, float, float, int, raft::layout_f_contiguous); #undef instantiate_raft_distance_getWorkspaceSize #define instantiate_raft_distance_pairwise_distance(DataT, IdxT) \ - extern template void raft::distance::pairwise_distance(raft::resources const& handle, \ + extern template void cuvs::distance::pairwise_distance(raft::resources const& handle, \ const DataT* x, \ const DataT* y, \ DataT* dist, \ @@ -728,7 +728,7 @@ instantiate_raft_distance_getWorkspaceSize( IdxT n, \ IdxT k, \ rmm::device_uvector& workspace, \ - raft::distance::DistanceType metric, \ + cuvs::distance::DistanceType metric, \ bool isRowMajor, \ DataT metric_arg) @@ -739,14 +739,14 @@ instantiate_raft_distance_pairwise_distance(double, int); // Same, but without workspace #define instantiate_raft_distance_pairwise_distance(DataT, IdxT) \ - extern template void raft::distance::pairwise_distance(raft::resources const& handle, \ + extern template void cuvs::distance::pairwise_distance(raft::resources const& handle, \ const DataT* x, \ const DataT* y, \ DataT* dist, \ IdxT m, \ IdxT n, \ IdxT k, \ - raft::distance::DistanceType metric, \ + cuvs::distance::DistanceType metric, \ bool isRowMajor, \ DataT metric_arg) @@ -757,7 +757,7 @@ instantiate_raft_distance_pairwise_distance(double, int); // Version with mdspan #define instantiate_raft_distance_distance(DistT, DataT, AccT, OutT, layout, IdxT) \ - extern template void raft::distance::distance( \ + extern template void cuvs::distance::distance( \ raft::resources const& handle, \ raft::device_matrix_view const x, \ raft::device_matrix_view const y, \ @@ -766,280 +766,280 @@ instantiate_raft_distance_pairwise_distance(double, int); // Again, we might want to consider reigning in the number of instantiations... instantiate_raft_distance_distance( - raft::distance::DistanceType::Canberra, float, float, float, raft::layout_c_contiguous, int); + cuvs::distance::DistanceType::Canberra, float, float, float, raft::layout_c_contiguous, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::Canberra, double, double, double, raft::layout_c_contiguous, int); + cuvs::distance::DistanceType::Canberra, double, double, double, raft::layout_c_contiguous, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::Canberra, float, float, float, raft::layout_f_contiguous, int); + cuvs::distance::DistanceType::Canberra, float, float, float, raft::layout_f_contiguous, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::Canberra, double, double, double, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::CorrelationExpanded, + cuvs::distance::DistanceType::Canberra, double, double, double, raft::layout_f_contiguous, int); +instantiate_raft_distance_distance(cuvs::distance::DistanceType::CorrelationExpanded, float, float, float, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::CorrelationExpanded, +instantiate_raft_distance_distance(cuvs::distance::DistanceType::CorrelationExpanded, double, double, double, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::CorrelationExpanded, +instantiate_raft_distance_distance(cuvs::distance::DistanceType::CorrelationExpanded, float, float, float, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::CorrelationExpanded, +instantiate_raft_distance_distance(cuvs::distance::DistanceType::CorrelationExpanded, double, double, double, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::CosineExpanded, +instantiate_raft_distance_distance(cuvs::distance::DistanceType::CosineExpanded, float, float, float, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::CosineExpanded, +instantiate_raft_distance_distance(cuvs::distance::DistanceType::CosineExpanded, double, double, double, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::CosineExpanded, +instantiate_raft_distance_distance(cuvs::distance::DistanceType::CosineExpanded, float, float, float, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::CosineExpanded, +instantiate_raft_distance_distance(cuvs::distance::DistanceType::CosineExpanded, double, double, double, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::HammingUnexpanded, +instantiate_raft_distance_distance(cuvs::distance::DistanceType::HammingUnexpanded, float, float, float, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::HammingUnexpanded, +instantiate_raft_distance_distance(cuvs::distance::DistanceType::HammingUnexpanded, double, double, double, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::HammingUnexpanded, +instantiate_raft_distance_distance(cuvs::distance::DistanceType::HammingUnexpanded, float, float, float, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::HammingUnexpanded, +instantiate_raft_distance_distance(cuvs::distance::DistanceType::HammingUnexpanded, double, double, double, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::HellingerExpanded, +instantiate_raft_distance_distance(cuvs::distance::DistanceType::HellingerExpanded, float, float, float, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::HellingerExpanded, +instantiate_raft_distance_distance(cuvs::distance::DistanceType::HellingerExpanded, double, double, double, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::HellingerExpanded, +instantiate_raft_distance_distance(cuvs::distance::DistanceType::HellingerExpanded, float, float, float, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::HellingerExpanded, +instantiate_raft_distance_distance(cuvs::distance::DistanceType::HellingerExpanded, double, double, double, raft::layout_f_contiguous, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::InnerProduct, float, float, float, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::InnerProduct, + cuvs::distance::DistanceType::InnerProduct, float, float, float, raft::layout_c_contiguous, int); +instantiate_raft_distance_distance(cuvs::distance::DistanceType::InnerProduct, double, double, double, raft::layout_c_contiguous, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::InnerProduct, float, float, float, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::InnerProduct, + cuvs::distance::DistanceType::InnerProduct, float, float, float, raft::layout_f_contiguous, int); +instantiate_raft_distance_distance(cuvs::distance::DistanceType::InnerProduct, double, double, double, raft::layout_f_contiguous, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::JensenShannon, float, float, float, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::JensenShannon, + cuvs::distance::DistanceType::JensenShannon, float, float, float, raft::layout_c_contiguous, int); +instantiate_raft_distance_distance(cuvs::distance::DistanceType::JensenShannon, double, double, double, raft::layout_c_contiguous, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::JensenShannon, float, float, float, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::JensenShannon, + cuvs::distance::DistanceType::JensenShannon, float, float, float, raft::layout_f_contiguous, int); +instantiate_raft_distance_distance(cuvs::distance::DistanceType::JensenShannon, double, double, double, raft::layout_f_contiguous, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::KLDivergence, float, float, float, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::KLDivergence, + cuvs::distance::DistanceType::KLDivergence, float, float, float, raft::layout_c_contiguous, int); +instantiate_raft_distance_distance(cuvs::distance::DistanceType::KLDivergence, double, double, double, raft::layout_c_contiguous, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::KLDivergence, float, float, float, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::KLDivergence, + cuvs::distance::DistanceType::KLDivergence, float, float, float, raft::layout_f_contiguous, int); +instantiate_raft_distance_distance(cuvs::distance::DistanceType::KLDivergence, double, double, double, raft::layout_f_contiguous, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::L1, float, float, float, raft::layout_c_contiguous, int); + cuvs::distance::DistanceType::L1, float, float, float, raft::layout_c_contiguous, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::L1, double, double, double, raft::layout_c_contiguous, int); + cuvs::distance::DistanceType::L1, double, double, double, raft::layout_c_contiguous, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::L1, float, float, float, raft::layout_f_contiguous, int); + cuvs::distance::DistanceType::L1, float, float, float, raft::layout_f_contiguous, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::L1, double, double, double, raft::layout_f_contiguous, int); + cuvs::distance::DistanceType::L1, double, double, double, raft::layout_f_contiguous, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Expanded, float, float, float, raft::layout_c_contiguous, int); + cuvs::distance::DistanceType::L2Expanded, float, float, float, raft::layout_c_contiguous, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Expanded, double, double, double, raft::layout_c_contiguous, int); + cuvs::distance::DistanceType::L2Expanded, double, double, double, raft::layout_c_contiguous, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Expanded, float, float, float, raft::layout_f_contiguous, int); + cuvs::distance::DistanceType::L2Expanded, float, float, float, raft::layout_f_contiguous, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Expanded, double, double, double, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L2SqrtExpanded, + cuvs::distance::DistanceType::L2Expanded, double, double, double, raft::layout_f_contiguous, int); +instantiate_raft_distance_distance(cuvs::distance::DistanceType::L2SqrtExpanded, float, float, float, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L2SqrtExpanded, +instantiate_raft_distance_distance(cuvs::distance::DistanceType::L2SqrtExpanded, double, double, double, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L2SqrtExpanded, +instantiate_raft_distance_distance(cuvs::distance::DistanceType::L2SqrtExpanded, float, float, float, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L2SqrtExpanded, +instantiate_raft_distance_distance(cuvs::distance::DistanceType::L2SqrtExpanded, double, double, double, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L2SqrtUnexpanded, +instantiate_raft_distance_distance(cuvs::distance::DistanceType::L2SqrtUnexpanded, float, float, float, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L2SqrtUnexpanded, +instantiate_raft_distance_distance(cuvs::distance::DistanceType::L2SqrtUnexpanded, double, double, double, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L2SqrtUnexpanded, +instantiate_raft_distance_distance(cuvs::distance::DistanceType::L2SqrtUnexpanded, float, float, float, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L2SqrtUnexpanded, +instantiate_raft_distance_distance(cuvs::distance::DistanceType::L2SqrtUnexpanded, double, double, double, raft::layout_f_contiguous, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Unexpanded, float, float, float, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L2Unexpanded, + cuvs::distance::DistanceType::L2Unexpanded, float, float, float, raft::layout_c_contiguous, int); +instantiate_raft_distance_distance(cuvs::distance::DistanceType::L2Unexpanded, double, double, double, raft::layout_c_contiguous, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::L2Unexpanded, float, float, float, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::L2Unexpanded, + cuvs::distance::DistanceType::L2Unexpanded, float, float, float, raft::layout_f_contiguous, int); +instantiate_raft_distance_distance(cuvs::distance::DistanceType::L2Unexpanded, double, double, double, raft::layout_f_contiguous, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::Linf, float, float, float, raft::layout_c_contiguous, int); + cuvs::distance::DistanceType::Linf, float, float, float, raft::layout_c_contiguous, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::Linf, double, double, double, raft::layout_c_contiguous, int); + cuvs::distance::DistanceType::Linf, double, double, double, raft::layout_c_contiguous, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::Linf, float, float, float, raft::layout_f_contiguous, int); + cuvs::distance::DistanceType::Linf, float, float, float, raft::layout_f_contiguous, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::Linf, double, double, double, raft::layout_f_contiguous, int); + cuvs::distance::DistanceType::Linf, double, double, double, raft::layout_f_contiguous, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::LpUnexpanded, float, float, float, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::LpUnexpanded, + cuvs::distance::DistanceType::LpUnexpanded, float, float, float, raft::layout_c_contiguous, int); +instantiate_raft_distance_distance(cuvs::distance::DistanceType::LpUnexpanded, double, double, double, raft::layout_c_contiguous, int); instantiate_raft_distance_distance( - raft::distance::DistanceType::LpUnexpanded, float, float, float, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::LpUnexpanded, + cuvs::distance::DistanceType::LpUnexpanded, float, float, float, raft::layout_f_contiguous, int); +instantiate_raft_distance_distance(cuvs::distance::DistanceType::LpUnexpanded, double, double, double, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::RusselRaoExpanded, +instantiate_raft_distance_distance(cuvs::distance::DistanceType::RusselRaoExpanded, float, float, float, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::RusselRaoExpanded, +instantiate_raft_distance_distance(cuvs::distance::DistanceType::RusselRaoExpanded, double, double, double, raft::layout_c_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::RusselRaoExpanded, +instantiate_raft_distance_distance(cuvs::distance::DistanceType::RusselRaoExpanded, float, float, float, raft::layout_f_contiguous, int); -instantiate_raft_distance_distance(raft::distance::DistanceType::RusselRaoExpanded, +instantiate_raft_distance_distance(cuvs::distance::DistanceType::RusselRaoExpanded, double, double, double, @@ -1049,12 +1049,12 @@ instantiate_raft_distance_distance(raft::distance::DistanceType::RusselRaoExpand #undef instantiate_raft_distance_distance #define instantiate_raft_distance_pairwise_distance(DataT, layout, IdxT) \ - extern template void raft::distance::pairwise_distance( \ + extern template void cuvs::distance::pairwise_distance( \ raft::resources const& handle, \ raft::device_matrix_view const x, \ raft::device_matrix_view const y, \ raft::device_matrix_view dist, \ - raft::distance::DistanceType metric, \ + cuvs::distance::DistanceType metric, \ DataT metric_arg) instantiate_raft_distance_pairwise_distance(float, raft::layout_c_contiguous, int); diff --git a/cpp/include/cuvs/distance/distance-inl.cuh b/cpp/include/cuvs/distance/distance-inl.cuh index d17e5767b..0d2ad958a 100644 --- a/cpp/include/cuvs/distance/distance-inl.cuh +++ b/cpp/include/cuvs/distance/distance-inl.cuh @@ -15,10 +15,10 @@ */ #pragma once +#include +#include #include #include -#include -#include #include #include @@ -58,7 +58,7 @@ namespace distance { * as follows: 
OutT fin_op(AccT in, int g_idx);
. If one needs * any other parameters, feel free to pass them via closure. */ -template const& x, * @param isRowMajor whether the matrices are row-major or col-major * @param metric_arg metric argument (used for Minkowski distance) */ -template & workspace, - raft::distance::DistanceType metric, + cuvs::distance::DistanceType metric, bool isRowMajor = true, Type metric_arg = 2.0f) { @@ -272,7 +272,7 @@ void pairwise_distance(raft::resources const& handle, case DistanceType::HellingerExpanded: dispatch(std::integral_constant{}); break; - case raft::distance::DistanceType::InnerProduct: + case cuvs::distance::DistanceType::InnerProduct: dispatch(std::integral_constant{}); break; case DistanceType::JensenShannon: @@ -333,7 +333,7 @@ void pairwise_distance(raft::resources const& handle, IdxT m, IdxT n, IdxT k, - raft::distance::DistanceType metric, + cuvs::distance::DistanceType metric, bool isRowMajor = true, Type metric_arg = 2.0f) { @@ -360,7 +360,7 @@ void pairwise_distance(raft::resources const& handle, * #include * #include * #include - * #include + * #include * * raft::raft::resources handle; * int n_samples = 5000; @@ -371,8 +371,8 @@ void pairwise_distance(raft::resources const& handle, * auto output = raft::make_device_matrix(handle, n_samples, n_samples); * * raft::random::make_blobs(handle, input.view(), labels.view()); - * auto metric = raft::distance::DistanceType::L2SqrtExpanded; - * raft::distance::pairwise_distance(handle, input.view(), input.view(), output.view(), metric); + * auto metric = cuvs::distance::DistanceType::L2SqrtExpanded; + * cuvs::distance::pairwise_distance(handle, input.view(), input.view(), output.view(), metric); * @endcode * * @tparam DistanceType which distance to evaluate @@ -386,7 +386,7 @@ void pairwise_distance(raft::resources const& handle, * @param dist output distance matrix (size n*m) * @param metric_arg metric argument (used for Minkowski distance) */ -template const x, device_matrix_view const y, device_matrix_view dist, - raft::distance::DistanceType metric, + cuvs::distance::DistanceType metric, Type metric_arg = 2.0f) { RAFT_EXPECTS(x.extent(1) == y.extent(1), "Number of columns must be equal."); diff --git a/cpp/include/cuvs/distance/distance_types.hpp b/cpp/include/cuvs/distance/distance_types.hpp index d17ef358e..0b2fa4c26 100644 --- a/cpp/include/cuvs/distance/distance_types.hpp +++ b/cpp/include/cuvs/distance/distance_types.hpp @@ -16,8 +16,7 @@ #pragma once -namespace raft { -namespace distance { +namespace cuvs::distance { /** enum to tell how to compute distance */ enum DistanceType : unsigned short { @@ -104,5 +103,4 @@ struct KernelParams { }; } // end namespace kernels -}; // namespace distance -}; // end namespace raft +}; // namespace cuvs::distance diff --git a/cpp/include/cuvs/distance/fused_l2_nn-ext.cuh b/cpp/include/cuvs/distance/fused_l2_nn-ext.cuh index c99c1eb01..efc61c687 100644 --- a/cpp/include/cuvs/distance/fused_l2_nn-ext.cuh +++ b/cpp/include/cuvs/distance/fused_l2_nn-ext.cuh @@ -17,9 +17,9 @@ #pragma once #include // int64_t +#include // include initialize and reduce operations #include // raft::KeyValuePair #include // raft::resources -#include // include initialize and reduce operations #include // RAFT_EXPLICIT #ifdef RAFT_EXPLICIT_INSTANTIATE_ONLY @@ -47,7 +47,7 @@ void fusedL2NNMinReduce(OutT* min, #endif // RAFT_EXPLICIT_INSTANTIATE_ONLY #define instantiate_raft_distance_fusedL2NNMinReduce(DataT, OutT, IdxT) \ - extern template void raft::distance::fusedL2NNMinReduce(OutT * min, \ + extern template void cuvs::distance::fusedL2NNMinReduce(OutT * min, \ const DataT* x, \ const DataT* y, \ const DataT* xn, \ diff --git a/cpp/include/cuvs/distance/fused_l2_nn-inl.cuh b/cpp/include/cuvs/distance/fused_l2_nn-inl.cuh index 17373e3bc..3b9d0b567 100644 --- a/cpp/include/cuvs/distance/fused_l2_nn-inl.cuh +++ b/cpp/include/cuvs/distance/fused_l2_nn-inl.cuh @@ -20,10 +20,10 @@ #pragma once #include +#include +#include #include #include -#include -#include #include #include #include diff --git a/cpp/include/cuvs/distance/fused_l2_nn_helpers.cuh b/cpp/include/cuvs/distance/fused_l2_nn_helpers.cuh index 996f696ef..29a4ae523 100644 --- a/cpp/include/cuvs/distance/fused_l2_nn_helpers.cuh +++ b/cpp/include/cuvs/distance/fused_l2_nn_helpers.cuh @@ -16,10 +16,10 @@ #pragma once +#include #include -#include -namespace raft::distance { +namespace cuvs::distance { /** * \defgroup fused_l2_nn Fused 1-nearest neighbors @@ -47,4 +47,4 @@ void initialize(raft::resources const& handle, OutT* min, IdxT m, DataT maxVal, min, m, maxVal, redOp, resource::get_cuda_stream(handle)); } -} // namespace raft::distance +} // namespace cuvs::distance diff --git a/cpp/include/cuvs/distance/kernels.cuh b/cpp/include/cuvs/distance/kernels.cuh index 961068966..0133892a6 100644 --- a/cpp/include/cuvs/distance/kernels.cuh +++ b/cpp/include/cuvs/distance/kernels.cuh @@ -16,17 +16,17 @@ #pragma once -#include -#include +#include +#include #include -#include +#include #include -namespace raft::distance::kernels { +namespace cuvs::distance::kernels { // TODO: Need to expose formal APIs for this that are more consistent w/ other APIs in RAFT -using raft::distance::kernels::detail::GramMatrixBase; -using raft::distance::kernels::detail::KernelFactory; +using cuvs::distance::kernels::detail::GramMatrixBase; +using cuvs::distance::kernels::detail::KernelFactory; -}; // end namespace raft::distance::kernels +}; // end namespace cuvs::distance::kernels diff --git a/cpp/include/cuvs/distance/masked_nn.cuh b/cpp/include/cuvs/distance/masked_nn.cuh index 33a6c0456..c86ffe6c8 100644 --- a/cpp/include/cuvs/distance/masked_nn.cuh +++ b/cpp/include/cuvs/distance/masked_nn.cuh @@ -19,10 +19,10 @@ #pragma once +#include +#include #include #include -#include -#include #include #include @@ -41,13 +41,13 @@ namespace distance { * * Usage example: * @code{.cpp} - * #include + * #include * * using IdxT = int; * using DataT = float; - * using RedOpT = raft::distance::MinAndDistanceReduceOp; - * using PairRedOpT = raft::distance::KVPMinReduce; - * using ParamT = raft::distance::masked_l2_nn_params; + * using RedOpT = cuvs::distance::MinAndDistanceReduceOp; + * using PairRedOpT = cuvs::distance::KVPMinReduce; + * using ParamT = cuvs::distance::masked_l2_nn_params; * * bool init_out = true; * bool sqrt = false; @@ -146,7 +146,7 @@ struct masked_l2_nn_params { */ template void masked_l2_nn(raft::resources const& handle, - raft::distance::masked_l2_nn_params params, + cuvs::distance::masked_l2_nn_params params, raft::device_matrix_view x, raft::device_matrix_view y, raft::device_vector_view x_norm, diff --git a/cpp/include/cuvs/distance/specializations.cuh b/cpp/include/cuvs/distance/specializations.cuh deleted file mode 100644 index ed0b6848a..000000000 --- a/cpp/include/cuvs/distance/specializations.cuh +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#pragma message( \ - __FILE__ \ - " is deprecated and will be removed." \ - " Including specializations is not necessary any more." \ - " For more information, see: https://docs.rapids.ai/api/raft/nightly/using_libraft.html") diff --git a/cpp/include/cuvs/distance/specializations/distance.cuh b/cpp/include/cuvs/distance/specializations/distance.cuh deleted file mode 100644 index ed0b6848a..000000000 --- a/cpp/include/cuvs/distance/specializations/distance.cuh +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#pragma message( \ - __FILE__ \ - " is deprecated and will be removed." \ - " Including specializations is not necessary any more." \ - " For more information, see: https://docs.rapids.ai/api/raft/nightly/using_libraft.html") diff --git a/cpp/include/cuvs/distance/specializations/fused_l2_nn_min.cuh b/cpp/include/cuvs/distance/specializations/fused_l2_nn_min.cuh deleted file mode 100644 index 9588a7f32..000000000 --- a/cpp/include/cuvs/distance/specializations/fused_l2_nn_min.cuh +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#pragma message( \ - __FILE__ \ - " is deprecated and will be removed." \ - " Including specializations is not necessary any more." \ - " For more information, see: https://docs.rapids.ai/api/raft/nightly/using_libraft.html") diff --git a/cpp/include/cuvs/neighbors/ann_types.hpp b/cpp/include/cuvs/neighbors/ann_types.hpp index c17be4a8f..36b545ebd 100644 --- a/cpp/include/cuvs/neighbors/ann_types.hpp +++ b/cpp/include/cuvs/neighbors/ann_types.hpp @@ -16,9 +16,9 @@ #pragma once -#include +#include -namespace raft::neighbors::ann { +namespace cuvs::neighbors::ann { /** * @defgroup ann_types Approximate Nearest Neighbors Types @@ -31,7 +31,7 @@ struct index {}; /** The base for KNN index parameters. */ struct index_params { /** Distance type. */ - raft::distance::DistanceType metric = distance::DistanceType::L2Expanded; + cuvs::distance::DistanceType metric = distance::DistanceType::L2Expanded; /** The argument used by some distance metrics. */ float metric_arg = 2.0f; /** @@ -49,4 +49,4 @@ struct search_params {}; /** @} */ // end group ann_types -}; // namespace raft::neighbors::ann +}; // namespace cuvs::neighbors::ann diff --git a/cpp/include/cuvs/neighbors/ball_cover-ext.cuh b/cpp/include/cuvs/neighbors/ball_cover-ext.cuh index bc5fe934a..199444542 100644 --- a/cpp/include/cuvs/neighbors/ball_cover-ext.cuh +++ b/cpp/include/cuvs/neighbors/ball_cover-ext.cuh @@ -16,13 +16,13 @@ #pragma once #include // uint32_t -#include // raft::distance::DistanceType -#include // BallCoverIndex +#include // cuvs::distance::DistanceType +#include // BallCoverIndex #include // RAFT_EXPLICIT #ifdef RAFT_EXPLICIT_INSTANTIATE_ONLY -namespace raft::neighbors::ball_cover { +namespace cuvs::neighbors::ball_cover { template void build_index(raft::resources const& handle, @@ -67,20 +67,20 @@ void knn_query(raft::resources const& handle, bool perform_post_filtering = true, float weight = 1.0) RAFT_EXPLICIT; -} // namespace raft::neighbors::ball_cover +} // namespace cuvs::neighbors::ball_cover #endif // RAFT_EXPLICIT_INSTANTIATE_ONLY #define instantiate_raft_neighbors_ball_cover(idx_t, value_t, int_t, matrix_idx_t) \ extern template void \ - raft::neighbors::ball_cover::build_index( \ + cuvs::neighbors::ball_cover::build_index( \ raft::resources const& handle, \ - raft::neighbors::ball_cover::BallCoverIndex& index); \ + cuvs::neighbors::ball_cover::BallCoverIndex& index); \ \ extern template void \ - raft::neighbors::ball_cover::all_knn_query( \ + cuvs::neighbors::ball_cover::all_knn_query( \ raft::resources const& handle, \ - raft::neighbors::ball_cover::BallCoverIndex& index, \ + cuvs::neighbors::ball_cover::BallCoverIndex& index, \ int_t k, \ idx_t* inds, \ value_t* dists, \ @@ -88,18 +88,18 @@ void knn_query(raft::resources const& handle, float weight); \ \ extern template void \ - raft::neighbors::ball_cover::all_knn_query( \ + cuvs::neighbors::ball_cover::all_knn_query( \ raft::resources const& handle, \ - raft::neighbors::ball_cover::BallCoverIndex& index, \ + cuvs::neighbors::ball_cover::BallCoverIndex& index, \ raft::device_matrix_view inds, \ raft::device_matrix_view dists, \ int_t k, \ bool perform_post_filtering, \ float weight); \ \ - extern template void raft::neighbors::ball_cover::knn_query( \ + extern template void cuvs::neighbors::ball_cover::knn_query( \ raft::resources const& handle, \ - const raft::neighbors::ball_cover::BallCoverIndex& index, \ + const cuvs::neighbors::ball_cover::BallCoverIndex& index, \ int_t k, \ const value_t* query, \ int_t n_query_pts, \ @@ -109,9 +109,9 @@ void knn_query(raft::resources const& handle, float weight); \ \ extern template void \ - raft::neighbors::ball_cover::knn_query( \ + cuvs::neighbors::ball_cover::knn_query( \ raft::resources const& handle, \ - const raft::neighbors::ball_cover::BallCoverIndex& index, \ + const cuvs::neighbors::ball_cover::BallCoverIndex& index, \ raft::device_matrix_view query, \ raft::device_matrix_view inds, \ raft::device_matrix_view dists, \ diff --git a/cpp/include/cuvs/neighbors/ball_cover-inl.cuh b/cpp/include/cuvs/neighbors/ball_cover-inl.cuh index d35c1dc61..1ecfa8b0a 100644 --- a/cpp/include/cuvs/neighbors/ball_cover-inl.cuh +++ b/cpp/include/cuvs/neighbors/ball_cover-inl.cuh @@ -20,13 +20,13 @@ #include -#include -#include +#include +#include #include #include #include -namespace raft::neighbors::ball_cover { +namespace cuvs::neighbors::ball_cover { /** * @defgroup random_ball_cover Random Ball Cover algorithm @@ -40,13 +40,13 @@ namespace raft::neighbors::ball_cover { * @code{.cpp} * * #include - * #include - * #include - * using namespace raft::neighbors; + * #include + * #include + * using namespace cuvs::neighbors; * * raft::resources handle; * ... - * auto metric = raft::distance::DistanceType::L2Expanded; + * auto metric = cuvs::distance::DistanceType::L2Expanded; * BallCoverIndex index(handle, X, metric); * * ball_cover::build_index(handle, index); @@ -64,11 +64,11 @@ void build_index(raft::resources const& handle, BallCoverIndex& index) { ASSERT(index.n <= 3, "only 2d and 3d vectors are supported in current implementation"); - if (index.metric == raft::distance::DistanceType::Haversine) { + if (index.metric == cuvs::distance::DistanceType::Haversine) { raft::spatial::knn::detail::rbc_build_index( handle, index, spatial::knn::detail::HaversineFunc()); - } else if (index.metric == raft::distance::DistanceType::L2SqrtExpanded || - index.metric == raft::distance::DistanceType::L2SqrtUnexpanded) { + } else if (index.metric == cuvs::distance::DistanceType::L2SqrtExpanded || + index.metric == cuvs::distance::DistanceType::L2SqrtUnexpanded) { raft::spatial::knn::detail::rbc_build_index( handle, index, spatial::knn::detail::EuclideanFunc()); } else { @@ -118,7 +118,7 @@ void all_knn_query(raft::resources const& handle, float weight = 1.0) { ASSERT(index.n <= 3, "only 2d and 3d vectors are supported in current implementation"); - if (index.metric == raft::distance::DistanceType::Haversine) { + if (index.metric == cuvs::distance::DistanceType::Haversine) { raft::spatial::knn::detail::rbc_all_knn_query( handle, index, @@ -128,8 +128,8 @@ void all_knn_query(raft::resources const& handle, spatial::knn::detail::HaversineFunc(), perform_post_filtering, weight); - } else if (index.metric == raft::distance::DistanceType::L2SqrtExpanded || - index.metric == raft::distance::DistanceType::L2SqrtUnexpanded) { + } else if (index.metric == cuvs::distance::DistanceType::L2SqrtExpanded || + index.metric == cuvs::distance::DistanceType::L2SqrtUnexpanded) { raft::spatial::knn::detail::rbc_all_knn_query( handle, index, @@ -164,13 +164,13 @@ void all_knn_query(raft::resources const& handle, * @code{.cpp} * * #include - * #include - * #include - * using namespace raft::neighbors; + * #include + * #include + * using namespace cuvs::neighbors; * * raft::resources handle; * ... - * auto metric = raft::distance::DistanceType::L2Expanded; + * auto metric = cuvs::distance::DistanceType::L2Expanded; * * // Construct a ball cover index * BallCoverIndex index(handle, X, metric); @@ -267,7 +267,7 @@ void knn_query(raft::resources const& handle, float weight = 1.0) { ASSERT(index.n <= 3, "only 2d and 3d vectors are supported in current implementation"); - if (index.metric == raft::distance::DistanceType::Haversine) { + if (index.metric == cuvs::distance::DistanceType::Haversine) { raft::spatial::knn::detail::rbc_knn_query(handle, index, k, @@ -278,8 +278,8 @@ void knn_query(raft::resources const& handle, spatial::knn::detail::HaversineFunc(), perform_post_filtering, weight); - } else if (index.metric == raft::distance::DistanceType::L2SqrtExpanded || - index.metric == raft::distance::DistanceType::L2SqrtUnexpanded) { + } else if (index.metric == cuvs::distance::DistanceType::L2SqrtExpanded || + index.metric == cuvs::distance::DistanceType::L2SqrtUnexpanded) { raft::spatial::knn::detail::rbc_knn_query(handle, index, k, @@ -312,13 +312,13 @@ void knn_query(raft::resources const& handle, * @code{.cpp} * * #include - * #include - * #include - * using namespace raft::neighbors; + * #include + * #include + * using namespace cuvs::neighbors; * * raft::resources handle; * ... - * auto metric = raft::distance::DistanceType::L2Expanded; + * auto metric = cuvs::distance::DistanceType::L2Expanded; * * // Build a ball cover index * BallCoverIndex index(handle, X, metric); @@ -390,6 +390,6 @@ void knn_query(raft::resources const& handle, // 4. rbc_eps_neigh() - given a populated index, perform query against different query array // 5. rbc_all_eps_neigh() - populate a BallCoverIndex and query against training data -} // namespace raft::neighbors::ball_cover +} // namespace cuvs::neighbors::ball_cover #endif diff --git a/cpp/include/cuvs/neighbors/ball_cover_types.hpp b/cpp/include/cuvs/neighbors/ball_cover_types.hpp index 0a6ad8c40..7b6203ec2 100644 --- a/cpp/include/cuvs/neighbors/ball_cover_types.hpp +++ b/cpp/include/cuvs/neighbors/ball_cover_types.hpp @@ -17,13 +17,13 @@ #pragma once #include +#include #include #include #include -#include #include -namespace raft::neighbors::ball_cover { +namespace cuvs::neighbors::ball_cover { /** * @ingroup random_ball_cover @@ -49,7 +49,7 @@ class BallCoverIndex { const value_t* X_, value_int m_, value_int n_, - raft::distance::DistanceType metric_) + cuvs::distance::DistanceType metric_) : handle(handle_), X(raft::make_device_matrix_view(X_, m_, n_)), m(m_), @@ -73,7 +73,7 @@ class BallCoverIndex { explicit BallCoverIndex(raft::resources const& handle_, raft::device_matrix_view X_, - raft::distance::DistanceType metric_) + cuvs::distance::DistanceType metric_) : handle(handle_), X(X_), m(X_.extent(0)), @@ -131,7 +131,7 @@ class BallCoverIndex { } raft::device_matrix_view get_X() const { return X; } - raft::distance::DistanceType get_metric() const { return metric; } + cuvs::distance::DistanceType get_metric() const { return metric; } value_int get_n_landmarks() const { return n_landmarks; } bool is_index_trained() const { return index_trained; }; @@ -147,7 +147,7 @@ class BallCoverIndex { raft::device_matrix_view X; - raft::distance::DistanceType metric; + cuvs::distance::DistanceType metric; private: // CSR storing the neighborhoods for each data point @@ -166,4 +166,4 @@ class BallCoverIndex { /** @} */ -} // namespace raft::neighbors::ball_cover +} // namespace cuvs::neighbors::ball_cover diff --git a/cpp/include/cuvs/neighbors/brute_force-ext.cuh b/cpp/include/cuvs/neighbors/brute_force-ext.cuh index 4c1f7ea21..bd9a1a86e 100644 --- a/cpp/include/cuvs/neighbors/brute_force-ext.cuh +++ b/cpp/include/cuvs/neighbors/brute_force-ext.cuh @@ -18,16 +18,16 @@ #include -#include // raft::device_matrix_view -#include // raft::identity_op -#include // raft::resources -#include // raft::distance::DistanceType -#include +#include // cuvs::distance::DistanceType +#include +#include // raft::device_matrix_view +#include // raft::identity_op +#include // raft::resources #include // RAFT_EXPLICIT #ifdef RAFT_EXPLICIT_INSTANTIATE_ONLY -namespace raft::neighbors::brute_force { +namespace cuvs::neighbors::brute_force { template inline void knn_merge_parts( @@ -42,7 +42,7 @@ inline void knn_merge_parts( template index build(raft::resources const& res, mdspan, row_major, Accessor> dataset, - raft::distance::DistanceType metric = distance::DistanceType::L2Unexpanded, + cuvs::distance::DistanceType metric = distance::DistanceType::L2Unexpanded, T metric_arg = 0.0) RAFT_EXPLICIT; template @@ -74,24 +74,24 @@ void fused_l2_knn(raft::resources const& handle, raft::device_matrix_view query, raft::device_matrix_view out_inds, raft::device_matrix_view out_dists, - raft::distance::DistanceType metric) RAFT_EXPLICIT; + cuvs::distance::DistanceType metric) RAFT_EXPLICIT; -} // namespace raft::neighbors::brute_force +} // namespace cuvs::neighbors::brute_force #endif // RAFT_EXPLICIT_INSTANTIATE_ONLY -// No extern template for raft::neighbors::brute_force::knn_merge_parts +// No extern template for cuvs::neighbors::brute_force::knn_merge_parts #define instantiate_raft_neighbors_brute_force_knn( \ idx_t, value_t, matrix_idx, index_layout, search_layout, epilogue_op) \ - extern template void raft::neighbors::brute_force:: \ + extern template void cuvs::neighbors::brute_force:: \ knn( \ raft::resources const& handle, \ std::vector> index, \ raft::device_matrix_view search, \ raft::device_matrix_view indices, \ raft::device_matrix_view distances, \ - raft::distance::DistanceType metric, \ + cuvs::distance::DistanceType metric, \ std::optional metric_arg, \ std::optional global_id_offset, \ epilogue_op distance_epilogue); @@ -107,38 +107,38 @@ instantiate_raft_neighbors_brute_force_knn( #undef instantiate_raft_neighbors_brute_force_knn -namespace raft::neighbors::brute_force { +namespace cuvs::neighbors::brute_force { extern template void search( raft::resources const& res, - const raft::neighbors::brute_force::index& idx, + const cuvs::neighbors::brute_force::index& idx, raft::device_matrix_view queries, raft::device_matrix_view neighbors, raft::device_matrix_view distances); extern template void search( raft::resources const& res, - const raft::neighbors::brute_force::index& idx, + const cuvs::neighbors::brute_force::index& idx, raft::device_matrix_view queries, raft::device_matrix_view neighbors, raft::device_matrix_view distances); -extern template raft::neighbors::brute_force::index build( +extern template cuvs::neighbors::brute_force::index build( raft::resources const& res, raft::device_matrix_view dataset, - raft::distance::DistanceType metric, + cuvs::distance::DistanceType metric, float metric_arg); -} // namespace raft::neighbors::brute_force +} // namespace cuvs::neighbors::brute_force #define instantiate_raft_neighbors_brute_force_fused_l2_knn( \ value_t, idx_t, idx_layout, query_layout) \ - extern template void raft::neighbors::brute_force::fused_l2_knn( \ + extern template void cuvs::neighbors::brute_force::fused_l2_knn( \ raft::resources const& handle, \ raft::device_matrix_view index, \ raft::device_matrix_view query, \ raft::device_matrix_view out_inds, \ raft::device_matrix_view out_dists, \ - raft::distance::DistanceType metric); + cuvs::distance::DistanceType metric); instantiate_raft_neighbors_brute_force_fused_l2_knn(float, int64_t, diff --git a/cpp/include/cuvs/neighbors/brute_force-inl.cuh b/cpp/include/cuvs/neighbors/brute_force-inl.cuh index 6b86f2463..25bc9e595 100644 --- a/cpp/include/cuvs/neighbors/brute_force-inl.cuh +++ b/cpp/include/cuvs/neighbors/brute_force-inl.cuh @@ -16,14 +16,14 @@ #pragma once +#include +#include +#include #include #include -#include -#include -#include #include -namespace raft::neighbors::brute_force { +namespace cuvs::neighbors::brute_force { /** * @defgroup brute_force_knn Brute-force K-Nearest Neighbors @@ -56,8 +56,8 @@ namespace raft::neighbors::brute_force { * Usage example: * @code{.cpp} * #include - * #include - * using namespace raft::neighbors; + * #include + * using namespace cuvs::neighbors; * * raft::resources handle; * ... @@ -122,13 +122,13 @@ inline void knn_merge_parts( * Usage example: * @code{.cpp} * #include - * #include - * #include - * using namespace raft::neighbors; + * #include + * #include + * using namespace cuvs::neighbors; * * raft::resources handle; * ... - * auto metric = raft::distance::DistanceType::L2SqrtExpanded; + * auto metric = cuvs::distance::DistanceType::L2SqrtExpanded; * brute_force::knn(handle, index, search, indices, distances, metric); * @endcode * @@ -187,7 +187,7 @@ void knn(raft::resources const& handle, std::vector* trans_arg = global_id_offset.has_value() ? &trans : nullptr; - raft::neighbors::detail::brute_force_knn_impl(handle, + cuvs::neighbors::detail::brute_force_knn_impl(handle, inputs, sizes, index[0].extent(1), @@ -215,13 +215,13 @@ void knn(raft::resources const& handle, * Usage example: * @code{.cpp} * #include - * #include - * #include - * using namespace raft::neighbors; + * #include + * #include + * using namespace cuvs::neighbors; * * raft::resources handle; * ... - * auto metric = raft::distance::DistanceType::L2SqrtExpanded; + * auto metric = cuvs::distance::DistanceType::L2SqrtExpanded; * brute_force::fused_l2_knn(handle, index, search, indices, distances, metric); * @endcode @@ -242,7 +242,7 @@ void fused_l2_knn(raft::resources const& handle, raft::device_matrix_view query, raft::device_matrix_view out_inds, raft::device_matrix_view out_dists, - raft::distance::DistanceType metric) + cuvs::distance::DistanceType metric) { int k = static_cast(out_inds.extent(1)); @@ -297,18 +297,18 @@ void fused_l2_knn(raft::resources const& handle, template index build(raft::resources const& res, mdspan, row_major, Accessor> dataset, - raft::distance::DistanceType metric = distance::DistanceType::L2Unexpanded, + cuvs::distance::DistanceType metric = distance::DistanceType::L2Unexpanded, T metric_arg = 0.0) { // certain distance metrics can benefit by pre-calculating the norms for the index dataset // which lets us avoid calculating these at query time std::optional> norms; - if (metric == raft::distance::DistanceType::L2Expanded || - metric == raft::distance::DistanceType::L2SqrtExpanded || - metric == raft::distance::DistanceType::CosineExpanded) { + if (metric == cuvs::distance::DistanceType::L2Expanded || + metric == cuvs::distance::DistanceType::L2SqrtExpanded || + metric == cuvs::distance::DistanceType::CosineExpanded) { norms = make_device_vector(res, dataset.extent(0)); // cosine needs the l2norm, where as l2 distances needs the squared norm - if (metric == raft::distance::DistanceType::CosineExpanded) { + if (metric == cuvs::distance::DistanceType::CosineExpanded) { raft::linalg::norm(res, dataset, norms->view(), @@ -348,7 +348,7 @@ void search(raft::resources const& res, raft::device_matrix_view neighbors, raft::device_matrix_view distances) { - raft::neighbors::detail::brute_force_search(res, idx, queries, neighbors, distances); + cuvs::neighbors::detail::brute_force_search(res, idx, queries, neighbors, distances); } /** @} */ // end group brute_force_knn -} // namespace raft::neighbors::brute_force +} // namespace cuvs::neighbors::brute_force diff --git a/cpp/include/cuvs/neighbors/brute_force.cuh b/cpp/include/cuvs/neighbors/brute_force.cuh index 4ba915955..867790f02 100644 --- a/cpp/include/cuvs/neighbors/brute_force.cuh +++ b/cpp/include/cuvs/neighbors/brute_force.cuh @@ -24,9 +24,9 @@ #include "brute_force-ext.cuh" #endif -#include +#include -namespace raft::neighbors::brute_force { +namespace cuvs::neighbors::brute_force { /** * @brief Make a brute force query over batches of k * @@ -35,7 +35,7 @@ namespace raft::neighbors::brute_force { * * Example usage: * @code{.cpp} - * #include + * #include * #include * #include @@ -50,7 +50,7 @@ namespace raft::neighbors::brute_force { * raft::make_blobs(res, dataset.view(), labels.view()); * * // create a brute_force knn index from the dataset - * auto index = raft::neighbors::brute_force::build(res, + * auto index = cuvs::neighbors::brute_force::build(res, * raft::make_const_mdspan(dataset.view())); * * // search the index in batches of 128 nearest neighbors @@ -82,11 +82,11 @@ namespace raft::neighbors::brute_force { template std::shared_ptr> make_batch_k_query( const raft::resources& res, - const raft::neighbors::brute_force::index& index, + const cuvs::neighbors::brute_force::index& index, raft::device_matrix_view query, int64_t batch_size) { return std::shared_ptr>( new detail::gpu_batch_k_query(res, index, query, batch_size)); } -} // namespace raft::neighbors::brute_force +} // namespace cuvs::neighbors::brute_force diff --git a/cpp/include/cuvs/neighbors/brute_force_types.hpp b/cpp/include/cuvs/neighbors/brute_force_types.hpp index 039599845..b2d60a92b 100644 --- a/cpp/include/cuvs/neighbors/brute_force_types.hpp +++ b/cpp/include/cuvs/neighbors/brute_force_types.hpp @@ -19,17 +19,17 @@ #include "ann_types.hpp" #include +#include +#include #include #include #include #include #include -#include -#include #include -namespace raft::neighbors::brute_force { +namespace cuvs::neighbors::brute_force { /** * @addtogroup brute_force_knn * @{ @@ -46,7 +46,7 @@ template struct index : ann::index { public: /** Distance metric used for retrieval */ - [[nodiscard]] constexpr inline raft::distance::DistanceType metric() const noexcept + [[nodiscard]] constexpr inline cuvs::distance::DistanceType metric() const noexcept { return metric_; } @@ -95,7 +95,7 @@ struct index : ann::index { index(raft::resources const& res, mdspan, row_major, data_accessor> dataset, std::optional>&& norms, - raft::distance::DistanceType metric, + cuvs::distance::DistanceType metric, T metric_arg = 0.0) : ann::index(), metric_(metric), @@ -116,7 +116,7 @@ struct index : ann::index { index(raft::resources const& res, raft::device_matrix_view dataset_view, std::optional> norms_view, - raft::distance::DistanceType metric, + cuvs::distance::DistanceType metric, T metric_arg = 0.0) : ann::index(), metric_(metric), @@ -153,7 +153,7 @@ struct index : ann::index { dataset_view_ = make_const_mdspan(dataset_.view()); } - raft::distance::DistanceType metric_; + cuvs::distance::DistanceType metric_; raft::device_matrix dataset_; std::optional> norms_; std::optional> norms_view_; @@ -174,7 +174,7 @@ struct index : ann::index { * * Note that this class is an abstract class without any cuda dependencies, meaning * that it doesn't require a cuda compiler to use - but also means it can't be directly - * instantiated. See the raft::neighbors::brute_force::make_batch_k_query + * instantiated. See the cuvs::neighbors::brute_force::make_batch_k_query * function for usage examples. * * @tparam T data element type @@ -192,11 +192,11 @@ class batch_k_query { } virtual ~batch_k_query() {} - using value_type = raft::neighbors::batch; + using value_type = cuvs::neighbors::batch; class iterator { public: - using value_type = raft::neighbors::batch; + using value_type = cuvs::neighbors::batch; using reference = const value_type&; using pointer = const value_type*; @@ -230,7 +230,7 @@ class batch_k_query { * Using operator++ means that we will load up the same batch_size for each * batch. This method allows us to get around this restriction, and load up * arbitrary batch sizes on each iteration. - * See raft::neighbors::brute_force::make_batch_k_query for a usage example. + * See cuvs::neighbors::brute_force::make_batch_k_query for a usage example. * * @param[in] next_batch_size: size of the next batch to load up */ @@ -279,4 +279,4 @@ class batch_k_query { }; /** @} */ -} // namespace raft::neighbors::brute_force +} // namespace cuvs::neighbors::brute_force diff --git a/cpp/include/cuvs/neighbors/cagra.cuh b/cpp/include/cuvs/neighbors/cagra.cuh index 384ed05e1..f748ba595 100644 --- a/cpp/include/cuvs/neighbors/cagra.cuh +++ b/cpp/include/cuvs/neighbors/cagra.cuh @@ -20,14 +20,14 @@ #include "detail/cagra/cagra_search.cuh" #include "detail/cagra/graph_core.cuh" +#include #include #include #include #include -#include #include -namespace raft::neighbors::cagra { +namespace cuvs::neighbors::cagra { /** * @defgroup cagra CUDA ANN Graph-based nearest neighbor search @@ -49,7 +49,7 @@ namespace raft::neighbors::cagra { * * Usage example: * @code{.cpp} - * using namespace raft::neighbors; + * using namespace cuvs::neighbors; * // use default index parameters * ivf_pq::index_params build_params; * ivf_pq::search_params search_params @@ -109,8 +109,8 @@ void build_knn_graph(raft::resources const& res, * * Usage example: * @code{.cpp} - * using namespace raft::neighbors; - * using namespace raft::neighbors::experimental; + * using namespace cuvs::neighbors; + * using namespace cuvs::neighbors::experimental; * // use default index parameters * nn_descent::index_params build_params; * build_params.graph_degree = 128; @@ -155,7 +155,7 @@ void build_knn_graph(raft::resources const& res, * * Usage example: * @code{.cpp} - * using namespace raft::neighbors; + * using namespace cuvs::neighbors; * cagra::index_params build_params; * auto knn_graph = raft::make_host_matrix(dataset.extent(0), 128); * // build KNN graph not using `cagra::build_knn_graph` @@ -245,7 +245,7 @@ void optimize(raft::resources const& res, * * Usage example: * @code{.cpp} - * using namespace raft::neighbors; + * using namespace cuvs::neighbors; * // use default index parameters * cagra::index_params index_params; * // create and fill the index from a [N, D] dataset @@ -324,14 +324,14 @@ void search(raft::resources const& res, cagra::detail::search_main(res, params, idx, queries_internal, neighbors_internal, distances_internal, - raft::neighbors::filtering::none_cagra_sample_filter()); + cuvs::neighbors::filtering::none_cagra_sample_filter()); } /** @@ -339,7 +339,7 @@ void search(raft::resources const& res, * * Usage example: * @code{.cpp} - * using namespace raft::neighbors; + * using namespace cuvs::neighbors; * // use default index parameters * cagra::index_params index_params; * // create and fill the index from a [N, D] dataset @@ -406,13 +406,13 @@ void search_with_filtering(raft::resources const& res, /** @} */ // end group cagra -} // namespace raft::neighbors::cagra +} // namespace cuvs::neighbors::cagra // TODO: Remove deprecated experimental namespace in 23.12 release -namespace raft::neighbors::experimental::cagra { -using raft::neighbors::cagra::build; -using raft::neighbors::cagra::build_knn_graph; -using raft::neighbors::cagra::optimize; -using raft::neighbors::cagra::search; -using raft::neighbors::cagra::sort_knn_graph; -} // namespace raft::neighbors::experimental::cagra +namespace cuvs::neighbors::experimental::cagra { +using cuvs::neighbors::cagra::build; +using cuvs::neighbors::cagra::build_knn_graph; +using cuvs::neighbors::cagra::optimize; +using cuvs::neighbors::cagra::search; +using cuvs::neighbors::cagra::sort_knn_graph; +} // namespace cuvs::neighbors::experimental::cagra diff --git a/cpp/include/cuvs/neighbors/cagra_serialize.cuh b/cpp/include/cuvs/neighbors/cagra_serialize.cuh index c801bc9ed..ee492ea8c 100644 --- a/cpp/include/cuvs/neighbors/cagra_serialize.cuh +++ b/cpp/include/cuvs/neighbors/cagra_serialize.cuh @@ -18,7 +18,7 @@ #include "detail/cagra/cagra_serialize.cuh" -namespace raft::neighbors::cagra { +namespace cuvs::neighbors::cagra { /** * \defgroup cagra_serialize CAGRA Serialize @@ -180,7 +180,7 @@ void serialize_to_hnswlib(raft::resources const& handle, * @param[in] handle the raft handle * @param[in] is input stream * - * @return raft::neighbors::experimental::cagra::index + * @return cuvs::neighbors::experimental::cagra::index */ template index deserialize(raft::resources const& handle, std::istream& is) @@ -211,7 +211,7 @@ index deserialize(raft::resources const& handle, std::istream& is) * @param[in] handle the raft handle * @param[in] filename the name of the file that stores the index * - * @return raft::neighbors::experimental::cagra::index + * @return cuvs::neighbors::experimental::cagra::index */ template index deserialize(raft::resources const& handle, const std::string& filename) @@ -221,11 +221,11 @@ index deserialize(raft::resources const& handle, const std::string& fil /**@}*/ -} // namespace raft::neighbors::cagra +} // namespace cuvs::neighbors::cagra // TODO: Remove deprecated experimental namespace in 23.12 release -namespace raft::neighbors::experimental::cagra { -using raft::neighbors::cagra::deserialize; -using raft::neighbors::cagra::serialize; +namespace cuvs::neighbors::experimental::cagra { +using cuvs::neighbors::cagra::deserialize; +using cuvs::neighbors::cagra::serialize; -} // namespace raft::neighbors::experimental::cagra +} // namespace cuvs::neighbors::experimental::cagra diff --git a/cpp/include/cuvs/neighbors/cagra_types.hpp b/cpp/include/cuvs/neighbors/cagra_types.hpp index e8a0b8a7b..ea5954cd8 100644 --- a/cpp/include/cuvs/neighbors/cagra_types.hpp +++ b/cpp/include/cuvs/neighbors/cagra_types.hpp @@ -19,13 +19,13 @@ #include "ann_types.hpp" #include +#include +#include #include #include #include #include #include -#include -#include #include #include @@ -35,7 +35,7 @@ #include #include -namespace raft::neighbors::cagra { +namespace cuvs::neighbors::cagra { /** * @addtogroup cagra * @{ @@ -137,7 +137,7 @@ struct index : ann::index { public: /** Distance metric used for clustering. */ - [[nodiscard]] constexpr inline auto metric() const noexcept -> raft::distance::DistanceType + [[nodiscard]] constexpr inline auto metric() const noexcept -> cuvs::distance::DistanceType { return metric_; } @@ -182,7 +182,7 @@ struct index : ann::index { /** Construct an empty index. */ index(raft::resources const& res, - raft::distance::DistanceType metric = raft::distance::DistanceType::L2Expanded) + cuvs::distance::DistanceType metric = cuvs::distance::DistanceType::L2Expanded) : ann::index(), metric_(metric), dataset_(make_device_matrix(res, 0, 0)), @@ -205,7 +205,7 @@ struct index : ann::index { * * - Cagra index is normally created by the cagra::build * @code{.cpp} - * using namespace raft::neighbors::experimental; + * using namespace cuvs::neighbors::experimental; * auto dataset = raft::make_host_matrix(n_rows, n_cols); * load_dataset(dataset.view()); * // use default index parameters @@ -225,7 +225,7 @@ struct index : ann::index { * * - Constructing index using existing knn-graph * @code{.cpp} - * using namespace raft::neighbors::experimental; + * using namespace cuvs::neighbors::experimental; * * auto dataset = raft::make_device_matrix(res, n_rows, n_cols); * auto knn_graph = raft::make_device_matrix(res, n_rows, graph_degree); @@ -246,7 +246,7 @@ struct index : ann::index { */ template index(raft::resources const& res, - raft::distance::DistanceType metric, + cuvs::distance::DistanceType metric, mdspan, row_major, data_accessor> dataset, mdspan, row_major, graph_accessor> knn_graph) : ann::index(), @@ -341,7 +341,7 @@ struct index : ann::index { static_cast(dataset_view_.stride(0))); } - raft::distance::DistanceType metric_; + cuvs::distance::DistanceType metric_; raft::device_matrix dataset_; raft::device_matrix graph_; raft::device_matrix_view dataset_view_; @@ -350,14 +350,14 @@ struct index : ann::index { /** @} */ -} // namespace raft::neighbors::cagra +} // namespace cuvs::neighbors::cagra // TODO: Remove deprecated experimental namespace in 23.12 release -namespace raft::neighbors::experimental::cagra { -using raft::neighbors::cagra::graph_build_algo; -using raft::neighbors::cagra::hash_mode; -using raft::neighbors::cagra::index; -using raft::neighbors::cagra::index_params; -using raft::neighbors::cagra::search_algo; -using raft::neighbors::cagra::search_params; -} // namespace raft::neighbors::experimental::cagra +namespace cuvs::neighbors::experimental::cagra { +using cuvs::neighbors::cagra::graph_build_algo; +using cuvs::neighbors::cagra::hash_mode; +using cuvs::neighbors::cagra::index; +using cuvs::neighbors::cagra::index_params; +using cuvs::neighbors::cagra::search_algo; +using cuvs::neighbors::cagra::search_params; +} // namespace cuvs::neighbors::experimental::cagra diff --git a/cpp/include/cuvs/neighbors/detail/cagra/bitonic.hpp b/cpp/include/cuvs/neighbors/detail/cagra/bitonic.hpp index 9fca7f8eb..d1fa0b41a 100644 --- a/cpp/include/cuvs/neighbors/detail/cagra/bitonic.hpp +++ b/cpp/include/cuvs/neighbors/detail/cagra/bitonic.hpp @@ -18,7 +18,7 @@ #include #include -namespace raft::neighbors::cagra::detail { +namespace cuvs::neighbors::cagra::detail { namespace bitonic { namespace detail { @@ -223,4 +223,4 @@ __device__ void warp_sort(K k[N], V v[N], const bool asc = true) } } // namespace bitonic -} // namespace raft::neighbors::cagra::detail +} // namespace cuvs::neighbors::cagra::detail diff --git a/cpp/include/cuvs/neighbors/detail/cagra/cagra_build.cuh b/cpp/include/cuvs/neighbors/detail/cagra/cagra_build.cuh index ddaf77a22..e6afea806 100644 --- a/cpp/include/cuvs/neighbors/detail/cagra/cagra_build.cuh +++ b/cpp/include/cuvs/neighbors/detail/cagra/cagra_build.cuh @@ -22,6 +22,7 @@ #include #include +#include #include #include #include @@ -29,16 +30,15 @@ #include #include #include -#include #include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include -namespace raft::neighbors::cagra::detail { +namespace cuvs::neighbors::cagra::detail { template void build_knn_graph(raft::resources const& res, @@ -48,7 +48,7 @@ void build_knn_graph(raft::resources const& res, std::optional build_params = std::nullopt, std::optional search_params = std::nullopt) { - resource::detail::warn_non_pool_workspace(res, "raft::neighbors::cagra::build"); + resource::detail::warn_non_pool_workspace(res, "cuvs::neighbors::cagra::build"); RAFT_EXPECTS(!build_params || build_params->metric == distance::DistanceType::L2Expanded, "Currently only L2Expanded metric is supported"); @@ -170,7 +170,7 @@ void build_knn_graph(raft::resources const& res, refined_distances_host.data_handle(), batch.size(), top_k); resource::sync_stream(res); - raft::neighbors::detail::refine_host( + cuvs::neighbors::detail::refine_host( dataset, queries_host_view, neighbors_host_view, @@ -187,7 +187,7 @@ void build_knn_graph(raft::resources const& res, auto dataset_view = make_device_matrix_view( dataset.data_handle(), dataset.extent(0), dataset.extent(1)); - raft::neighbors::detail::refine_device( + cuvs::neighbors::detail::refine_device( res, dataset_view, queries_view, @@ -346,4 +346,4 @@ index build( // Construct an index from dataset and optimized knn graph. return index(res, params.metric, dataset, raft::make_const_mdspan(cagra_graph.view())); } -} // namespace raft::neighbors::cagra::detail +} // namespace cuvs::neighbors::cagra::detail diff --git a/cpp/include/cuvs/neighbors/detail/cagra/cagra_search.cuh b/cpp/include/cuvs/neighbors/detail/cagra/cagra_search.cuh index 81e714dc4..cd9ce7335 100644 --- a/cpp/include/cuvs/neighbors/detail/cagra/cagra_search.cuh +++ b/cpp/include/cuvs/neighbors/detail/cagra/cagra_search.cuh @@ -16,24 +16,24 @@ #pragma once +#include +#include #include -#include -#include #include +#include #include #include #include #include #include -#include #include #include "factory.cuh" #include "search_plan.cuh" #include "search_single_cta.cuh" -namespace raft::neighbors::cagra::detail { +namespace cuvs::neighbors::cagra::detail { template struct CagraSampleFilterWithQueryIdOffset { @@ -56,8 +56,8 @@ struct CagraSampleFilterT_Selector { using type = CagraSampleFilterWithQueryIdOffset; }; template <> -struct CagraSampleFilterT_Selector { - using type = raft::neighbors::filtering::none_cagra_sample_filter; +struct CagraSampleFilterT_Selector { + using type = cuvs::neighbors::filtering::none_cagra_sample_filter; }; // A helper function to set a query id offset @@ -70,9 +70,9 @@ inline typename CagraSampleFilterT_Selector::type set_offset } template <> inline - typename CagraSampleFilterT_Selector::type - set_offset( - raft::neighbors::filtering::none_cagra_sample_filter filter, const uint32_t) + typename CagraSampleFilterT_Selector::type + set_offset( + cuvs::neighbors::filtering::none_cagra_sample_filter filter, const uint32_t) { return filter; } @@ -110,7 +110,7 @@ void search_main(raft::resources const& res, raft::device_matrix_view distances, CagraSampleFilterT sample_filter = CagraSampleFilterT()) { - resource::detail::warn_non_pool_workspace(res, "raft::neighbors::cagra::search"); + resource::detail::warn_non_pool_workspace(res, "cuvs::neighbors::cagra::search"); RAFT_LOG_DEBUG("# dataset size = %lu, dim = %lu\n", static_cast(index.dataset().extent(0)), static_cast(index.dataset().extent(1))); @@ -190,4 +190,4 @@ void search_main(raft::resources const& res, } /** @} */ // end group cagra -} // namespace raft::neighbors::cagra::detail +} // namespace cuvs::neighbors::cagra::detail diff --git a/cpp/include/cuvs/neighbors/detail/cagra/cagra_serialize.cuh b/cpp/include/cuvs/neighbors/detail/cagra/cagra_serialize.cuh index 51c947543..25ab7a873 100644 --- a/cpp/include/cuvs/neighbors/detail/cagra/cagra_serialize.cuh +++ b/cpp/include/cuvs/neighbors/detail/cagra/cagra_serialize.cuh @@ -18,18 +18,18 @@ #include #include +#include #include #include #include #include #include #include -#include #include #include -namespace raft::neighbors::cagra::detail { +namespace cuvs::neighbors::cagra::detail { constexpr int serialization_version = 3; @@ -245,7 +245,7 @@ auto deserialize(raft::resources const& res, std::istream& is) -> index auto n_rows = deserialize_scalar(res, is); auto dim = deserialize_scalar(res, is); auto graph_degree = deserialize_scalar(res, is); - auto metric = deserialize_scalar(res, is); + auto metric = deserialize_scalar(res, is); auto graph = raft::make_host_matrix(n_rows, graph_degree); deserialize_mdspan(res, is, graph.view()); @@ -278,4 +278,4 @@ auto deserialize(raft::resources const& res, const std::string& filename) -> ind return index; } -} // namespace raft::neighbors::cagra::detail +} // namespace cuvs::neighbors::cagra::detail diff --git a/cpp/include/cuvs/neighbors/detail/cagra/compute_distance.hpp b/cpp/include/cuvs/neighbors/detail/cagra/compute_distance.hpp index 55b7b4750..f4f2e654a 100644 --- a/cpp/include/cuvs/neighbors/detail/cagra/compute_distance.hpp +++ b/cpp/include/cuvs/neighbors/detail/cagra/compute_distance.hpp @@ -22,7 +22,7 @@ #include "utils.hpp" #include -namespace raft::neighbors::cagra::detail { +namespace cuvs::neighbors::cagra::detail { namespace device { // using LOAD_256BIT_T = ulonglong4; @@ -257,4 +257,4 @@ _RAFT_DEVICE void compute_distance_to_child_nodes(INDEX_T* const result_child_in } } // namespace device -} // namespace raft::neighbors::cagra::detail +} // namespace cuvs::neighbors::cagra::detail diff --git a/cpp/include/cuvs/neighbors/detail/cagra/device_common.hpp b/cpp/include/cuvs/neighbors/detail/cagra/device_common.hpp index b1a2207a4..82139ef59 100644 --- a/cpp/include/cuvs/neighbors/detail/cagra/device_common.hpp +++ b/cpp/include/cuvs/neighbors/detail/cagra/device_common.hpp @@ -21,7 +21,7 @@ #include #include -namespace raft::neighbors::cagra::detail { +namespace cuvs::neighbors::cagra::detail { namespace device { // warpSize for compile time calculation @@ -49,4 +49,4 @@ _RAFT_DEVICE inline T swizzling(T x) } } // namespace device -} // namespace raft::neighbors::cagra::detail +} // namespace cuvs::neighbors::cagra::detail diff --git a/cpp/include/cuvs/neighbors/detail/cagra/factory.cuh b/cpp/include/cuvs/neighbors/detail/cagra/factory.cuh index 78111a931..abe8d28a5 100644 --- a/cpp/include/cuvs/neighbors/detail/cagra/factory.cuh +++ b/cpp/include/cuvs/neighbors/detail/cagra/factory.cuh @@ -20,14 +20,14 @@ #include "search_multi_kernel.cuh" #include "search_plan.cuh" #include "search_single_cta.cuh" -#include +#include -namespace raft::neighbors::cagra::detail { +namespace cuvs::neighbors::cagra::detail { template + typename CagraSampleFilterT = cuvs::neighbors::filtering::none_cagra_sample_filter> class factory { public: /** @@ -94,4 +94,4 @@ class factory { } } }; -}; // namespace raft::neighbors::cagra::detail +}; // namespace cuvs::neighbors::cagra::detail diff --git a/cpp/include/cuvs/neighbors/detail/cagra/fragment.hpp b/cpp/include/cuvs/neighbors/detail/cagra/fragment.hpp index e124b3fc8..256e46627 100644 --- a/cpp/include/cuvs/neighbors/detail/cagra/fragment.hpp +++ b/cpp/include/cuvs/neighbors/detail/cagra/fragment.hpp @@ -20,7 +20,7 @@ #include #include -namespace raft::neighbors::cagra::detail { +namespace cuvs::neighbors::cagra::detail { namespace device { namespace detail { @@ -208,4 +208,4 @@ _RAFT_DEVICE void print_fragment(const device::fragment& a) } } // namespace device -} // namespace raft::neighbors::cagra::detail +} // namespace cuvs::neighbors::cagra::detail diff --git a/cpp/include/cuvs/neighbors/detail/cagra/graph_core.cuh b/cpp/include/cuvs/neighbors/detail/cagra/graph_core.cuh index 192850cf2..160545bee 100644 --- a/cpp/include/cuvs/neighbors/detail/cagra/graph_core.cuh +++ b/cpp/include/cuvs/neighbors/detail/cagra/graph_core.cuh @@ -36,7 +36,7 @@ #include "utils.hpp" -namespace raft::neighbors::cagra::detail { +namespace cuvs::neighbors::cagra::detail { namespace graph { // unnamed namespace to avoid multiple definition error @@ -570,4 +570,4 @@ void optimize(raft::resources const& res, } } // namespace graph -} // namespace raft::neighbors::cagra::detail +} // namespace cuvs::neighbors::cagra::detail diff --git a/cpp/include/cuvs/neighbors/detail/cagra/hashmap.hpp b/cpp/include/cuvs/neighbors/detail/cagra/hashmap.hpp index ed4763e47..2ac7438a9 100644 --- a/cpp/include/cuvs/neighbors/detail/cagra/hashmap.hpp +++ b/cpp/include/cuvs/neighbors/detail/cagra/hashmap.hpp @@ -23,7 +23,7 @@ // #pragma GCC diagnostic push // #pragma GCC diagnostic ignored // #pragma GCC diagnostic pop -namespace raft::neighbors::cagra::detail { +namespace cuvs::neighbors::cagra::detail { namespace hashmap { _RAFT_HOST_DEVICE inline uint32_t get_size(const uint32_t bitlen) { return 1U << bitlen; } @@ -76,4 +76,4 @@ _RAFT_DEVICE inline uint32_t insert(IdxT* const table, const uint32_t bitlen, co } } // namespace hashmap -} // namespace raft::neighbors::cagra::detail +} // namespace cuvs::neighbors::cagra::detail diff --git a/cpp/include/cuvs/neighbors/detail/cagra/search_multi_cta.cuh b/cpp/include/cuvs/neighbors/detail/cagra/search_multi_cta.cuh index c6478bef8..e54ba6fee 100644 --- a/cpp/include/cuvs/neighbors/detail/cagra/search_multi_cta.cuh +++ b/cpp/include/cuvs/neighbors/detail/cagra/search_multi_cta.cuh @@ -41,7 +41,7 @@ #include #include // RAFT_CUDA_TRY_NOT_THROW is used TODO(tfeher): consider moving this to cuda_rt_essentials.hpp -namespace raft::neighbors::cagra::detail { +namespace cuvs::neighbors::cagra::detail { namespace multi_cta_search { template // none_cagra_sample_filter +#include // none_cagra_sample_filter #include // RAFT_EXPLICIT -namespace raft::neighbors::cagra::detail { +namespace cuvs::neighbors::cagra::detail { namespace multi_cta_search { #ifdef RAFT_EXPLICIT_INSTANTIATE_ONLY @@ -85,30 +85,30 @@ void select_and_run(raft::device_matrix_view #include +#include #include #include #include @@ -26,7 +27,6 @@ #include #include #include -#include #include @@ -41,7 +41,7 @@ #include #include // RAFT_CUDA_TRY_NOT_THROW is used TODO(tfeher): consider moving this to cuda_rt_essentials.hpp -namespace raft::neighbors::cagra::detail { +namespace cuvs::neighbors::cagra::detail { namespace multi_cta_search { // #define _CLK_BREAKDOWN @@ -292,7 +292,7 @@ __launch_bounds__(1024, 1) RAFT_KERNEL search_kernel( // Filtering if constexpr (!std::is_same::value) { + cuvs::neighbors::filtering::none_cagra_sample_filter>::value) { constexpr INDEX_T index_msb_1_mask = utils::gen_index_msb_1_mask::value; const INDEX_T invalid_index = utils::get_max_value(); @@ -315,7 +315,7 @@ __launch_bounds__(1024, 1) RAFT_KERNEL search_kernel( // Post process for filtering if constexpr (!std::is_same::value) { + cuvs::neighbors::filtering::none_cagra_sample_filter>::value) { constexpr INDEX_T index_msb_1_mask = utils::gen_index_msb_1_mask::value; const INDEX_T invalid_index = utils::get_max_value(); @@ -527,4 +527,4 @@ void select_and_run( // raft::resources const& res, } } // namespace multi_cta_search -} // namespace raft::neighbors::cagra::detail +} // namespace cuvs::neighbors::cagra::detail diff --git a/cpp/include/cuvs/neighbors/detail/cagra/search_multi_kernel.cuh b/cpp/include/cuvs/neighbors/detail/cagra/search_multi_kernel.cuh index eddd954e9..97d151d40 100644 --- a/cpp/include/cuvs/neighbors/detail/cagra/search_multi_kernel.cuh +++ b/cpp/include/cuvs/neighbors/detail/cagra/search_multi_kernel.cuh @@ -19,13 +19,13 @@ #include #include +#include #include #include #include #include #include #include -#include #include #include #include @@ -41,7 +41,7 @@ #include #include // RAFT_CUDA_TRY_NOT_THROW is used TODO(tfeher): consider moving this to cuda_rt_essentials.hpp -namespace raft::neighbors::cagra::detail { +namespace cuvs::neighbors::cagra::detail { namespace multi_kernel_search { template @@ -376,7 +376,7 @@ RAFT_KERNEL compute_distance_to_child_nodes_kernel( } if constexpr (!std::is_same::value) { + cuvs::neighbors::filtering::none_cagra_sample_filter>::value) { if (!sample_filter(query_id, parent_index)) { parent_candidates_ptr[parent_list_index + (lds * query_id)] = utils::get_max_value(); parent_distance_ptr[parent_list_index + (lds * query_id)] = @@ -790,7 +790,7 @@ struct search : search_plan_impl { auto result_distances_ptr = result_distances.data() + (iter & 0x1) * result_buffer_size; if constexpr (!std::is_same::value) { + cuvs::neighbors::filtering::none_cagra_sample_filter>::value) { // Remove parent bit in search results remove_parent_bit(num_queries, result_buffer_size, @@ -859,4 +859,4 @@ struct search : search_plan_impl { }; } // namespace multi_kernel_search -} // namespace raft::neighbors::cagra::detail +} // namespace cuvs::neighbors::cagra::detail diff --git a/cpp/include/cuvs/neighbors/detail/cagra/search_plan.cuh b/cpp/include/cuvs/neighbors/detail/cagra/search_plan.cuh index 147b8b753..410c323d6 100644 --- a/cpp/include/cuvs/neighbors/detail/cagra/search_plan.cuh +++ b/cpp/include/cuvs/neighbors/detail/cagra/search_plan.cuh @@ -21,12 +21,12 @@ // #include "search_single_cta.cuh" // #include "topk_for_cagra/topk_core.cuh" +#include #include #include -#include #include -namespace raft::neighbors::cagra::detail { +namespace cuvs::neighbors::cagra::detail { struct search_plan_impl_base : public search_params { int64_t max_dim; @@ -292,7 +292,7 @@ struct search_plan_impl : public search_plan_impl_base { std::to_string(hashmap_max_fill_rate) + " has been given."; } if constexpr (!std::is_same::value) { + cuvs::neighbors::filtering::none_cagra_sample_filter>::value) { if (hashmap_mode == hash_mode::SMALL) { error_message += "`SMALL` hash is not available when filtering"; } else { @@ -327,4 +327,4 @@ struct search_plan_impl : public search_plan_impl_base { // }; /** @} */ // end group cagra -} // namespace raft::neighbors::cagra::detail +} // namespace cuvs::neighbors::cagra::detail diff --git a/cpp/include/cuvs/neighbors/detail/cagra/search_single_cta.cuh b/cpp/include/cuvs/neighbors/detail/cagra/search_single_cta.cuh index b36bc6f77..fc45bc9f0 100644 --- a/cpp/include/cuvs/neighbors/detail/cagra/search_single_cta.cuh +++ b/cpp/include/cuvs/neighbors/detail/cagra/search_single_cta.cuh @@ -42,7 +42,7 @@ #include #include // RAFT_CUDA_TRY_NOT_THROW is used TODO(tfeher): consider moving this to cuda_rt_essentials.hpp -namespace raft::neighbors::cagra::detail { +namespace cuvs::neighbors::cagra::detail { namespace single_cta_search { template { }; } // namespace single_cta_search -} // namespace raft::neighbors::cagra::detail +} // namespace cuvs::neighbors::cagra::detail diff --git a/cpp/include/cuvs/neighbors/detail/cagra/search_single_cta_kernel-ext.cuh b/cpp/include/cuvs/neighbors/detail/cagra/search_single_cta_kernel-ext.cuh index 35d239563..38eb6d9ba 100644 --- a/cpp/include/cuvs/neighbors/detail/cagra/search_single_cta_kernel-ext.cuh +++ b/cpp/include/cuvs/neighbors/detail/cagra/search_single_cta_kernel-ext.cuh @@ -15,10 +15,10 @@ */ #pragma once -#include +#include #include // RAFT_EXPLICIT -namespace raft::neighbors::cagra::detail { +namespace cuvs::neighbors::cagra::detail { namespace single_cta_search { #ifdef RAFT_EXPLICIT_INSTANTIATE_ONLY @@ -89,31 +89,31 @@ void select_and_run( // raft::resources const& res, cudaStream_t stream); instantiate_single_cta_select_and_run( - 32, 1024, float, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); + 32, 1024, float, uint32_t, float, cuvs::neighbors::filtering::none_cagra_sample_filter); instantiate_single_cta_select_and_run( - 8, 128, float, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); + 8, 128, float, uint32_t, float, cuvs::neighbors::filtering::none_cagra_sample_filter); instantiate_single_cta_select_and_run( - 16, 256, float, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); + 16, 256, float, uint32_t, float, cuvs::neighbors::filtering::none_cagra_sample_filter); instantiate_single_cta_select_and_run( - 32, 512, float, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); + 32, 512, float, uint32_t, float, cuvs::neighbors::filtering::none_cagra_sample_filter); instantiate_single_cta_select_and_run( - 32, 1024, int8_t, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); + 32, 1024, int8_t, uint32_t, float, cuvs::neighbors::filtering::none_cagra_sample_filter); instantiate_single_cta_select_and_run( - 8, 128, int8_t, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); + 8, 128, int8_t, uint32_t, float, cuvs::neighbors::filtering::none_cagra_sample_filter); instantiate_single_cta_select_and_run( - 16, 256, int8_t, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); + 16, 256, int8_t, uint32_t, float, cuvs::neighbors::filtering::none_cagra_sample_filter); instantiate_single_cta_select_and_run( - 32, 512, int8_t, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); + 32, 512, int8_t, uint32_t, float, cuvs::neighbors::filtering::none_cagra_sample_filter); instantiate_single_cta_select_and_run( - 32, 1024, uint8_t, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); + 32, 1024, uint8_t, uint32_t, float, cuvs::neighbors::filtering::none_cagra_sample_filter); instantiate_single_cta_select_and_run( - 8, 128, uint8_t, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); + 8, 128, uint8_t, uint32_t, float, cuvs::neighbors::filtering::none_cagra_sample_filter); instantiate_single_cta_select_and_run( - 16, 256, uint8_t, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); + 16, 256, uint8_t, uint32_t, float, cuvs::neighbors::filtering::none_cagra_sample_filter); instantiate_single_cta_select_and_run( - 32, 512, uint8_t, uint32_t, float, raft::neighbors::filtering::none_cagra_sample_filter); + 32, 512, uint8_t, uint32_t, float, cuvs::neighbors::filtering::none_cagra_sample_filter); #undef instantiate_single_cta_select_and_run } // namespace single_cta_search -} // namespace raft::neighbors::cagra::detail +} // namespace cuvs::neighbors::cagra::detail diff --git a/cpp/include/cuvs/neighbors/detail/cagra/search_single_cta_kernel-inl.cuh b/cpp/include/cuvs/neighbors/detail/cagra/search_single_cta_kernel-inl.cuh index 96535e5f2..c009769f2 100644 --- a/cpp/include/cuvs/neighbors/detail/cagra/search_single_cta_kernel-inl.cuh +++ b/cpp/include/cuvs/neighbors/detail/cagra/search_single_cta_kernel-inl.cuh @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -25,7 +26,6 @@ #include #include #include -#include #include #include #include @@ -42,7 +42,7 @@ #include #include // RAFT_CUDA_TRY_NOT_THROW is used TODO(tfeher): consider moving this to cuda_rt_essentials.hpp -namespace raft::neighbors::cagra::detail { +namespace cuvs::neighbors::cagra::detail { namespace single_cta_search { // #define _CLK_BREAKDOWN @@ -625,7 +625,7 @@ __launch_bounds__(1024, 1) RAFT_KERNEL // topk with bitonic sort _CLK_START(); if (std::is_same::value || + cuvs::neighbors::filtering::none_cagra_sample_filter>::value || *filter_flag == 0) { topk_by_bitonic_sort(result_distances_buffer, result_indices_buffer, @@ -722,7 +722,7 @@ __launch_bounds__(1024, 1) RAFT_KERNEL // Filtering if constexpr (!std::is_same::value) { + cuvs::neighbors::filtering::none_cagra_sample_filter>::value) { if (threadIdx.x == 0) { *filter_flag = 0; } __syncthreads(); @@ -748,7 +748,7 @@ __launch_bounds__(1024, 1) RAFT_KERNEL // Post process for filtering if constexpr (!std::is_same::value) { + cuvs::neighbors::filtering::none_cagra_sample_filter>::value) { constexpr INDEX_T index_msb_1_mask = utils::gen_index_msb_1_mask::value; const INDEX_T invalid_index = utils::get_max_value(); @@ -953,4 +953,4 @@ void select_and_run( // raft::resources const& res, RAFT_CUDA_TRY(cudaPeekAtLastError()); } } // namespace single_cta_search -} // namespace raft::neighbors::cagra::detail +} // namespace cuvs::neighbors::cagra::detail diff --git a/cpp/include/cuvs/neighbors/detail/cagra/topk_by_radix.cuh b/cpp/include/cuvs/neighbors/detail/cagra/topk_by_radix.cuh index 6a6a3cddf..67173026b 100644 --- a/cpp/include/cuvs/neighbors/detail/cagra/topk_by_radix.cuh +++ b/cpp/include/cuvs/neighbors/detail/cagra/topk_by_radix.cuh @@ -17,7 +17,7 @@ #include "topk_for_cagra/topk_core.cuh" -namespace raft::neighbors::cagra::detail { +namespace cuvs::neighbors::cagra::detail { namespace single_cta_search { template @@ -88,4 +88,4 @@ TOP_FUNC_PARTIAL_SPECIALIZATION(512); TOP_FUNC_PARTIAL_SPECIALIZATION(1024); } // namespace single_cta_search -} // namespace raft::neighbors::cagra::detail +} // namespace cuvs::neighbors::cagra::detail diff --git a/cpp/include/cuvs/neighbors/detail/cagra/topk_for_cagra/topk.h b/cpp/include/cuvs/neighbors/detail/cagra/topk_for_cagra/topk.h index 92b947404..41141ac27 100644 --- a/cpp/include/cuvs/neighbors/detail/cagra/topk_for_cagra/topk.h +++ b/cpp/include/cuvs/neighbors/detail/cagra/topk_for_cagra/topk.h @@ -18,7 +18,7 @@ #include #include -namespace raft::neighbors::cagra::detail { +namespace cuvs::neighbors::cagra::detail { // size_t _cuann_find_topk_bufferSize(uint32_t topK, @@ -55,4 +55,4 @@ CUDA_DEVICE_HOST_FUNC inline size_t _cuann_aligned(size_t size, size_t unit = 12 if (size % unit) { size += unit - (size % unit); } return size; } -} // namespace raft::neighbors::cagra::detail +} // namespace cuvs::neighbors::cagra::detail diff --git a/cpp/include/cuvs/neighbors/detail/cagra/topk_for_cagra/topk_core.cuh b/cpp/include/cuvs/neighbors/detail/cagra/topk_for_cagra/topk_core.cuh index 8e04a2eb0..a57fda93b 100644 --- a/cpp/include/cuvs/neighbors/detail/cagra/topk_for_cagra/topk_core.cuh +++ b/cpp/include/cuvs/neighbors/detail/cagra/topk_for_cagra/topk_core.cuh @@ -21,7 +21,7 @@ #include #include -namespace raft::neighbors::cagra::detail { +namespace cuvs::neighbors::cagra::detail { // __device__ inline uint32_t convert(uint32_t x) { @@ -1035,4 +1035,4 @@ inline void _cuann_find_topk(uint32_t topK, return; } -} // namespace raft::neighbors::cagra::detail +} // namespace cuvs::neighbors::cagra::detail diff --git a/cpp/include/cuvs/neighbors/detail/cagra/utils.hpp b/cpp/include/cuvs/neighbors/detail/cagra/utils.hpp index 5e57a9589..e26f69715 100644 --- a/cpp/include/cuvs/neighbors/detail/cagra/utils.hpp +++ b/cpp/include/cuvs/neighbors/detail/cagra/utils.hpp @@ -26,7 +26,7 @@ #include #include -namespace raft::neighbors::cagra::detail { +namespace cuvs::neighbors::cagra::detail { namespace utils { template inline cudaDataType_t get_cuda_data_type(); @@ -279,4 +279,4 @@ void copy_with_padding(raft::resources const& res, resource::get_cuda_stream(res))); } } -} // namespace raft::neighbors::cagra::detail +} // namespace cuvs::neighbors::cagra::detail diff --git a/cpp/include/cuvs/neighbors/detail/div_utils.hpp b/cpp/include/cuvs/neighbors/detail/div_utils.hpp index 0455d0ec9..fc6b6f5c5 100644 --- a/cpp/include/cuvs/neighbors/detail/div_utils.hpp +++ b/cpp/include/cuvs/neighbors/detail/div_utils.hpp @@ -27,7 +27,7 @@ * * @tparam Value_ a compile-time value representable as a power-of-two. */ -namespace raft::neighbors::detail { +namespace cuvs::neighbors::detail { template struct div_utils { typedef decltype(Value_) Type; @@ -63,4 +63,4 @@ struct div_utils { #endif } }; -} // namespace raft::neighbors::detail \ No newline at end of file +} // namespace cuvs::neighbors::detail \ No newline at end of file diff --git a/cpp/include/cuvs/neighbors/detail/faiss_select/Comparators.cuh b/cpp/include/cuvs/neighbors/detail/faiss_select/Comparators.cuh index 1a34d2f68..9ced61e13 100644 --- a/cpp/include/cuvs/neighbors/detail/faiss_select/Comparators.cuh +++ b/cpp/include/cuvs/neighbors/detail/faiss_select/Comparators.cuh @@ -10,7 +10,7 @@ #include #include -namespace raft::neighbors::detail::faiss_select { +namespace cuvs::neighbors::detail::faiss_select { template struct Comparator { @@ -26,4 +26,4 @@ struct Comparator { __device__ static inline bool gt(half a, half b) { return __hgt(a, b); } }; -} // namespace raft::neighbors::detail::faiss_select +} // namespace cuvs::neighbors::detail::faiss_select diff --git a/cpp/include/cuvs/neighbors/detail/faiss_select/DistanceUtils.h b/cpp/include/cuvs/neighbors/detail/faiss_select/DistanceUtils.h index cd4a52e5d..e8a41c1aa 100644 --- a/cpp/include/cuvs/neighbors/detail/faiss_select/DistanceUtils.h +++ b/cpp/include/cuvs/neighbors/detail/faiss_select/DistanceUtils.h @@ -7,7 +7,7 @@ #pragma once -namespace raft::neighbors::detail::faiss_select { +namespace cuvs::neighbors::detail::faiss_select { // If the inner size (dim) of the vectors is small, we want a larger query tile // size, like 1024 inline void chooseTileSize(size_t numQueries, @@ -49,4 +49,4 @@ inline void chooseTileSize(size_t numQueries, // tileCols is the remainder size tileCols = std::min(targetUsage / preferredTileRows, numCentroids); } -} // namespace raft::neighbors::detail::faiss_select +} // namespace cuvs::neighbors::detail::faiss_select diff --git a/cpp/include/cuvs/neighbors/detail/faiss_select/MergeNetworkBlock.cuh b/cpp/include/cuvs/neighbors/detail/faiss_select/MergeNetworkBlock.cuh index 1f4308fa2..9e53c729a 100644 --- a/cpp/include/cuvs/neighbors/detail/faiss_select/MergeNetworkBlock.cuh +++ b/cpp/include/cuvs/neighbors/detail/faiss_select/MergeNetworkBlock.cuh @@ -8,10 +8,10 @@ #pragma once #include -#include -#include +#include +#include -namespace raft::neighbors::detail::faiss_select { +namespace cuvs::neighbors::detail::faiss_select { // Merge pairs of lists smaller than blockDim.x (NumThreads) template ::merge(listK, listV); } -} // namespace raft::neighbors::detail::faiss_select +} // namespace cuvs::neighbors::detail::faiss_select diff --git a/cpp/include/cuvs/neighbors/detail/faiss_select/MergeNetworkUtils.cuh b/cpp/include/cuvs/neighbors/detail/faiss_select/MergeNetworkUtils.cuh index 78f794bff..7f7796fad 100644 --- a/cpp/include/cuvs/neighbors/detail/faiss_select/MergeNetworkUtils.cuh +++ b/cpp/include/cuvs/neighbors/detail/faiss_select/MergeNetworkUtils.cuh @@ -7,7 +7,7 @@ #pragma once -namespace raft::neighbors::detail::faiss_select { +namespace cuvs::neighbors::detail::faiss_select { template inline __device__ void swap(bool swap, T& x, T& y) @@ -22,4 +22,4 @@ inline __device__ void assign(bool assign, T& x, T y) { x = assign ? y : x; } -} // namespace raft::neighbors::detail::faiss_select +} // namespace cuvs::neighbors::detail::faiss_select diff --git a/cpp/include/cuvs/neighbors/detail/faiss_select/MergeNetworkWarp.cuh b/cpp/include/cuvs/neighbors/detail/faiss_select/MergeNetworkWarp.cuh index 3d87e284e..a06e282f7 100644 --- a/cpp/include/cuvs/neighbors/detail/faiss_select/MergeNetworkWarp.cuh +++ b/cpp/include/cuvs/neighbors/detail/faiss_select/MergeNetworkWarp.cuh @@ -7,12 +7,12 @@ #pragma once -#include -#include +#include +#include #include -namespace raft::neighbors::detail::faiss_select { +namespace cuvs::neighbors::detail::faiss_select { // // This file contains functions to: @@ -517,4 +517,4 @@ inline __device__ void warpSortAnyRegisters(K k[N], V v[N]) BitonicSortStep::sort(k, v); } -} // namespace raft::neighbors::detail::faiss_select +} // namespace cuvs::neighbors::detail::faiss_select diff --git a/cpp/include/cuvs/neighbors/detail/faiss_select/Select.cuh b/cpp/include/cuvs/neighbors/detail/faiss_select/Select.cuh index fdb986ac6..c92ecc77e 100644 --- a/cpp/include/cuvs/neighbors/detail/faiss_select/Select.cuh +++ b/cpp/include/cuvs/neighbors/detail/faiss_select/Select.cuh @@ -7,14 +7,14 @@ #pragma once -#include -#include -#include +#include +#include +#include #include #include -namespace raft::neighbors::detail::faiss_select { +namespace cuvs::neighbors::detail::faiss_select { // Specialization for block-wide monotonic merges producing a merge sort // since what we really want is a constexpr loop expansion @@ -551,4 +551,4 @@ struct WarpSelect { V threadV; }; -} // namespace raft::neighbors::detail::faiss_select +} // namespace cuvs::neighbors::detail::faiss_select diff --git a/cpp/include/cuvs/neighbors/detail/faiss_select/StaticUtils.h b/cpp/include/cuvs/neighbors/detail/faiss_select/StaticUtils.h index 5a25c7a32..6f53cf7f8 100644 --- a/cpp/include/cuvs/neighbors/detail/faiss_select/StaticUtils.h +++ b/cpp/include/cuvs/neighbors/detail/faiss_select/StaticUtils.h @@ -15,7 +15,7 @@ #define __device__ #endif -namespace raft::neighbors::detail::faiss_select::utils { +namespace cuvs::neighbors::detail::faiss_select::utils { template constexpr __host__ __device__ bool isPowerOf2(T v) @@ -45,4 +45,4 @@ static_assert(nextHighestPowerOf2(1536000000u) == 2147483648u, "nextHighestPower static_assert(nextHighestPowerOf2((size_t)2147483648ULL) == (size_t)4294967296ULL, "nextHighestPowerOf2"); -} // namespace raft::neighbors::detail::faiss_select::utils +} // namespace cuvs::neighbors::detail::faiss_select::utils diff --git a/cpp/include/cuvs/neighbors/detail/faiss_select/key_value_block_select.cuh b/cpp/include/cuvs/neighbors/detail/faiss_select/key_value_block_select.cuh index ff06b7dca..a9095751f 100644 --- a/cpp/include/cuvs/neighbors/detail/faiss_select/key_value_block_select.cuh +++ b/cpp/include/cuvs/neighbors/detail/faiss_select/key_value_block_select.cuh @@ -7,14 +7,14 @@ #pragma once -#include -#include +#include +#include // TODO: Need to think further about the impact (and new boundaries created) on the registers // because this will change the max k that can be processed. One solution might be to break // up k into multiple batches for larger k. -namespace raft::neighbors::detail::faiss_select { +namespace cuvs::neighbors::detail::faiss_select { // `Dir` true, produce largest values. // `Dir` false, produce smallest values. @@ -221,4 +221,4 @@ struct KeyValueBlockSelect { int kMinus1; }; -} // namespace raft::neighbors::detail::faiss_select +} // namespace cuvs::neighbors::detail::faiss_select diff --git a/cpp/include/cuvs/neighbors/detail/ivf_flat_build.cuh b/cpp/include/cuvs/neighbors/detail/ivf_flat_build.cuh index a9a6ac025..e9f7efb73 100644 --- a/cpp/include/cuvs/neighbors/detail/ivf_flat_build.cuh +++ b/cpp/include/cuvs/neighbors/detail/ivf_flat_build.cuh @@ -16,7 +16,11 @@ #pragma once -#include +#include +#include +#include +#include +#include #include #include #include @@ -26,10 +30,6 @@ #include #include #include -#include -#include -#include -#include #include #include #include @@ -38,7 +38,7 @@ #include -namespace raft::neighbors::ivf_flat::detail { +namespace cuvs::neighbors::ivf_flat::detail { using namespace raft::spatial::knn::detail; // NOLINT @@ -156,7 +156,7 @@ RAFT_KERNEL build_index_kernel(const LabelT* labels, } } -/** See raft::neighbors::ivf_flat::extend docs */ +/** See cuvs::neighbors::ivf_flat::extend docs */ template void extend(raft::resources const& handle, index* index, @@ -179,12 +179,12 @@ void extend(raft::resources const& handle, "You must pass data indices when the index is non-empty."); auto new_labels = raft::make_device_vector(handle, n_rows); - raft::cluster::kmeans_balanced_params kmeans_params; + cuvs::cluster::kmeans_balanced_params kmeans_params; kmeans_params.metric = index->metric(); auto new_vectors_view = raft::make_device_matrix_view(new_vectors, n_rows, dim); auto orig_centroids_view = raft::make_device_matrix_view(index->centers().data_handle(), n_lists, dim); - raft::cluster::kmeans_balanced::predict(handle, + cuvs::cluster::kmeans_balanced::predict(handle, kmeans_params, new_vectors_view, orig_centroids_view, @@ -203,7 +203,7 @@ void extend(raft::resources const& handle, raft::make_device_vector_view, IdxT>( list_sizes_ptr, n_lists); auto const_labels_view = make_const_mdspan(new_labels.view()); - raft::cluster::kmeans_balanced::helpers::calc_centers_and_sizes(handle, + cuvs::cluster::kmeans_balanced::helpers::calc_centers_and_sizes(handle, new_vectors_view, const_labels_view, centroids_view, @@ -283,7 +283,7 @@ void extend(raft::resources const& handle, } } -/** See raft::neighbors::ivf_flat::extend docs */ +/** See cuvs::neighbors::ivf_flat::extend docs */ template auto extend(raft::resources const& handle, const index& orig_index, @@ -296,7 +296,7 @@ auto extend(raft::resources const& handle, return ext_index; } -/** See raft::neighbors::ivf_flat::build docs */ +/** See cuvs::neighbors::ivf_flat::build docs */ template inline auto build(raft::resources const& handle, const index_params& params, @@ -336,10 +336,10 @@ inline auto build(raft::resources const& handle, raft::make_device_matrix_view(trainset.data(), n_rows_train, index.dim()); auto centers_view = raft::make_device_matrix_view( index.centers().data_handle(), index.n_lists(), index.dim()); - raft::cluster::kmeans_balanced_params kmeans_params; + cuvs::cluster::kmeans_balanced_params kmeans_params; kmeans_params.n_iters = params.kmeans_n_iters; kmeans_params.metric = index.metric(); - raft::cluster::kmeans_balanced::fit( + cuvs::cluster::kmeans_balanced::fit( handle, kmeans_params, trainset_const_view, centers_view, utils::mapping{}); } @@ -353,7 +353,7 @@ inline auto build(raft::resources const& handle, /** * Build an index that can be used in refinement operation. * - * See raft::neighbors::refine for details on the refinement operation. + * See cuvs::neighbors::refine for details on the refinement operation. * * The returned index cannot be used for a regular ivf_flat::search. The index misses information * about coarse clusters. Instead, the neighbor candidates are assumed to form clusters, one for @@ -490,4 +490,4 @@ void unpack_list_data( RAFT_CUDA_TRY(cudaPeekAtLastError()); } -} // namespace raft::neighbors::ivf_flat::detail +} // namespace cuvs::neighbors::ivf_flat::detail diff --git a/cpp/include/cuvs/neighbors/detail/ivf_flat_interleaved_scan-ext.cuh b/cpp/include/cuvs/neighbors/detail/ivf_flat_interleaved_scan-ext.cuh index 47f3e8888..cc32ff22a 100644 --- a/cpp/include/cuvs/neighbors/detail/ivf_flat_interleaved_scan-ext.cuh +++ b/cpp/include/cuvs/neighbors/detail/ivf_flat_interleaved_scan-ext.cuh @@ -17,22 +17,22 @@ #pragma once #include // uintX_t -#include // raft::neighbors::ivf_flat::index -#include // none_ivf_sample_filter +#include // cuvs::neighbors::ivf_flat::index +#include // none_ivf_sample_filter #include // RAFT_EXPLICIT #include // rmm:cuda_stream_view #ifdef RAFT_EXPLICIT_INSTANTIATE_ONLY -namespace raft::neighbors::ivf_flat::detail { +namespace cuvs::neighbors::ivf_flat::detail { template -void ivfflat_interleaved_scan(const raft::neighbors::ivf_flat::index& index, +void ivfflat_interleaved_scan(const cuvs::neighbors::ivf_flat::index& index, const T* queries, const uint32_t* coarse_query_results, const uint32_t n_queries, const uint32_t queries_offset, - const raft::distance::DistanceType metric, + const cuvs::distance::DistanceType metric, const uint32_t n_probes, const uint32_t k, const bool select_min, @@ -42,20 +42,20 @@ void ivfflat_interleaved_scan(const raft::neighbors::ivf_flat::index& i uint32_t& grid_dim_x, rmm::cuda_stream_view stream) RAFT_EXPLICIT; -} // namespace raft::neighbors::ivf_flat::detail +} // namespace cuvs::neighbors::ivf_flat::detail #endif // RAFT_EXPLICIT_INSTANTIATE_ONLY #define instantiate_raft_neighbors_ivf_flat_detail_ivfflat_interleaved_scan( \ T, AccT, IdxT, IvfSampleFilterT) \ extern template void \ - raft::neighbors::ivf_flat::detail::ivfflat_interleaved_scan( \ - const raft::neighbors::ivf_flat::index& index, \ + cuvs::neighbors::ivf_flat::detail::ivfflat_interleaved_scan( \ + const cuvs::neighbors::ivf_flat::index& index, \ const T* queries, \ const uint32_t* coarse_query_results, \ const uint32_t n_queries, \ const uint32_t queries_offset, \ - const raft::distance::DistanceType metric, \ + const cuvs::distance::DistanceType metric, \ const uint32_t n_probes, \ const uint32_t k, \ const bool select_min, \ @@ -66,10 +66,10 @@ void ivfflat_interleaved_scan(const raft::neighbors::ivf_flat::index& i rmm::cuda_stream_view stream) instantiate_raft_neighbors_ivf_flat_detail_ivfflat_interleaved_scan( - float, float, int64_t, raft::neighbors::filtering::none_ivf_sample_filter); + float, float, int64_t, cuvs::neighbors::filtering::none_ivf_sample_filter); instantiate_raft_neighbors_ivf_flat_detail_ivfflat_interleaved_scan( - int8_t, int32_t, int64_t, raft::neighbors::filtering::none_ivf_sample_filter); + int8_t, int32_t, int64_t, cuvs::neighbors::filtering::none_ivf_sample_filter); instantiate_raft_neighbors_ivf_flat_detail_ivfflat_interleaved_scan( - uint8_t, uint32_t, int64_t, raft::neighbors::filtering::none_ivf_sample_filter); + uint8_t, uint32_t, int64_t, cuvs::neighbors::filtering::none_ivf_sample_filter); #undef instantiate_raft_neighbors_ivf_flat_detail_ivfflat_interleaved_scan diff --git a/cpp/include/cuvs/neighbors/detail/ivf_flat_interleaved_scan-inl.cuh b/cpp/include/cuvs/neighbors/detail/ivf_flat_interleaved_scan-inl.cuh index f1f0ce10d..71b88177c 100644 --- a/cpp/include/cuvs/neighbors/detail/ivf_flat_interleaved_scan-inl.cuh +++ b/cpp/include/cuvs/neighbors/detail/ivf_flat_interleaved_scan-inl.cuh @@ -16,12 +16,12 @@ #pragma once +#include +#include +#include #include // RAFT_LOG_TRACE #include -#include #include -#include -#include #include #include // RAFT_CUDA_TRY #include @@ -30,7 +30,7 @@ #include #include -namespace raft::neighbors::ivf_flat::detail { +namespace cuvs::neighbors::ivf_flat::detail { using namespace raft::spatial::knn::detail; // NOLINT @@ -952,11 +952,11 @@ template -void launch_with_fixed_consts(raft::distance::DistanceType metric, Args&&... args) +void launch_with_fixed_consts(cuvs::distance::DistanceType metric, Args&&... args) { switch (metric) { - case raft::distance::DistanceType::L2Expanded: - case raft::distance::DistanceType::L2Unexpanded: + case cuvs::distance::DistanceType::L2Expanded: + case cuvs::distance::DistanceType::L2Unexpanded: return launch_kernel, raft::identity_op>({}, {}, std::forward(args)...); - case raft::distance::DistanceType::L2SqrtExpanded: - case raft::distance::DistanceType::L2SqrtUnexpanded: + case cuvs::distance::DistanceType::L2SqrtExpanded: + case cuvs::distance::DistanceType::L2SqrtUnexpanded: return launch_kernel, raft::sqrt_op>({}, {}, std::forward(args)...); - case raft::distance::DistanceType::InnerProduct: + case cuvs::distance::DistanceType::InnerProduct: return launch_kernel& index, const uint32_t* coarse_query_results, const uint32_t n_queries, const uint32_t queries_offset, - const raft::distance::DistanceType metric, + const cuvs::distance::DistanceType metric, const uint32_t n_probes, const uint32_t k, const bool select_min, @@ -1099,7 +1099,7 @@ void ivfflat_interleaved_scan(const index& index, { const int capacity = bound_by_power_of_two(k); - auto filter_adapter = raft::neighbors::filtering::ivf_to_sample_filter( + auto filter_adapter = cuvs::neighbors::filtering::ivf_to_sample_filter( index.inds_ptrs().data_handle(), sample_filter); select_interleaved_scan_kernel::run(capacity, index.veclen(), @@ -1119,4 +1119,4 @@ void ivfflat_interleaved_scan(const index& index, stream); } -} // namespace raft::neighbors::ivf_flat::detail +} // namespace cuvs::neighbors::ivf_flat::detail diff --git a/cpp/include/cuvs/neighbors/detail/ivf_flat_search-ext.cuh b/cpp/include/cuvs/neighbors/detail/ivf_flat_search-ext.cuh index 976d15a61..3a8776f7c 100644 --- a/cpp/include/cuvs/neighbors/detail/ivf_flat_search-ext.cuh +++ b/cpp/include/cuvs/neighbors/detail/ivf_flat_search-ext.cuh @@ -17,18 +17,18 @@ #pragma once #include // uintX_t -#include // raft::neighbors::ivf_flat::index -#include // none_ivf_sample_filter +#include // cuvs::neighbors::ivf_flat::index +#include // none_ivf_sample_filter #include // RAFT_EXPLICIT #ifdef RAFT_EXPLICIT_INSTANTIATE_ONLY -namespace raft::neighbors::ivf_flat::detail { +namespace cuvs::neighbors::ivf_flat::detail { template void search(raft::resources const& handle, const search_params& params, - const raft::neighbors::ivf_flat::index& index, + const cuvs::neighbors::ivf_flat::index& index, const T* queries, uint32_t n_queries, uint32_t k, @@ -37,15 +37,15 @@ void search(raft::resources const& handle, rmm::mr::device_memory_resource* mr = nullptr, IvfSampleFilterT sample_filter = IvfSampleFilterT()) RAFT_EXPLICIT; -} // namespace raft::neighbors::ivf_flat::detail +} // namespace cuvs::neighbors::ivf_flat::detail #endif // RAFT_EXPLICIT_INSTANTIATE_ONLY #define instantiate_raft_neighbors_ivf_flat_detail_search(T, IdxT, IvfSampleFilterT) \ - extern template void raft::neighbors::ivf_flat::detail::search( \ + extern template void cuvs::neighbors::ivf_flat::detail::search( \ raft::resources const& handle, \ const search_params& params, \ - const raft::neighbors::ivf_flat::index& index, \ + const cuvs::neighbors::ivf_flat::index& index, \ const T* queries, \ uint32_t n_queries, \ uint32_t k, \ @@ -55,10 +55,10 @@ void search(raft::resources const& handle, IvfSampleFilterT sample_filter) instantiate_raft_neighbors_ivf_flat_detail_search( - float, int64_t, raft::neighbors::filtering::none_ivf_sample_filter); + float, int64_t, cuvs::neighbors::filtering::none_ivf_sample_filter); instantiate_raft_neighbors_ivf_flat_detail_search( - int8_t, int64_t, raft::neighbors::filtering::none_ivf_sample_filter); + int8_t, int64_t, cuvs::neighbors::filtering::none_ivf_sample_filter); instantiate_raft_neighbors_ivf_flat_detail_search( - uint8_t, int64_t, raft::neighbors::filtering::none_ivf_sample_filter); + uint8_t, int64_t, cuvs::neighbors::filtering::none_ivf_sample_filter); #undef instantiate_raft_neighbors_ivf_flat_detail_search diff --git a/cpp/include/cuvs/neighbors/detail/ivf_flat_search-inl.cuh b/cpp/include/cuvs/neighbors/detail/ivf_flat_search-inl.cuh index c0f856103..a45740bde 100644 --- a/cpp/include/cuvs/neighbors/detail/ivf_flat_search-inl.cuh +++ b/cpp/include/cuvs/neighbors/detail/ivf_flat_search-inl.cuh @@ -16,27 +16,27 @@ #pragma once -#include // RAFT_LOG_TRACE +#include // is_min_close, DistanceType +#include // interleaved_scan +#include // cuvs::neighbors::ivf_flat::index +#include // none_ivf_sample_filter +#include // RAFT_LOG_TRACE #include -#include // raft::resources -#include // is_min_close, DistanceType -#include // raft::linalg::gemm -#include // raft::linalg::norm -#include // raft::linalg::unary_op -#include // matrix::detail::select_k -#include // interleaved_scan -#include // raft::neighbors::ivf_flat::index -#include // none_ivf_sample_filter -#include // utils::mapping -#include // rmm::device_memory_resource +#include // raft::resources +#include // raft::linalg::gemm +#include // raft::linalg::norm +#include // raft::linalg::unary_op +#include // matrix::detail::select_k +#include // utils::mapping +#include // rmm::device_memory_resource -namespace raft::neighbors::ivf_flat::detail { +namespace cuvs::neighbors::ivf_flat::detail { using namespace raft::spatial::knn::detail; // NOLINT template void search_impl(raft::resources const& handle, - const raft::neighbors::ivf_flat::index& index, + const cuvs::neighbors::ivf_flat::index& index, const T* queries, uint32_t n_queries, uint32_t queries_offset, @@ -83,8 +83,8 @@ void search_impl(raft::resources const& handle, // todo(lsugy): raft distance? (if performance is similar/better than gemm) switch (index.metric()) { - case raft::distance::DistanceType::L2Expanded: - case raft::distance::DistanceType::L2SqrtExpanded: { + case cuvs::distance::DistanceType::L2Expanded: + case cuvs::distance::DistanceType::L2SqrtExpanded: { alpha = -2.0f; beta = 1.0f; raft::linalg::rowNorm(query_norm_dev.data(), @@ -204,10 +204,10 @@ void search_impl(raft::resources const& handle, } } -/** See raft::neighbors::ivf_flat::search docs */ +/** See cuvs::neighbors::ivf_flat::search docs */ template + typename IvfSampleFilterT = cuvs::neighbors::filtering::none_ivf_sample_filter> inline void search(raft::resources const& handle, const search_params& params, const index& index, @@ -249,7 +249,7 @@ inline void search(raft::resources const& handle, offset_q, k, n_probes, - raft::distance::is_min_close(index.metric()), + cuvs::distance::is_min_close(index.metric()), neighbors + offset_q * k, distances + offset_q * k, mr, @@ -257,4 +257,4 @@ inline void search(raft::resources const& handle, } } -} // namespace raft::neighbors::ivf_flat::detail +} // namespace cuvs::neighbors::ivf_flat::detail diff --git a/cpp/include/cuvs/neighbors/detail/ivf_flat_serialize.cuh b/cpp/include/cuvs/neighbors/detail/ivf_flat_serialize.cuh index aaf48ae83..478a8d420 100644 --- a/cpp/include/cuvs/neighbors/detail/ivf_flat_serialize.cuh +++ b/cpp/include/cuvs/neighbors/detail/ivf_flat_serialize.cuh @@ -16,18 +16,18 @@ #pragma once +#include +#include +#include #include #include #include #include -#include -#include -#include #include #include -namespace raft::neighbors::ivf_flat::detail { +namespace cuvs::neighbors::ivf_flat::detail { // Serialization version // No backward compatibility yet; that is, can't add additional fields without breaking @@ -127,7 +127,7 @@ auto deserialize(raft::resources const& handle, std::istream& is) -> index(handle, is); auto dim = deserialize_scalar(handle, is); auto n_lists = deserialize_scalar(handle, is); - auto metric = deserialize_scalar(handle, is); + auto metric = deserialize_scalar(handle, is); bool adaptive_centers = deserialize_scalar(handle, is); bool cma = deserialize_scalar(handle, is); @@ -171,4 +171,4 @@ auto deserialize(raft::resources const& handle, const std::string& filename) -> return index; } -} // namespace raft::neighbors::ivf_flat::detail +} // namespace cuvs::neighbors::ivf_flat::detail diff --git a/cpp/include/cuvs/neighbors/detail/ivf_pq_build.cuh b/cpp/include/cuvs/neighbors/detail/ivf_pq_build.cuh index e57133fc2..843d880b4 100644 --- a/cpp/include/cuvs/neighbors/detail/ivf_pq_build.cuh +++ b/cpp/include/cuvs/neighbors/detail/ivf_pq_build.cuh @@ -20,18 +20,18 @@ #include #include -#include -#include -#include +#include +#include +#include -#include +#include +#include #include #include #include #include #include #include -#include #include #include #include @@ -60,7 +60,7 @@ #include #include -namespace raft::neighbors::ivf_pq::detail { +namespace cuvs::neighbors::ivf_pq::detail { using namespace raft::spatial::knn::detail; // NOLINT @@ -454,10 +454,10 @@ void train_per_subset(raft::resources const& handle, auto sub_labels_view = raft::make_device_vector_view(sub_labels.data(), n_rows); auto cluster_sizes_view = raft::make_device_vector_view(pq_cluster_sizes.data(), index.pq_book_size()); - raft::cluster::kmeans_balanced_params kmeans_params; + cuvs::cluster::kmeans_balanced_params kmeans_params; kmeans_params.n_iters = kmeans_n_iters; - kmeans_params.metric = raft::distance::DistanceType::L2Expanded; - raft::cluster::kmeans_balanced::helpers::build_clusters(handle, + kmeans_params.metric = cuvs::distance::DistanceType::L2Expanded; + cuvs::cluster::kmeans_balanced::helpers::build_clusters(handle, kmeans_params, sub_trainset_view, centers_tmp_view, @@ -539,10 +539,10 @@ void train_per_cluster(raft::resources const& handle, raft::make_device_vector_view(pq_labels.data(), pq_n_rows); auto pq_cluster_sizes_view = raft::make_device_vector_view(pq_cluster_sizes.data(), index.pq_book_size()); - raft::cluster::kmeans_balanced_params kmeans_params; + cuvs::cluster::kmeans_balanced_params kmeans_params; kmeans_params.n_iters = kmeans_n_iters; - kmeans_params.metric = raft::distance::DistanceType::L2Expanded; - raft::cluster::kmeans_balanced::helpers::build_clusters(handle, + kmeans_params.metric = cuvs::distance::DistanceType::L2Expanded; + cuvs::cluster::kmeans_balanced::helpers::build_clusters(handle, kmeans_params, rot_vectors_view, centers_tmp_view, @@ -1669,9 +1669,9 @@ void extend(raft::resources const& handle, new_data_labels.data() + batch.offset(), batch.size()); auto centers_view = raft::make_device_matrix_view( cluster_centers.data(), n_clusters, index->dim()); - raft::cluster::kmeans_balanced_params kmeans_params; + cuvs::cluster::kmeans_balanced_params kmeans_params; kmeans_params.metric = index->metric(); - raft::cluster::kmeans_balanced::predict(handle, + cuvs::cluster::kmeans_balanced::predict(handle, kmeans_params, batch_data_view, centers_view, @@ -1858,10 +1858,10 @@ auto build(raft::resources const& handle, raft::make_device_matrix_view(trainset.data(), n_rows_train, index.dim()); auto centers_view = raft::make_device_matrix_view(cluster_centers, index.n_lists(), index.dim()); - raft::cluster::kmeans_balanced_params kmeans_params; + cuvs::cluster::kmeans_balanced_params kmeans_params; kmeans_params.n_iters = params.kmeans_n_iters; kmeans_params.metric = index.metric(); - raft::cluster::kmeans_balanced::fit( + cuvs::cluster::kmeans_balanced::fit( handle, kmeans_params, trainset_const_view, centers_view, utils::mapping{}); // Trainset labels are needed for training PQ codebooks @@ -1869,7 +1869,7 @@ auto build(raft::resources const& handle, auto centers_const_view = raft::make_device_matrix_view( cluster_centers, index.n_lists(), index.dim()); auto labels_view = raft::make_device_vector_view(labels.data(), n_rows_train); - raft::cluster::kmeans_balanced::predict(handle, + cuvs::cluster::kmeans_balanced::predict(handle, kmeans_params, trainset_const_view, centers_const_view, @@ -1915,4 +1915,4 @@ auto build(raft::resources const& handle, } return index; } -} // namespace raft::neighbors::ivf_pq::detail +} // namespace cuvs::neighbors::ivf_pq::detail diff --git a/cpp/include/cuvs/neighbors/detail/ivf_pq_codepacking.cuh b/cpp/include/cuvs/neighbors/detail/ivf_pq_codepacking.cuh index 52969dd17..93a3459a9 100644 --- a/cpp/include/cuvs/neighbors/detail/ivf_pq_codepacking.cuh +++ b/cpp/include/cuvs/neighbors/detail/ivf_pq_codepacking.cuh @@ -16,8 +16,8 @@ #pragma once -#include -#include +#include +#include #include #include @@ -28,7 +28,7 @@ #include -namespace raft::neighbors::ivf_pq::detail { +namespace cuvs::neighbors::ivf_pq::detail { /** A chunk of PQ-encoded vector managed by one CUDA thread. */ using pq_vec_t = TxN_t::io_t; @@ -211,4 +211,4 @@ __device__ void write_list( } } -} // namespace raft::neighbors::ivf_pq::detail +} // namespace cuvs::neighbors::ivf_pq::detail diff --git a/cpp/include/cuvs/neighbors/detail/ivf_pq_compute_similarity-ext.cuh b/cpp/include/cuvs/neighbors/detail/ivf_pq_compute_similarity-ext.cuh index 37b6efc1e..26fd7e493 100644 --- a/cpp/include/cuvs/neighbors/detail/ivf_pq_compute_similarity-ext.cuh +++ b/cpp/include/cuvs/neighbors/detail/ivf_pq_compute_similarity-ext.cuh @@ -17,17 +17,17 @@ #pragma once #include // __half +#include // cuvs::distance::DistanceType +#include // cuvs::neighbors::ivf_pq::detail::fp_8bit +#include // cuvs::neighbors::ivf_pq::codebook_gen +#include // none_ivf_sample_filter #include // RAFT_WEAK_FUNCTION -#include // raft::distance::DistanceType -#include // raft::neighbors::ivf_pq::detail::fp_8bit -#include // raft::neighbors::ivf_pq::codebook_gen -#include // none_ivf_sample_filter #include // RAFT_EXPLICIT #include // rmm::cuda_stream_view #ifdef RAFT_EXPLICIT_INSTANTIATE_ONLY -namespace raft::neighbors::ivf_pq::detail { +namespace cuvs::neighbors::ivf_pq::detail { // is_local_topk_feasible is not inline here, because we would have to define it // here as well. That would run the risk of the definitions here and in the @@ -130,14 +130,14 @@ auto compute_similarity_select(const cudaDeviceProp& dev_props, uint32_t topk) -> selected RAFT_EXPLICIT; -} // namespace raft::neighbors::ivf_pq::detail +} // namespace cuvs::neighbors::ivf_pq::detail #endif // RAFT_EXPLICIT_INSTANTIATE_ONLY #define instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( \ OutT, LutT, IvfSampleFilterT) \ extern template auto \ - raft::neighbors::ivf_pq::detail::compute_similarity_select( \ + cuvs::neighbors::ivf_pq::detail::compute_similarity_select( \ const cudaDeviceProp& dev_props, \ bool manage_local_topk, \ int locality_hint, \ @@ -148,19 +148,19 @@ auto compute_similarity_select(const cudaDeviceProp& dev_props, uint32_t n_queries, \ uint32_t n_probes, \ uint32_t topk) \ - ->raft::neighbors::ivf_pq::detail::selected; \ + ->cuvs::neighbors::ivf_pq::detail::selected; \ \ extern template void \ - raft::neighbors::ivf_pq::detail::compute_similarity_run( \ - raft::neighbors::ivf_pq::detail::selected s, \ + cuvs::neighbors::ivf_pq::detail::compute_similarity_run( \ + cuvs::neighbors::ivf_pq::detail::selected s, \ rmm::cuda_stream_view stream, \ uint32_t dim, \ uint32_t n_probes, \ uint32_t pq_dim, \ uint32_t n_queries, \ uint32_t queries_offset, \ - raft::distance::DistanceType metric, \ - raft::neighbors::ivf_pq::codebook_gen codebook_kind, \ + cuvs::distance::DistanceType metric, \ + cuvs::neighbors::ivf_pq::codebook_gen codebook_kind, \ uint32_t topk, \ uint32_t max_samples, \ const float* cluster_centers, \ @@ -179,39 +179,39 @@ auto compute_similarity_select(const cudaDeviceProp& dev_props, #define COMMA , instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( half, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA false>, - raft::neighbors::filtering::ivf_to_sample_filter< - int64_t COMMA raft::neighbors::filtering::none_ivf_sample_filter>); + cuvs::neighbors::ivf_pq::detail::fp_8bit<5u COMMA false>, + cuvs::neighbors::filtering::ivf_to_sample_filter< + int64_t COMMA cuvs::neighbors::filtering::none_ivf_sample_filter>); instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( half, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA true>, - raft::neighbors::filtering::ivf_to_sample_filter< - int64_t COMMA raft::neighbors::filtering::none_ivf_sample_filter>); + cuvs::neighbors::ivf_pq::detail::fp_8bit<5u COMMA true>, + cuvs::neighbors::filtering::ivf_to_sample_filter< + int64_t COMMA cuvs::neighbors::filtering::none_ivf_sample_filter>); instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( half, half, - raft::neighbors::filtering::ivf_to_sample_filter< - int64_t COMMA raft::neighbors::filtering::none_ivf_sample_filter>); + cuvs::neighbors::filtering::ivf_to_sample_filter< + int64_t COMMA cuvs::neighbors::filtering::none_ivf_sample_filter>); instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( float, half, - raft::neighbors::filtering::ivf_to_sample_filter< - int64_t COMMA raft::neighbors::filtering::none_ivf_sample_filter>); + cuvs::neighbors::filtering::ivf_to_sample_filter< + int64_t COMMA cuvs::neighbors::filtering::none_ivf_sample_filter>); instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( float, float, - raft::neighbors::filtering::ivf_to_sample_filter< - int64_t COMMA raft::neighbors::filtering::none_ivf_sample_filter>); + cuvs::neighbors::filtering::ivf_to_sample_filter< + int64_t COMMA cuvs::neighbors::filtering::none_ivf_sample_filter>); instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( float, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA false>, - raft::neighbors::filtering::ivf_to_sample_filter< - int64_t COMMA raft::neighbors::filtering::none_ivf_sample_filter>); + cuvs::neighbors::ivf_pq::detail::fp_8bit<5u COMMA false>, + cuvs::neighbors::filtering::ivf_to_sample_filter< + int64_t COMMA cuvs::neighbors::filtering::none_ivf_sample_filter>); instantiate_raft_neighbors_ivf_pq_detail_compute_similarity_select( float, - raft::neighbors::ivf_pq::detail::fp_8bit<5u COMMA true>, - raft::neighbors::filtering::ivf_to_sample_filter< - int64_t COMMA raft::neighbors::filtering::none_ivf_sample_filter>); + cuvs::neighbors::ivf_pq::detail::fp_8bit<5u COMMA true>, + cuvs::neighbors::filtering::ivf_to_sample_filter< + int64_t COMMA cuvs::neighbors::filtering::none_ivf_sample_filter>); #undef COMMA diff --git a/cpp/include/cuvs/neighbors/detail/ivf_pq_compute_similarity-inl.cuh b/cpp/include/cuvs/neighbors/detail/ivf_pq_compute_similarity-inl.cuh index bd88c029e..7526a8052 100644 --- a/cpp/include/cuvs/neighbors/detail/ivf_pq_compute_similarity-inl.cuh +++ b/cpp/include/cuvs/neighbors/detail/ivf_pq_compute_similarity-inl.cuh @@ -16,18 +16,18 @@ #pragma once -#include // raft::distance::DistanceType +#include // cuvs::distance::DistanceType +#include // dummy_block_sort_t +#include // codebook_gen +#include // none_ivf_sample_filter #include // matrix::detail::select::warpsort::warp_sort_distributed -#include // dummy_block_sort_t -#include // codebook_gen -#include // none_ivf_sample_filter -#include // RAFT_CUDA_TRY -#include // raft::atomicMin -#include // raft::Pow2 -#include // raft::TxN_t -#include // rmm::cuda_stream_view +#include // RAFT_CUDA_TRY +#include // raft::atomicMin +#include // raft::Pow2 +#include // raft::TxN_t +#include // rmm::cuda_stream_view -namespace raft::neighbors::ivf_pq::detail { +namespace cuvs::neighbors::ivf_pq::detail { /** * Maximum value of k for the fused calculate & select in ivfpq. @@ -512,7 +512,7 @@ RAFT_KERNEL compute_similarity_kernel(uint32_t dim, // The signature of the kernel defined by a minimal set of template parameters template + typename IvfSampleFilterT = cuvs::neighbors::filtering::none_ivf_sample_filter> using compute_similarity_kernel_t = decltype(&compute_similarity_kernel); @@ -521,7 +521,7 @@ template + typename IvfSampleFilterT = cuvs::neighbors::filtering::none_ivf_sample_filter> struct compute_similarity_kernel_config { public: static auto get(uint32_t pq_bits, uint32_t k_max) @@ -571,7 +571,7 @@ template + typename IvfSampleFilterT = cuvs::neighbors::filtering::none_ivf_sample_filter> auto get_compute_similarity_kernel(uint32_t pq_bits, uint32_t k_max) -> compute_similarity_kernel_t { @@ -616,7 +616,7 @@ struct selected { template + typename IvfSampleFilterT = cuvs::neighbors::filtering::none_ivf_sample_filter> void compute_similarity_run(selected s, rmm::cuda_stream_view stream, uint32_t dim, @@ -681,7 +681,7 @@ void compute_similarity_run(selected s, */ template + typename IvfSampleFilterT = cuvs::neighbors::filtering::none_ivf_sample_filter> auto compute_similarity_select(const cudaDeviceProp& dev_props, bool manage_local_topk, int locality_hint, @@ -938,4 +938,4 @@ auto compute_similarity_select(const cudaDeviceProp& dev_props, return selected_config; } -} // namespace raft::neighbors::ivf_pq::detail +} // namespace cuvs::neighbors::ivf_pq::detail diff --git a/cpp/include/cuvs/neighbors/detail/ivf_pq_dummy_block_sort.cuh b/cpp/include/cuvs/neighbors/detail/ivf_pq_dummy_block_sort.cuh index a00b6a50f..34273af26 100644 --- a/cpp/include/cuvs/neighbors/detail/ivf_pq_dummy_block_sort.cuh +++ b/cpp/include/cuvs/neighbors/detail/ivf_pq_dummy_block_sort.cuh @@ -27,7 +27,7 @@ * */ -namespace raft::neighbors::ivf_pq::detail { +namespace cuvs::neighbors::ivf_pq::detail { template struct dummy_block_sort_t { @@ -36,4 +36,4 @@ struct dummy_block_sort_t { __device__ dummy_block_sort_t(int k, Args...){}; }; -} // namespace raft::neighbors::ivf_pq::detail +} // namespace cuvs::neighbors::ivf_pq::detail diff --git a/cpp/include/cuvs/neighbors/detail/ivf_pq_fp_8bit.cuh b/cpp/include/cuvs/neighbors/detail/ivf_pq_fp_8bit.cuh index 68c8a513f..f95926c09 100644 --- a/cpp/include/cuvs/neighbors/detail/ivf_pq_fp_8bit.cuh +++ b/cpp/include/cuvs/neighbors/detail/ivf_pq_fp_8bit.cuh @@ -18,15 +18,15 @@ #include -#include +#include +#include #include #include #include #include #include #include -#include #include #include #include @@ -47,7 +47,7 @@ #include -namespace raft::neighbors::ivf_pq::detail { +namespace cuvs::neighbors::ivf_pq::detail { /** 8-bit floating-point storage type. * @@ -125,4 +125,4 @@ struct fp_8bit { } }; -} // namespace raft::neighbors::ivf_pq::detail +} // namespace cuvs::neighbors::ivf_pq::detail diff --git a/cpp/include/cuvs/neighbors/detail/ivf_pq_search.cuh b/cpp/include/cuvs/neighbors/detail/ivf_pq_search.cuh index 7f5b316d4..877605777 100644 --- a/cpp/include/cuvs/neighbors/detail/ivf_pq_search.cuh +++ b/cpp/include/cuvs/neighbors/detail/ivf_pq_search.cuh @@ -20,12 +20,13 @@ #include #include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include #include #include #include @@ -34,7 +35,6 @@ #include #include #include -#include #include #include #include @@ -55,7 +55,7 @@ #include -namespace raft::neighbors::ivf_pq::detail { +namespace cuvs::neighbors::ivf_pq::detail { using namespace raft::spatial::knn::detail; // NOLINT @@ -75,7 +75,7 @@ void select_clusters(raft::resources const& handle, uint32_t n_lists, uint32_t dim, uint32_t dim_ext, - raft::distance::DistanceType metric, + cuvs::distance::DistanceType metric, const T* queries, // [n_queries, dim] const float* cluster_centers, // [n_lists, dim_ext] rmm::mr::device_memory_resource* mr) @@ -103,9 +103,9 @@ void select_clusters(raft::resources const& handle, */ float norm_factor; switch (metric) { - case raft::distance::DistanceType::L2SqrtExpanded: - case raft::distance::DistanceType::L2Expanded: norm_factor = 1.0 / -2.0; break; - case raft::distance::DistanceType::InnerProduct: norm_factor = 0.0; break; + case cuvs::distance::DistanceType::L2SqrtExpanded: + case cuvs::distance::DistanceType::L2Expanded: norm_factor = 1.0 / -2.0; break; + case cuvs::distance::DistanceType::InnerProduct: norm_factor = 0.0; break; default: RAFT_FAIL("Unsupported distance type %d.", int(metric)); } auto float_queries_view = @@ -121,14 +121,14 @@ void select_clusters(raft::resources const& handle, float beta; uint32_t gemm_k = dim; switch (metric) { - case raft::distance::DistanceType::L2SqrtExpanded: - case raft::distance::DistanceType::L2Expanded: { + case cuvs::distance::DistanceType::L2SqrtExpanded: + case cuvs::distance::DistanceType::L2Expanded: { alpha = -2.0; beta = 0.0; gemm_k = dim + 1; RAFT_EXPECTS(gemm_k <= dim_ext, "unexpected gemm_k or dim_ext"); } break; - case raft::distance::DistanceType::InnerProduct: { + case cuvs::distance::DistanceType::InnerProduct: { alpha = -1.0; beta = 0.0; } break; @@ -647,7 +647,7 @@ struct ivfpq_search { { bool signed_metric = false; switch (metric) { - case raft::distance::DistanceType::InnerProduct: signed_metric = true; break; + case cuvs::distance::DistanceType::InnerProduct: signed_metric = true; break; default: break; } @@ -728,7 +728,7 @@ inline auto get_max_batch_size(raft::resources const& res, /** See raft::spatial::knn::ivf_pq::search docs */ template + typename IvfSampleFilterT = cuvs::neighbors::filtering::none_ivf_sample_filter> inline void search(raft::resources const& handle, const search_params& params, const index& index, @@ -794,7 +794,7 @@ inline void search(raft::resources const& handle, rmm::device_uvector rot_queries(max_queries * index.rot_dim(), stream, mr); rmm::device_uvector clusters_to_probe(max_queries * n_probes, stream, mr); - auto filter_adapter = raft::neighbors::filtering::ivf_to_sample_filter( + auto filter_adapter = cuvs::neighbors::filtering::ivf_to_sample_filter( index.inds_ptrs().data_handle(), sample_filter); auto search_instance = ivfpq_search::fun(params, index.metric()); @@ -857,4 +857,4 @@ inline void search(raft::resources const& handle, } } -} // namespace raft::neighbors::ivf_pq::detail +} // namespace cuvs::neighbors::ivf_pq::detail diff --git a/cpp/include/cuvs/neighbors/detail/ivf_pq_serialize.cuh b/cpp/include/cuvs/neighbors/detail/ivf_pq_serialize.cuh index 038bf8d7c..684125d6f 100644 --- a/cpp/include/cuvs/neighbors/detail/ivf_pq_serialize.cuh +++ b/cpp/include/cuvs/neighbors/detail/ivf_pq_serialize.cuh @@ -16,10 +16,10 @@ #pragma once +#include +#include +#include #include -#include -#include -#include #include #include @@ -29,7 +29,7 @@ #include #include -namespace raft::neighbors::ivf_pq::detail { +namespace cuvs::neighbors::ivf_pq::detail { // Serialization version // No backward compatibility yet; that is, can't add additional fields without breaking @@ -133,8 +133,8 @@ auto deserialize(raft::resources const& handle_, std::istream& is) -> index(handle_, is); auto cma = deserialize_scalar(handle_, is); - auto metric = deserialize_scalar(handle_, is); - auto codebook_kind = deserialize_scalar(handle_, is); + auto metric = deserialize_scalar(handle_, is); + auto codebook_kind = deserialize_scalar(handle_, is); auto n_lists = deserialize_scalar(handle_, is); RAFT_LOG_DEBUG("n_rows %zu, dim %d, pq_dim %d, pq_bits %d, n_lists %d", @@ -144,7 +144,7 @@ auto deserialize(raft::resources const& handle_, std::istream& is) -> index(pq_bits), static_cast(n_lists)); - auto index = raft::neighbors::ivf_pq::index( + auto index = cuvs::neighbors::ivf_pq::index( handle_, metric, codebook_kind, n_lists, dim, pq_bits, pq_dim, cma); deserialize_mdspan(handle_, is, index.pq_centers()); @@ -188,4 +188,4 @@ auto deserialize(raft::resources const& handle_, const std::string& filename) -> return index; } -} // namespace raft::neighbors::ivf_pq::detail +} // namespace cuvs::neighbors::ivf_pq::detail diff --git a/cpp/include/cuvs/neighbors/detail/knn_brute_force.cuh b/cpp/include/cuvs/neighbors/detail/knn_brute_force.cuh index 27ef00e38..9002b3104 100644 --- a/cpp/include/cuvs/neighbors/detail/knn_brute_force.cuh +++ b/cpp/include/cuvs/neighbors/detail/knn_brute_force.cuh @@ -27,25 +27,25 @@ #include #include +#include +#include +#include +#include +#include +#include #include #include -#include -#include -#include #include #include #include #include -#include -#include -#include #include #include #include #include #include -namespace raft::neighbors::detail { +namespace cuvs::neighbors::detail { using namespace raft::spatial::knn::detail; using namespace raft::spatial::knn; @@ -65,7 +65,7 @@ void tiled_brute_force_knn(const raft::resources& handle, size_t k, ElementType* distances, // size (m, k) IndexType* indices, // size (m, k) - raft::distance::DistanceType metric, + cuvs::distance::DistanceType metric, float metric_arg = 2.0, size_t max_row_tile_size = 0, size_t max_col_tile_size = 0, @@ -97,13 +97,13 @@ void tiled_brute_force_knn(const raft::resources& handle, auto pairwise_metric = metric; rmm::device_uvector search_norms(0, stream); rmm::device_uvector index_norms(0, stream); - if (metric == raft::distance::DistanceType::L2Expanded || - metric == raft::distance::DistanceType::L2SqrtExpanded || - metric == raft::distance::DistanceType::CosineExpanded) { + if (metric == cuvs::distance::DistanceType::L2Expanded || + metric == cuvs::distance::DistanceType::L2SqrtExpanded || + metric == cuvs::distance::DistanceType::CosineExpanded) { if (!precomputed_search_norms) { search_norms.resize(m, stream); } if (!precomputed_index_norms) { index_norms.resize(n, stream); } // cosine needs the l2norm, where as l2 distances needs the squared norm - if (metric == raft::distance::DistanceType::CosineExpanded) { + if (metric == cuvs::distance::DistanceType::CosineExpanded) { if (!precomputed_search_norms) { raft::linalg::rowNorm(search_norms.data(), search, @@ -134,7 +134,7 @@ void tiled_brute_force_knn(const raft::resources& handle, index_norms.data(), index, d, n, raft::linalg::NormType::L2Norm, true, stream); } } - pairwise_metric = raft::distance::DistanceType::InnerProduct; + pairwise_metric = cuvs::distance::DistanceType::InnerProduct; } // if we're tiling over columns, we need additional buffers for temporary output @@ -163,7 +163,7 @@ void tiled_brute_force_knn(const raft::resources& handle, rmm::device_uvector temp_out_distances(tile_rows * temp_out_cols, stream); rmm::device_uvector temp_out_indices(tile_rows * temp_out_cols, stream); - bool select_min = raft::distance::is_min_close(metric); + bool select_min = cuvs::distance::is_min_close(metric); for (size_t i = 0; i < m; i += tile_rows) { size_t current_query_size = std::min(tile_rows, m - i); @@ -187,12 +187,12 @@ void tiled_brute_force_knn(const raft::resources& handle, pairwise_metric, true, metric_arg); - if (metric == raft::distance::DistanceType::L2Expanded || - metric == raft::distance::DistanceType::L2SqrtExpanded) { + if (metric == cuvs::distance::DistanceType::L2Expanded || + metric == cuvs::distance::DistanceType::L2SqrtExpanded) { auto row_norms = precomputed_search_norms ? precomputed_search_norms : search_norms.data(); auto col_norms = precomputed_index_norms ? precomputed_index_norms : index_norms.data(); auto dist = temp_distances.data(); - bool sqrt = metric == raft::distance::DistanceType::L2SqrtExpanded; + bool sqrt = metric == cuvs::distance::DistanceType::L2SqrtExpanded; raft::linalg::map_offset( handle, @@ -201,11 +201,11 @@ void tiled_brute_force_knn(const raft::resources& handle, IndexType row = i + (idx / current_centroid_size); IndexType col = j + (idx % current_centroid_size); - raft::distance::detail::ops::l2_exp_cutlass_op l2_op(sqrt); + cuvs::distance::detail::ops::l2_exp_cutlass_op l2_op(sqrt); auto val = l2_op(row_norms[row], col_norms[col], dist[idx]); return distance_epilogue(val, row, col); }); - } else if (metric == raft::distance::DistanceType::CosineExpanded) { + } else if (metric == cuvs::distance::DistanceType::CosineExpanded) { auto row_norms = precomputed_search_norms ? precomputed_search_norms : search_norms.data(); auto col_norms = precomputed_index_norms ? precomputed_index_norms : index_norms.data(); auto dist = temp_distances.data(); @@ -315,7 +315,7 @@ void tiled_brute_force_knn(const raft::resources& handle, * @param[in] rowMajorQuery are the query array in row-major layout? * @param[in] translations translation ids for indices when index rows represent * non-contiguous partitions - * @param[in] metric corresponds to the raft::distance::DistanceType enum (default is L2Expanded) + * @param[in] metric corresponds to the cuvs::distance::DistanceType enum (default is L2Expanded) * @param[in] metricArg metric argument to use. Corresponds to the p arg for lp norm */ template * translations = nullptr, - raft::distance::DistanceType metric = raft::distance::DistanceType::L2Expanded, + cuvs::distance::DistanceType metric = cuvs::distance::DistanceType::L2Expanded, float metricArg = 0, DistanceEpilogue distance_epilogue = raft::identity_op(), std::vector* input_norms = nullptr, @@ -418,10 +418,10 @@ void brute_force_knn_impl( if (k <= 64 && rowMajorQuery == rowMajorIndex && rowMajorQuery == true && std::is_same_v && - (metric == raft::distance::DistanceType::L2Unexpanded || - metric == raft::distance::DistanceType::L2SqrtUnexpanded || - metric == raft::distance::DistanceType::L2Expanded || - metric == raft::distance::DistanceType::L2SqrtExpanded)) { + (metric == cuvs::distance::DistanceType::L2Unexpanded || + metric == cuvs::distance::DistanceType::L2SqrtUnexpanded || + metric == cuvs::distance::DistanceType::L2Expanded || + metric == cuvs::distance::DistanceType::L2SqrtExpanded)) { fusedL2Knn(D, out_i_ptr, out_d_ptr, @@ -438,11 +438,11 @@ void brute_force_knn_impl( search_norms); // Perform necessary post-processing - if (metric == raft::distance::DistanceType::L2SqrtExpanded || - metric == raft::distance::DistanceType::L2SqrtUnexpanded || - metric == raft::distance::DistanceType::LpUnexpanded) { + if (metric == cuvs::distance::DistanceType::L2SqrtExpanded || + metric == cuvs::distance::DistanceType::L2SqrtUnexpanded || + metric == cuvs::distance::DistanceType::LpUnexpanded) { float p = 0.5; // standard l2 - if (metric == raft::distance::DistanceType::LpUnexpanded) p = 1.0 / metricArg; + if (metric == cuvs::distance::DistanceType::LpUnexpanded) p = 1.0 / metricArg; raft::linalg::unaryOp( res_D, res_D, @@ -452,7 +452,7 @@ void brute_force_knn_impl( } } else { switch (metric) { - case raft::distance::DistanceType::Haversine: + case cuvs::distance::DistanceType::Haversine: ASSERT(D == 2, "Haversine distance requires 2 dimensions " "(latitude / longitude)."); @@ -511,7 +511,7 @@ void brute_force_knn_impl( template void brute_force_search( raft::resources const& res, - const raft::neighbors::brute_force::index& idx, + const cuvs::neighbors::brute_force::index& idx, raft::device_matrix_view queries, raft::device_matrix_view neighbors, raft::device_matrix_view distances, @@ -547,4 +547,4 @@ void brute_force_search( norms.size() ? &norms : nullptr, query_norms ? query_norms->data_handle() : nullptr); } -} // namespace raft::neighbors::detail +} // namespace cuvs::neighbors::detail diff --git a/cpp/include/cuvs/neighbors/detail/knn_brute_force_batch_k_query.cuh b/cpp/include/cuvs/neighbors/detail/knn_brute_force_batch_k_query.cuh index 384eacae7..27e6c54ba 100644 --- a/cpp/include/cuvs/neighbors/detail/knn_brute_force_batch_k_query.cuh +++ b/cpp/include/cuvs/neighbors/detail/knn_brute_force_batch_k_query.cuh @@ -15,17 +15,17 @@ */ #pragma once +#include +#include #include #include -#include -#include -namespace raft::neighbors::brute_force::detail { +namespace cuvs::neighbors::brute_force::detail { template class gpu_batch_k_query : public batch_k_query { public: gpu_batch_k_query(const raft::resources& res, - const raft::neighbors::brute_force::index& index, + const cuvs::neighbors::brute_force::index& index, raft::device_matrix_view query, int64_t batch_size) : batch_k_query(res, index.size(), query.extent(0), batch_size), @@ -35,12 +35,12 @@ class gpu_batch_k_query : public batch_k_query { auto metric = index.metric(); // precompute query norms, and re-use across batches - if (metric == raft::distance::DistanceType::L2Expanded || - metric == raft::distance::DistanceType::L2SqrtExpanded || - metric == raft::distance::DistanceType::CosineExpanded) { + if (metric == cuvs::distance::DistanceType::L2Expanded || + metric == cuvs::distance::DistanceType::L2SqrtExpanded || + metric == cuvs::distance::DistanceType::CosineExpanded) { query_norms = make_device_vector(res, query.extent(0)); - if (metric == raft::distance::DistanceType::CosineExpanded) { + if (metric == cuvs::distance::DistanceType::CosineExpanded) { raft::linalg::norm(res, query, query_norms->view(), @@ -70,7 +70,7 @@ class gpu_batch_k_query : public batch_k_query { std::optional> query_norms_view; if (query_norms) { query_norms_view = query_norms->view(); } - raft::neighbors::detail::brute_force_search( + cuvs::neighbors::detail::brute_force_search( this->res, index, query, output->indices(), output->distances(), query_norms_view); }; @@ -91,8 +91,8 @@ class gpu_batch_k_query : public batch_k_query { matrix::slice(this->res, input.distances(), output->distances(), coords); } - const raft::neighbors::brute_force::index& index; + const cuvs::neighbors::brute_force::index& index; raft::device_matrix_view query; std::optional> query_norms; }; -} // namespace raft::neighbors::brute_force::detail +} // namespace cuvs::neighbors::brute_force::detail diff --git a/cpp/include/cuvs/neighbors/detail/knn_merge_parts.cuh b/cpp/include/cuvs/neighbors/detail/knn_merge_parts.cuh index 3d03d6db4..54b97f562 100644 --- a/cpp/include/cuvs/neighbors/detail/knn_merge_parts.cuh +++ b/cpp/include/cuvs/neighbors/detail/knn_merge_parts.cuh @@ -20,10 +20,10 @@ #include #include -#include -#include +#include +#include -namespace raft::neighbors::detail { +namespace cuvs::neighbors::detail { template ( inK, inV, outK, outV, n_samples, n_parts, k, stream, translations); } -} // namespace raft::neighbors::detail +} // namespace cuvs::neighbors::detail diff --git a/cpp/include/cuvs/neighbors/detail/nn_descent.cuh b/cpp/include/cuvs/neighbors/detail/nn_descent.cuh index 6e0636c37..909abaa65 100644 --- a/cpp/include/cuvs/neighbors/detail/nn_descent.cuh +++ b/cpp/include/cuvs/neighbors/detail/nn_descent.cuh @@ -35,12 +35,12 @@ #include "../nn_descent_types.hpp" +#include #include #include #include #include #include -#include #include #include // raft::util::arch::SM_* #include @@ -48,7 +48,7 @@ #include #include -namespace raft::neighbors::experimental::nn_descent::detail { +namespace cuvs::neighbors::experimental::nn_descent::detail { using pinned_memory_resource = thrust::universal_host_pinned_memory_resource; template @@ -1349,7 +1349,7 @@ void GNND::build(Data_t* data, const Index_t nrow, Index_t* out graph_shrink_buffer[i * build_config_.node_degree + j] = id; } else { graph_shrink_buffer[i * build_config_.node_degree + j] = - raft::neighbors::cagra::detail::device::xorshift64(idx) % nrow_; + cuvs::neighbors::cagra::detail::device::xorshift64(idx) % nrow_; } } } @@ -1451,4 +1451,4 @@ index build(raft::resources const& res, return idx; } -} // namespace raft::neighbors::experimental::nn_descent::detail +} // namespace cuvs::neighbors::experimental::nn_descent::detail diff --git a/cpp/include/cuvs/neighbors/detail/refine_common.hpp b/cpp/include/cuvs/neighbors/detail/refine_common.hpp index bfd3341ee..3def36a39 100644 --- a/cpp/include/cuvs/neighbors/detail/refine_common.hpp +++ b/cpp/include/cuvs/neighbors/detail/refine_common.hpp @@ -16,10 +16,10 @@ #pragma once +#include #include -#include -namespace raft::neighbors::detail { +namespace cuvs::neighbors::detail { /** Checks whether the input data extents are compatible. */ template @@ -54,4 +54,4 @@ void refine_check_input(ExtentsT dataset, static_cast(k)); } -} // namespace raft::neighbors::detail +} // namespace cuvs::neighbors::detail diff --git a/cpp/include/cuvs/neighbors/detail/refine_device.cuh b/cpp/include/cuvs/neighbors/detail/refine_device.cuh index 6ee96957f..96e9529a9 100644 --- a/cpp/include/cuvs/neighbors/detail/refine_device.cuh +++ b/cpp/include/cuvs/neighbors/detail/refine_device.cuh @@ -16,6 +16,10 @@ #pragma once +#include +#include +#include +#include #include #include #include @@ -23,18 +27,14 @@ #include #include #include -#include -#include -#include -#include #include #include -namespace raft::neighbors::detail { +namespace cuvs::neighbors::detail { /** - * See raft::neighbors::refine for docs. + * See cuvs::neighbors::refine for docs. */ template void refine_device(raft::resources const& handle, @@ -77,17 +77,17 @@ void refine_device(raft::resources const& handle, fake_coarse_idx.data(), fake_coarse_idx.data() + n_queries); - raft::neighbors::ivf_flat::index refinement_index( + cuvs::neighbors::ivf_flat::index refinement_index( handle, metric, n_queries, false, true, dim); - raft::neighbors::ivf_flat::detail::fill_refinement_index(handle, + cuvs::neighbors::ivf_flat::detail::fill_refinement_index(handle, &refinement_index, dataset.data_handle(), neighbor_candidates.data_handle(), n_queries, n_candidates); uint32_t grid_dim_x = 1; - raft::neighbors::ivf_flat::detail::ivfflat_interleaved_scan< + cuvs::neighbors::ivf_flat::detail::ivfflat_interleaved_scan< data_t, typename raft::spatial::knn::detail::utils::config::value_t, idx_t>(refinement_index, @@ -98,12 +98,12 @@ void refine_device(raft::resources const& handle, refinement_index.metric(), 1, k, - raft::distance::is_min_close(metric), - raft::neighbors::filtering::none_ivf_sample_filter(), + cuvs::distance::is_min_close(metric), + cuvs::neighbors::filtering::none_ivf_sample_filter(), indices.data_handle(), distances.data_handle(), grid_dim_x, resource::get_cuda_stream(handle)); } -} // namespace raft::neighbors::detail +} // namespace cuvs::neighbors::detail diff --git a/cpp/include/cuvs/neighbors/detail/refine_host-ext.hpp b/cpp/include/cuvs/neighbors/detail/refine_host-ext.hpp index 2a863b47b..bcf24ed8d 100644 --- a/cpp/include/cuvs/neighbors/detail/refine_host-ext.hpp +++ b/cpp/include/cuvs/neighbors/detail/refine_host-ext.hpp @@ -18,13 +18,13 @@ #include // int64_t +#include // cuvs::distance::DistanceType #include // raft::host_matrix_view -#include // raft::distance::DistanceType #include // RAFT_EXPLICIT #ifdef RAFT_EXPLICIT_INSTANTIATE_ONLY -namespace raft::neighbors::detail { +namespace cuvs::neighbors::detail { template [[gnu::optimize(3), gnu::optimize("tree-vectorize")]] void refine_host( @@ -40,7 +40,7 @@ template #endif // RAFT_EXPLICIT_INSTANTIATE_ONLY #define instantiate_raft_neighbors_refine(IdxT, DataT, DistanceT, ExtentsT) \ - extern template void raft::neighbors::detail::refine_host( \ + extern template void cuvs::neighbors::detail::refine_host( \ raft::host_matrix_view dataset, \ raft::host_matrix_view queries, \ raft::host_matrix_view neighbor_candidates, \ diff --git a/cpp/include/cuvs/neighbors/detail/refine_host-inl.hpp b/cpp/include/cuvs/neighbors/detail/refine_host-inl.hpp index 14c53a469..e5839e8d9 100644 --- a/cpp/include/cuvs/neighbors/detail/refine_host-inl.hpp +++ b/cpp/include/cuvs/neighbors/detail/refine_host-inl.hpp @@ -16,14 +16,14 @@ #pragma once +#include #include #include -#include #include #include -namespace raft::neighbors::detail { +namespace cuvs::neighbors::detail { template [[gnu::optimize(3), gnu::optimize("tree-vectorize")]] void refine_host_impl( @@ -126,14 +126,14 @@ template metric); switch (metric) { - case raft::distance::DistanceType::L2Expanded: + case cuvs::distance::DistanceType::L2Expanded: return refine_host_impl( dataset, queries, neighbor_candidates, indices, distances); - case raft::distance::DistanceType::InnerProduct: + case cuvs::distance::DistanceType::InnerProduct: return refine_host_impl( dataset, queries, neighbor_candidates, indices, distances); default: throw raft::logic_error("Unsupported metric"); } } -} // namespace raft::neighbors::detail +} // namespace cuvs::neighbors::detail diff --git a/cpp/include/cuvs/neighbors/detail/selection_faiss-ext.cuh b/cpp/include/cuvs/neighbors/detail/selection_faiss-ext.cuh index a6ed17e25..e123f81e7 100644 --- a/cpp/include/cuvs/neighbors/detail/selection_faiss-ext.cuh +++ b/cpp/include/cuvs/neighbors/detail/selection_faiss-ext.cuh @@ -19,12 +19,12 @@ #include // size_t #include // uint32_t #include // __half -#include // kFaissMaxK +#include // kFaissMaxK #include // RAFT_EXPLICIT #if defined(RAFT_EXPLICIT_INSTANTIATE_ONLY) -namespace raft::neighbors::detail { +namespace cuvs::neighbors::detail { template void select_k(const key_t* inK, @@ -36,12 +36,12 @@ void select_k(const key_t* inK, bool select_min, int k, cudaStream_t stream) RAFT_EXPLICIT; -}; // namespace raft::neighbors::detail +}; // namespace cuvs::neighbors::detail #endif // RAFT_EXPLICIT_INSTANTIATE_ONLY #define instantiate_raft_neighbors_detail_select_k(payload_t, key_t) \ - extern template void raft::neighbors::detail::select_k(const key_t* inK, \ + extern template void cuvs::neighbors::detail::select_k(const key_t* inK, \ const payload_t* inV, \ size_t n_rows, \ size_t n_cols, \ diff --git a/cpp/include/cuvs/neighbors/detail/selection_faiss-inl.cuh b/cpp/include/cuvs/neighbors/detail/selection_faiss-inl.cuh index 35f322bbf..acc931e4a 100644 --- a/cpp/include/cuvs/neighbors/detail/selection_faiss-inl.cuh +++ b/cpp/include/cuvs/neighbors/detail/selection_faiss-inl.cuh @@ -19,10 +19,10 @@ #include #include -#include -#include // kFaissMaxK +#include +#include // kFaissMaxK -namespace raft::neighbors::detail { +namespace cuvs::neighbors::detail { template RAFT_KERNEL select_k_kernel(const key_t* inK, @@ -160,4 +160,4 @@ inline void select_k(const key_t* inK, else ASSERT(k <= max_k, "Current max k is %d (requested %d)", max_k, k); } -}; // namespace raft::neighbors::detail +}; // namespace cuvs::neighbors::detail diff --git a/cpp/include/cuvs/neighbors/detail/selection_faiss_helpers.cuh b/cpp/include/cuvs/neighbors/detail/selection_faiss_helpers.cuh index c4b69f21e..bbe4752d2 100644 --- a/cpp/include/cuvs/neighbors/detail/selection_faiss_helpers.cuh +++ b/cpp/include/cuvs/neighbors/detail/selection_faiss_helpers.cuh @@ -16,7 +16,7 @@ #pragma once -namespace raft::neighbors::detail { +namespace cuvs::neighbors::detail { // This function is used in cpp/test/neighbors/select.cu. We want to make it // available through both the selection_faiss-inl.cuh and @@ -28,4 +28,4 @@ constexpr int kFaissMaxK() return 2048; } -} // namespace raft::neighbors::detail +} // namespace cuvs::neighbors::detail diff --git a/cpp/include/cuvs/neighbors/epsilon_neighborhood.cuh b/cpp/include/cuvs/neighbors/epsilon_neighborhood.cuh index bade4385f..1e27ecae7 100644 --- a/cpp/include/cuvs/neighbors/epsilon_neighborhood.cuh +++ b/cpp/include/cuvs/neighbors/epsilon_neighborhood.cuh @@ -24,7 +24,7 @@ #include #include -namespace raft::neighbors::epsilon_neighborhood { +namespace cuvs::neighbors::epsilon_neighborhood { /** * @brief Computes epsilon neighborhood for the L2-Squared distance metric @@ -72,10 +72,10 @@ void epsUnexpL2SqNeighborhood(bool* adj, * indptr array. * * @code{.cpp} - * #include + * #include * #include * #include - * using namespace raft::neighbors; + * using namespace cuvs::neighbors; * raft::raft::resources handle; * ... * auto adj = raft::make_device_matrix(handle, m * n); @@ -118,6 +118,6 @@ void eps_neighbors_l2sq(raft::resources const& handle, /** @} */ // end group epsilon_neighbors -} // namespace raft::neighbors::epsilon_neighborhood +} // namespace cuvs::neighbors::epsilon_neighborhood #endif \ No newline at end of file diff --git a/cpp/include/cuvs/neighbors/ivf_flat-ext.cuh b/cpp/include/cuvs/neighbors/ivf_flat-ext.cuh index 8dbe7587f..5b9d4b3a6 100644 --- a/cpp/include/cuvs/neighbors/ivf_flat-ext.cuh +++ b/cpp/include/cuvs/neighbors/ivf_flat-ext.cuh @@ -18,16 +18,16 @@ #include // int64_t -#include // raft::device_matrix_view -#include // raft::resources -#include -#include // raft::neighbors::ivf_flat::index +#include +#include // cuvs::neighbors::ivf_flat::index +#include // raft::device_matrix_view +#include // raft::resources #include // RAFT_EXPLICIT #include // rmm::mr::device_memory_resource #ifdef RAFT_EXPLICIT_INSTANTIATE_ONLY -namespace raft::neighbors::ivf_flat { +namespace cuvs::neighbors::ivf_flat { template auto build(raft::resources const& handle, @@ -46,7 +46,7 @@ template void build(raft::resources const& handle, const index_params& params, raft::device_matrix_view dataset, - raft::neighbors::ivf_flat::index& idx) RAFT_EXPLICIT; + cuvs::neighbors::ivf_flat::index& idx) RAFT_EXPLICIT; template auto extend(raft::resources const& handle, @@ -114,30 +114,30 @@ void search(raft::resources const& handle, raft::device_matrix_view neighbors, raft::device_matrix_view distances) RAFT_EXPLICIT; -} // namespace raft::neighbors::ivf_flat +} // namespace cuvs::neighbors::ivf_flat #endif // RAFT_EXPLICIT_INSTANTIATE_ONLY #define instantiate_raft_neighbors_ivf_flat_build(T, IdxT) \ - extern template auto raft::neighbors::ivf_flat::build( \ + extern template auto cuvs::neighbors::ivf_flat::build( \ raft::resources const& handle, \ - const raft::neighbors::ivf_flat::index_params& params, \ + const cuvs::neighbors::ivf_flat::index_params& params, \ const T* dataset, \ IdxT n_rows, \ uint32_t dim) \ - ->raft::neighbors::ivf_flat::index; \ + ->cuvs::neighbors::ivf_flat::index; \ \ - extern template auto raft::neighbors::ivf_flat::build( \ + extern template auto cuvs::neighbors::ivf_flat::build( \ raft::resources const& handle, \ - const raft::neighbors::ivf_flat::index_params& params, \ + const cuvs::neighbors::ivf_flat::index_params& params, \ raft::device_matrix_view dataset) \ - ->raft::neighbors::ivf_flat::index; \ + ->cuvs::neighbors::ivf_flat::index; \ \ - extern template void raft::neighbors::ivf_flat::build( \ + extern template void cuvs::neighbors::ivf_flat::build( \ raft::resources const& handle, \ - const raft::neighbors::ivf_flat::index_params& params, \ + const cuvs::neighbors::ivf_flat::index_params& params, \ raft::device_matrix_view dataset, \ - raft::neighbors::ivf_flat::index& idx); + cuvs::neighbors::ivf_flat::index& idx); instantiate_raft_neighbors_ivf_flat_build(float, int64_t); instantiate_raft_neighbors_ivf_flat_build(int8_t, int64_t); @@ -145,33 +145,33 @@ instantiate_raft_neighbors_ivf_flat_build(uint8_t, int64_t); #undef instantiate_raft_neighbors_ivf_flat_build #define instantiate_raft_neighbors_ivf_flat_extend(T, IdxT) \ - extern template auto raft::neighbors::ivf_flat::extend( \ + extern template auto cuvs::neighbors::ivf_flat::extend( \ raft::resources const& handle, \ - const raft::neighbors::ivf_flat::index& orig_index, \ + const cuvs::neighbors::ivf_flat::index& orig_index, \ const T* new_vectors, \ const IdxT* new_indices, \ IdxT n_rows) \ - ->raft::neighbors::ivf_flat::index; \ + ->cuvs::neighbors::ivf_flat::index; \ \ - extern template auto raft::neighbors::ivf_flat::extend( \ + extern template auto cuvs::neighbors::ivf_flat::extend( \ raft::resources const& handle, \ raft::device_matrix_view new_vectors, \ std::optional> new_indices, \ - const raft::neighbors::ivf_flat::index& orig_index) \ - ->raft::neighbors::ivf_flat::index; \ + const cuvs::neighbors::ivf_flat::index& orig_index) \ + ->cuvs::neighbors::ivf_flat::index; \ \ - extern template void raft::neighbors::ivf_flat::extend( \ + extern template void cuvs::neighbors::ivf_flat::extend( \ raft::resources const& handle, \ - raft::neighbors::ivf_flat::index* index, \ + cuvs::neighbors::ivf_flat::index* index, \ const T* new_vectors, \ const IdxT* new_indices, \ IdxT n_rows); \ \ - extern template void raft::neighbors::ivf_flat::extend( \ + extern template void cuvs::neighbors::ivf_flat::extend( \ raft::resources const& handle, \ raft::device_matrix_view new_vectors, \ std::optional> new_indices, \ - raft::neighbors::ivf_flat::index* index); + cuvs::neighbors::ivf_flat::index* index); instantiate_raft_neighbors_ivf_flat_extend(float, int64_t); instantiate_raft_neighbors_ivf_flat_extend(int8_t, int64_t); @@ -180,10 +180,10 @@ instantiate_raft_neighbors_ivf_flat_extend(uint8_t, int64_t); #undef instantiate_raft_neighbors_ivf_flat_extend #define instantiate_raft_neighbors_ivf_flat_search(T, IdxT) \ - extern template void raft::neighbors::ivf_flat::search( \ + extern template void cuvs::neighbors::ivf_flat::search( \ raft::resources const& handle, \ - const raft::neighbors::ivf_flat::search_params& params, \ - const raft::neighbors::ivf_flat::index& index, \ + const cuvs::neighbors::ivf_flat::search_params& params, \ + const cuvs::neighbors::ivf_flat::index& index, \ const T* queries, \ uint32_t n_queries, \ uint32_t k, \ @@ -191,10 +191,10 @@ instantiate_raft_neighbors_ivf_flat_extend(uint8_t, int64_t); float* distances, \ rmm::mr::device_memory_resource* mr); \ \ - extern template void raft::neighbors::ivf_flat::search( \ + extern template void cuvs::neighbors::ivf_flat::search( \ raft::resources const& handle, \ - const raft::neighbors::ivf_flat::search_params& params, \ - const raft::neighbors::ivf_flat::index& index, \ + const cuvs::neighbors::ivf_flat::search_params& params, \ + const cuvs::neighbors::ivf_flat::index& index, \ raft::device_matrix_view queries, \ raft::device_matrix_view neighbors, \ raft::device_matrix_view distances); diff --git a/cpp/include/cuvs/neighbors/ivf_flat-inl.cuh b/cpp/include/cuvs/neighbors/ivf_flat-inl.cuh index 692fb0881..53bdcfe5a 100644 --- a/cpp/include/cuvs/neighbors/ivf_flat-inl.cuh +++ b/cpp/include/cuvs/neighbors/ivf_flat-inl.cuh @@ -16,10 +16,10 @@ #pragma once -#include -#include -#include -#include +#include +#include +#include +#include #include @@ -27,7 +27,7 @@ #include #include -namespace raft::neighbors::ivf_flat { +namespace cuvs::neighbors::ivf_flat { /** * @brief Build the index from the dataset for efficient search. @@ -39,7 +39,7 @@ namespace raft::neighbors::ivf_flat { * * Usage example: * @code{.cpp} - * using namespace raft::neighbors; + * using namespace cuvs::neighbors; * // use default index parameters * ivf_flat::index_params index_params; * // create and fill the index from a [N, D] dataset @@ -68,7 +68,7 @@ auto build(raft::resources const& handle, IdxT n_rows, uint32_t dim) -> index { - return raft::neighbors::ivf_flat::detail::build(handle, params, dataset, n_rows, dim); + return cuvs::neighbors::ivf_flat::detail::build(handle, params, dataset, n_rows, dim); } /** @@ -86,7 +86,7 @@ auto build(raft::resources const& handle, * * Usage example: * @code{.cpp} - * using namespace raft::neighbors; + * using namespace cuvs::neighbors; * // use default index parameters * ivf_flat::index_params index_params; * // create and fill the index from a [N, D] dataset @@ -111,7 +111,7 @@ auto build(raft::resources const& handle, const index_params& params, raft::device_matrix_view dataset) -> index { - return raft::neighbors::ivf_flat::detail::build(handle, + return cuvs::neighbors::ivf_flat::detail::build(handle, params, dataset.data_handle(), static_cast(dataset.extent(0)), @@ -128,7 +128,7 @@ auto build(raft::resources const& handle, * * Usage example: * @code{.cpp} - * using namespace raft::neighbors; + * using namespace cuvs::neighbors; * // use default index parameters * ivf_flat::index_params index_params; * // create and fill the index from a [N, D] dataset @@ -153,9 +153,9 @@ template void build(raft::resources const& handle, const index_params& params, raft::device_matrix_view dataset, - raft::neighbors::ivf_flat::index& idx) + cuvs::neighbors::ivf_flat::index& idx) { - idx = raft::neighbors::ivf_flat::detail::build(handle, + idx = cuvs::neighbors::ivf_flat::detail::build(handle, params, dataset.data_handle(), static_cast(dataset.extent(0)), @@ -173,7 +173,7 @@ void build(raft::resources const& handle, * * Usage example: * @code{.cpp} - * using namespace raft::neighbors; + * using namespace cuvs::neighbors; * ivf_flat::index_params index_params; * index_params.add_data_on_build = false; // don't populate index on build * index_params.kmeans_trainset_fraction = 1.0; // use whole dataset for kmeans training @@ -203,7 +203,7 @@ auto extend(raft::resources const& handle, const IdxT* new_indices, IdxT n_rows) -> index { - return raft::neighbors::ivf_flat::detail::extend( + return cuvs::neighbors::ivf_flat::detail::extend( handle, orig_index, new_vectors, new_indices, n_rows); } @@ -221,7 +221,7 @@ auto extend(raft::resources const& handle, * * Usage example: * @code{.cpp} - * using namespace raft::neighbors; + * using namespace cuvs::neighbors; * ivf_flat::index_params index_params; * index_params.add_data_on_build = false; // don't populate index on build * index_params.kmeans_trainset_fraction = 1.0; // use whole dataset for kmeans training @@ -264,7 +264,7 @@ auto extend(raft::resources const& handle, * * Usage example: * @code{.cpp} - * using namespace raft::neighbors; + * using namespace cuvs::neighbors; * ivf_flat::index_params index_params; * index_params.add_data_on_build = false; // don't populate index on build * index_params.kmeans_trainset_fraction = 1.0; // use whole dataset for kmeans training @@ -292,7 +292,7 @@ void extend(raft::resources const& handle, const IdxT* new_indices, IdxT n_rows) { - raft::neighbors::ivf_flat::detail::extend(handle, index, new_vectors, new_indices, n_rows); + cuvs::neighbors::ivf_flat::detail::extend(handle, index, new_vectors, new_indices, n_rows); } /** @@ -305,7 +305,7 @@ void extend(raft::resources const& handle, * * Usage example: * @code{.cpp} - * using namespace raft::neighbors; + * using namespace cuvs::neighbors; * ivf_flat::index_params index_params; * index_params.add_data_on_build = false; // don't populate index on build * index_params.kmeans_trainset_fraction = 1.0; // use whole dataset for kmeans training @@ -403,7 +403,7 @@ void search_with_filtering(raft::resources const& handle, rmm::mr::device_memory_resource* mr = nullptr, IvfSampleFilterT sample_filter = IvfSampleFilterT()) { - raft::neighbors::ivf_flat::detail::search( + cuvs::neighbors::ivf_flat::detail::search( handle, params, index, queries, n_queries, k, neighbors, distances, mr, sample_filter); } @@ -460,7 +460,7 @@ void search(raft::resources const& handle, float* distances, rmm::mr::device_memory_resource* mr = nullptr) { - raft::neighbors::ivf_flat::detail::search(handle, + cuvs::neighbors::ivf_flat::detail::search(handle, params, index, queries, @@ -469,7 +469,7 @@ void search(raft::resources const& handle, neighbors, distances, mr, - raft::neighbors::filtering::none_ivf_sample_filter()); + cuvs::neighbors::filtering::none_ivf_sample_filter()); } /** @@ -594,9 +594,9 @@ void search(raft::resources const& handle, queries, neighbors, distances, - raft::neighbors::filtering::none_ivf_sample_filter()); + cuvs::neighbors::filtering::none_ivf_sample_filter()); } /** @} */ -} // namespace raft::neighbors::ivf_flat +} // namespace cuvs::neighbors::ivf_flat diff --git a/cpp/include/cuvs/neighbors/ivf_flat_codepacker.hpp b/cpp/include/cuvs/neighbors/ivf_flat_codepacker.hpp index 5379788ab..9f1b43380 100644 --- a/cpp/include/cuvs/neighbors/ivf_flat_codepacker.hpp +++ b/cpp/include/cuvs/neighbors/ivf_flat_codepacker.hpp @@ -16,13 +16,13 @@ #pragma once +#include +#include #include #include #include -#include -#include -namespace raft::neighbors::ivf_flat::codepacker { +namespace cuvs::neighbors::ivf_flat::codepacker { /** * Write one flat code into a block by the given offset. The offset indicates the id of the record @@ -87,4 +87,4 @@ _RAFT_HOST_DEVICE void unpack_1( } } } -} // namespace raft::neighbors::ivf_flat::codepacker \ No newline at end of file +} // namespace cuvs::neighbors::ivf_flat::codepacker \ No newline at end of file diff --git a/cpp/include/cuvs/neighbors/ivf_flat_helpers.cuh b/cpp/include/cuvs/neighbors/ivf_flat_helpers.cuh index 7a05c9991..e2ec252bb 100644 --- a/cpp/include/cuvs/neighbors/ivf_flat_helpers.cuh +++ b/cpp/include/cuvs/neighbors/ivf_flat_helpers.cuh @@ -16,15 +16,15 @@ #pragma once -#include -#include +#include +#include #include #include #include -namespace raft::neighbors::ivf_flat::helpers { +namespace cuvs::neighbors::ivf_flat::helpers { using namespace raft::spatial::knn::detail; // NOLINT /** * @defgroup ivf_flat_helpers Helper functions for manipulationg IVF Flat Index @@ -66,7 +66,7 @@ void pack( uint32_t offset, device_mdspan::list_extents, row_major> list_data) { - raft::neighbors::ivf_flat::detail::pack_list_data(res, codes, veclen, offset, list_data); + cuvs::neighbors::ivf_flat::detail::pack_list_data(res, codes, veclen, offset, list_data); } /** @@ -105,7 +105,7 @@ void unpack( uint32_t offset, device_matrix_view codes) { - raft::neighbors::ivf_flat::detail::unpack_list_data( + cuvs::neighbors::ivf_flat::detail::unpack_list_data( res, list_data, veclen, offset, codes); } } // namespace codepacker @@ -118,7 +118,7 @@ void unpack( * Usage example: * @code{.cpp} * raft::resources res; - * using namespace raft::neighbors; + * using namespace cuvs::neighbors; * // use default index parameters * ivf_flat::index_params index_params; * // initialize an empty index @@ -142,4 +142,4 @@ void reset_index(const raft::resources& res, index* index) utils::memzero(index->inds_ptrs().data_handle(), index->inds_ptrs().size(), stream); } /** @} */ -} // namespace raft::neighbors::ivf_flat::helpers +} // namespace cuvs::neighbors::ivf_flat::helpers diff --git a/cpp/include/cuvs/neighbors/ivf_flat_serialize.cuh b/cpp/include/cuvs/neighbors/ivf_flat_serialize.cuh index 311c31040..37062ea68 100644 --- a/cpp/include/cuvs/neighbors/ivf_flat_serialize.cuh +++ b/cpp/include/cuvs/neighbors/ivf_flat_serialize.cuh @@ -18,7 +18,7 @@ #include "detail/ivf_flat_serialize.cuh" -namespace raft::neighbors::ivf_flat { +namespace cuvs::neighbors::ivf_flat { /** * \defgroup ivf_flat_serialize IVF-Flat Serialize @@ -110,7 +110,7 @@ void serialize(raft::resources const& handle, * @param[in] handle the raft handle * @param[in] is input stream * - * @return raft::neighbors::ivf_flat::index + * @return cuvs::neighbors::ivf_flat::index */ template index deserialize(raft::resources const& handle, std::istream& is) @@ -141,7 +141,7 @@ index deserialize(raft::resources const& handle, std::istream& is) * @param[in] handle the raft handle * @param[in] filename the name of the file that stores the index * - * @return raft::neighbors::ivf_flat::index + * @return cuvs::neighbors::ivf_flat::index */ template index deserialize(raft::resources const& handle, const std::string& filename) @@ -151,4 +151,4 @@ index deserialize(raft::resources const& handle, const std::string& fil /**@}*/ -} // namespace raft::neighbors::ivf_flat +} // namespace cuvs::neighbors::ivf_flat diff --git a/cpp/include/cuvs/neighbors/ivf_flat_types.hpp b/cpp/include/cuvs/neighbors/ivf_flat_types.hpp index 180fe2e21..47b9fd9f1 100644 --- a/cpp/include/cuvs/neighbors/ivf_flat_types.hpp +++ b/cpp/include/cuvs/neighbors/ivf_flat_types.hpp @@ -19,6 +19,8 @@ #include "ann_types.hpp" #include +#include +#include #include #include #include @@ -26,8 +28,6 @@ #include #include #include -#include -#include #include #include @@ -37,7 +37,7 @@ #include #include -namespace raft::neighbors::ivf_flat { +namespace cuvs::neighbors::ivf_flat { /** * @addtogroup ivf_flat * @{ @@ -141,7 +141,7 @@ struct index : ann::index { */ [[nodiscard]] constexpr inline auto veclen() const noexcept -> uint32_t { return veclen_; } /** Distance metric used for clustering. */ - [[nodiscard]] constexpr inline auto metric() const noexcept -> raft::distance::DistanceType + [[nodiscard]] constexpr inline auto metric() const noexcept -> cuvs::distance::DistanceType { return metric_; } @@ -241,7 +241,7 @@ struct index : ann::index { /** Construct an empty index. It needs to be trained and then populated. */ index(raft::resources const& res, - raft::distance::DistanceType metric, + cuvs::distance::DistanceType metric, uint32_t n_lists, bool adaptive_centers, bool conservative_memory_allocation, @@ -328,10 +328,10 @@ struct index : ann::index { void allocate_center_norms(raft::resources const& res) { switch (metric_) { - case raft::distance::DistanceType::L2Expanded: - case raft::distance::DistanceType::L2SqrtExpanded: - case raft::distance::DistanceType::L2Unexpanded: - case raft::distance::DistanceType::L2SqrtUnexpanded: + case cuvs::distance::DistanceType::L2Expanded: + case cuvs::distance::DistanceType::L2SqrtExpanded: + case cuvs::distance::DistanceType::L2Unexpanded: + case cuvs::distance::DistanceType::L2SqrtUnexpanded: center_norms_ = make_device_vector(res, n_lists()); break; default: center_norms_ = std::nullopt; @@ -355,7 +355,7 @@ struct index : ann::index { * possible value by padding the `dim` of the data https://github.com/rapidsai/raft/issues/711 */ uint32_t veclen_; - raft::distance::DistanceType metric_; + cuvs::distance::DistanceType metric_; bool adaptive_centers_; bool conservative_memory_allocation_; std::vector>> lists_; @@ -397,4 +397,4 @@ struct index : ann::index { /** @} */ -} // namespace raft::neighbors::ivf_flat +} // namespace cuvs::neighbors::ivf_flat diff --git a/cpp/include/cuvs/neighbors/ivf_list.hpp b/cpp/include/cuvs/neighbors/ivf_list.hpp index ad06a3ee7..8dd211266 100644 --- a/cpp/include/cuvs/neighbors/ivf_list.hpp +++ b/cpp/include/cuvs/neighbors/ivf_list.hpp @@ -16,9 +16,9 @@ #pragma once +#include #include #include -#include #include #include @@ -34,7 +34,7 @@ #include #include -namespace raft::neighbors::ivf { +namespace cuvs::neighbors::ivf { /** The data for a single IVF list. */ template