Skip to content

Removing filter templates in ivfpq for binary size reduction #1211

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rapids-bot merged 27 commits into from
Aug 27, 2025
Merged

Removing filter templates in ivfpq for binary size reduction #1211

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
f3a4fa1
remove instantiations
jinsolp Jul 29, 2025
8e17b88
revert original filters
jinsolp Jul 29, 2025
7f556be
rm commented out code
jinsolp Jul 29, 2025
90ff83a
avoid device side dynamic dispatch
jinsolp Aug 4, 2025
5a9f8be
Merge branch 'rapidsai:branch-25.10' into bs/filter-ivfpq
jinsolp Aug 4, 2025
c46d827
change var names
jinsolp Aug 4, 2025
014f3e7
Merge branch 'branch-25.10' into bs/filter-ivfpq
jinsolp Aug 4, 2025
f4644f5
separate default declaration
jinsolp Aug 4, 2025
35424fe
anonymous namespace for init/destruct kernels
jinsolp Aug 5, 2025
8bd21c4
address comments
jinsolp Aug 5, 2025
9eefe78
Merge branch 'branch-25.10' into bs/filter-ivfpq
jinsolp Aug 5, 2025
0706391
create inside kernel with enums
jinsolp Aug 7, 2025
837bcc0
merge commit resolve conflicts
jinsolp Aug 7, 2025
99061e8
Merge branch 'branch-25.10' into bs/filter-ivfpq
jinsolp Aug 11, 2025
4f98807
single struct for args
jinsolp Aug 12, 2025
2fec231
Merge branch 'branch-25.10' into bs/filter-ivfpq
jinsolp Aug 12, 2025
09068b2
change arg order
jinsolp Aug 12, 2025
ff2452f
address feedback
jinsolp Aug 13, 2025
6a945da
remove explicit default
jinsolp Aug 13, 2025
4a314f3
rm comment
jinsolp Aug 13, 2025
e1ba613
Merge branch 'branch-25.10' into bs/filter-ivfpq
jinsolp Aug 14, 2025
f6204f6
rm destructor and use init list
jinsolp Aug 14, 2025
01c7c21
Merge branch 'bs/filter-ivfpq' of https://github.com/jinsolp/cuvs int…
jinsolp Aug 14, 2025
4c5bbf3
Merge branch 'branch-25.10' into bs/filter-ivfpq
jinsolp Aug 14, 2025
24aa2cd
Merge branch 'branch-25.10' into bs/filter-ivfpq
jinsolp Aug 19, 2025
e3fb085
Merge branch 'branch-25.10' into bs/filter-ivfpq
divyegala Aug 25, 2025
2c1f267
Merge branch 'branch-25.10' into bs/filter-ivfpq
jinsolp Aug 27, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 0 additions & 7 deletions cpp/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -458,13 +458,6 @@ if(BUILD_SHARED_LIBS)
src/neighbors/ivf_pq/detail/ivf_pq_compute_similarity_float_float.cu
src/neighbors/ivf_pq/detail/ivf_pq_compute_similarity_float_fp8_false.cu
src/neighbors/ivf_pq/detail/ivf_pq_compute_similarity_float_fp8_true.cu
src/neighbors/ivf_pq/detail/ivf_pq_compute_similarity_half_fp8_false_bitset64.cu
src/neighbors/ivf_pq/detail/ivf_pq_compute_similarity_half_fp8_true_bitset64.cu
src/neighbors/ivf_pq/detail/ivf_pq_compute_similarity_half_half_bitset64.cu
src/neighbors/ivf_pq/detail/ivf_pq_compute_similarity_float_half_bitset64.cu
src/neighbors/ivf_pq/detail/ivf_pq_compute_similarity_float_float_bitset64.cu
src/neighbors/ivf_pq/detail/ivf_pq_compute_similarity_float_fp8_false_bitset64.cu
src/neighbors/ivf_pq/detail/ivf_pq_compute_similarity_float_fp8_true_bitset64.cu
src/neighbors/ivf_pq/detail/ivf_pq_contiguous_list_data.cu
src/neighbors/ivf_pq/detail/ivf_pq_process_and_fill_codes.cu
src/neighbors/ivf_pq/detail/ivf_pq_search_float_int64_t.cu
Expand Down
14 changes: 8 additions & 6 deletions cpp/include/cuvs/neighbors/common.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -485,21 +485,21 @@ namespace filtering {
enum class FilterType { None, Bitmap, Bitset };

struct base_filter {
~base_filter() = default;
virtual ~base_filter() = default;
virtual FilterType get_filter_type() const = 0;
};

/* A filter that filters nothing. This is the default behavior. */
struct none_sample_filter : public base_filter {
inline _RAFT_HOST_DEVICE bool operator()(
constexpr __forceinline__ _RAFT_HOST_DEVICE bool operator()(
// query index
const uint32_t query_ix,
// the current inverted list index
const uint32_t cluster_ix,
// the index of the current sample inside the current inverted list
const uint32_t sample_ix) const;

inline _RAFT_HOST_DEVICE bool operator()(
constexpr __forceinline__ _RAFT_HOST_DEVICE bool operator()(
// query index
const uint32_t query_ix,
// the index of the current sample
Expand All @@ -517,7 +517,7 @@ struct none_sample_filter : public base_filter {
* @tparam filter_t
*/
template <typename index_t, typename filter_t>
struct ivf_to_sample_filter {
struct ivf_to_sample_filter : public base_filter {
const index_t* const* inds_ptrs_;
const filter_t next_filter_;

Expand All @@ -534,6 +534,8 @@ struct ivf_to_sample_filter {
const uint32_t cluster_ix,
// the index of the current sample inside the current inverted list
const uint32_t sample_ix) const;

FilterType get_filter_type() const override { return next_filter_.get_filter_type(); }
};

/**
Expand Down Expand Up @@ -577,8 +579,8 @@ struct bitset_filter : public base_filter {
// View of the bitset to use as a filter
const view_t bitset_view_;

bitset_filter(const view_t bitset_for_filtering);
inline _RAFT_HOST_DEVICE bool operator()(
_RAFT_HOST_DEVICE bitset_filter(const view_t bitset_for_filtering);
constexpr __forceinline__ _RAFT_HOST_DEVICE bool operator()(
// query index
const uint32_t query_ix,
// the index of the current sample
Expand Down
83 changes: 38 additions & 45 deletions cpp/src/neighbors/ivf_pq/detail/ivf_pq_compute_similarity_float_float.cu
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2024, NVIDIA CORPORATION.
* Copyright (c) 2024-2025, NVIDIA CORPORATION.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand All @@ -24,50 +24,43 @@
#include "../ivf_pq_compute_similarity_impl.cuh"
#include "../ivf_pq_fp_8bit.cuh"

#define instantiate_cuvs_neighbors_ivf_pq_detail_compute_similarity_select( \
OutT, LutT, IvfSampleFilterT) \
template auto \
cuvs::neighbors::ivf_pq::detail::compute_similarity_select<OutT, LutT, IvfSampleFilterT>( \
const cudaDeviceProp& dev_props, \
bool manage_local_topk, \
int locality_hint, \
double preferred_shmem_carveout, \
uint32_t pq_bits, \
uint32_t pq_dim, \
uint32_t precomp_data_count, \
uint32_t n_queries, \
uint32_t n_probes, \
uint32_t topk) -> cuvs::neighbors::ivf_pq::detail::selected<OutT, LutT, IvfSampleFilterT>; \
\
template void \
cuvs::neighbors::ivf_pq::detail::compute_similarity_run<OutT, LutT, IvfSampleFilterT>( \
cuvs::neighbors::ivf_pq::detail::selected<OutT, LutT, IvfSampleFilterT> s, \
rmm::cuda_stream_view stream, \
uint32_t dim, \
uint32_t n_probes, \
uint32_t pq_dim, \
uint32_t n_queries, \
uint32_t queries_offset, \
cuvs::distance::DistanceType metric, \
cuvs::neighbors::ivf_pq::codebook_gen codebook_kind, \
uint32_t topk, \
uint32_t max_samples, \
const float* cluster_centers, \
const float* pq_centers, \
const uint8_t* const* pq_dataset, \
const uint32_t* cluster_labels, \
const uint32_t* _chunk_indices, \
const float* queries, \
const uint32_t* index_list, \
float* query_kths, \
IvfSampleFilterT sample_filter, \
LutT* lut_scores, \
OutT* _out_scores, \
#define instantiate_cuvs_neighbors_ivf_pq_detail_compute_similarity_select(OutT, LutT) \
template auto cuvs::neighbors::ivf_pq::detail::compute_similarity_select<OutT, LutT>( \
const cudaDeviceProp& dev_props, \
bool manage_local_topk, \
int locality_hint, \
double preferred_shmem_carveout, \
uint32_t pq_bits, \
uint32_t pq_dim, \
uint32_t precomp_data_count, \
uint32_t n_queries, \
uint32_t n_probes, \
uint32_t topk) -> cuvs::neighbors::ivf_pq::detail::selected<OutT, LutT>; \
\
template void cuvs::neighbors::ivf_pq::detail::compute_similarity_run<OutT, LutT>( \
cuvs::neighbors::ivf_pq::detail::selected<OutT, LutT> s, \
rmm::cuda_stream_view stream, \
uint32_t dim, \
uint32_t n_probes, \
uint32_t pq_dim, \
uint32_t n_queries, \
uint32_t queries_offset, \
cuvs::distance::DistanceType metric, \
cuvs::neighbors::ivf_pq::codebook_gen codebook_kind, \
uint32_t topk, \
uint32_t max_samples, \
const float* cluster_centers, \
const float* pq_centers, \
const uint8_t* const* pq_dataset, \
const uint32_t* cluster_labels, \
const uint32_t* _chunk_indices, \
const float* queries, \
const uint32_t* index_list, \
float* query_kths, \
const cuvs::neighbors::filtering::base_filter& sample_filter, \
LutT* lut_scores, \
OutT* _out_scores, \
uint32_t* _out_indices);

#define COMMA ,
instantiate_cuvs_neighbors_ivf_pq_detail_compute_similarity_select(
float,
float,
cuvs::neighbors::filtering::ivf_to_sample_filter<
int64_t COMMA cuvs::neighbors::filtering::none_sample_filter>);
instantiate_cuvs_neighbors_ivf_pq_detail_compute_similarity_select(float, float);
73 changes: 0 additions & 73 deletions cpp/src/neighbors/ivf_pq/detail/ivf_pq_compute_similarity_float_float_bitset64.cu
View file
Open in desktop

This file was deleted.

Loading