diff --git a/faiss/IVFlib.cpp b/faiss/IVFlib.cpp index 8af652a103..6834a54b7f 100644 --- a/faiss/IVFlib.cpp +++ b/faiss/IVFlib.cpp @@ -326,14 +326,14 @@ void search_with_parameters( double* ms_per_stage) { FAISS_THROW_IF_NOT(params); const float* prev_x = x; - ScopeDeleter del; + std::unique_ptr del; double t0 = getmillisecs(); if (auto ip = dynamic_cast(index)) { x = ip->apply_chain(n, x); if (x != prev_x) { - del.set(x); + del.reset(x); } index = ip->index; } @@ -376,14 +376,14 @@ void range_search_with_parameters( double* ms_per_stage) { FAISS_THROW_IF_NOT(params); const float* prev_x = x; - ScopeDeleter del; + std::unique_ptr del; double t0 = getmillisecs(); if (auto ip = dynamic_cast(index)) { x = ip->apply_chain(n, x); if (x != prev_x) { - del.set(x); + del.reset(x); } index = ip->index; } diff --git a/faiss/Index2Layer.cpp b/faiss/Index2Layer.cpp index 9d1c2ccde6..743dac6cea 100644 --- a/faiss/Index2Layer.cpp +++ b/faiss/Index2Layer.cpp @@ -83,7 +83,7 @@ void Index2Layer::train(idx_t n, const float* x) { verbose, pq.cp.seed); - ScopeDeleter del_x(x_in == x ? nullptr : x); + std::unique_ptr del_x(x_in == x ? nullptr : x); std::vector assign(n); // assignement to coarse centroids q1.quantizer->assign(n, x, assign.data()); diff --git a/faiss/IndexHNSW.cpp b/faiss/IndexHNSW.cpp index f846223479..a9ebf27866 100644 --- a/faiss/IndexHNSW.cpp +++ b/faiss/IndexHNSW.cpp @@ -192,9 +192,8 @@ void hnsw_add_vertices( { VisitedTable vt(ntotal); - DistanceComputer* dis = - storage_distance_computer(index_hnsw.storage); - ScopeDeleter1 del(dis); + std::unique_ptr dis( + storage_distance_computer(index_hnsw.storage)); int prev_display = verbose && omp_get_thread_num() == 0 ? 0 : -1; size_t counter = 0; @@ -301,8 +300,8 @@ void IndexHNSW::search( { VisitedTable vt(ntotal); - DistanceComputer* dis = storage_distance_computer(storage); - ScopeDeleter1 del(dis); + std::unique_ptr dis( + storage_distance_computer(storage)); #pragma omp for reduction(+ : n1, n2, n3, ndis, nreorder) schedule(guided) for (idx_t i = i0; i < i1; i++) { @@ -373,8 +372,8 @@ void IndexHNSW::reconstruct(idx_t key, float* recons) const { void IndexHNSW::shrink_level_0_neighbors(int new_size) { #pragma omp parallel { - DistanceComputer* dis = storage_distance_computer(storage); - ScopeDeleter1 del(dis); + std::unique_ptr dis( + storage_distance_computer(storage)); #pragma omp for for (idx_t i = 0; i < ntotal; i++) { @@ -507,8 +506,8 @@ void IndexHNSW::init_level_0_from_entry_points( { VisitedTable vt(ntotal); - DistanceComputer* dis = storage_distance_computer(storage); - ScopeDeleter1 del(dis); + std::unique_ptr dis( + storage_distance_computer(storage)); std::vector vec(storage->d); #pragma omp for schedule(dynamic) @@ -543,8 +542,8 @@ void IndexHNSW::reorder_links() { std::vector distances(M); std::vector order(M); std::vector tmp(M); - DistanceComputer* dis = storage_distance_computer(storage); - ScopeDeleter1 del(dis); + std::unique_ptr dis( + storage_distance_computer(storage)); #pragma omp for for (storage_idx_t i = 0; i < ntotal; i++) { @@ -795,12 +794,11 @@ void ReconstructFromNeighbors::estimate_code( storage_idx_t i, uint8_t* code) const { // fill in tmp table with the neighbor values - float* tmp1 = new float[d * (M + 1) + (d * k)]; - float* tmp2 = tmp1 + d * (M + 1); - ScopeDeleter del(tmp1); + std::unique_ptr tmp1(new float[d * (M + 1) + (d * k)]); + float* tmp2 = tmp1.get() + d * (M + 1); // collect coordinates of base - get_neighbor_table(i, tmp1); + get_neighbor_table(i, tmp1.get()); for (size_t sq = 0; sq < nsq; sq++) { int d0 = sq * dsub; @@ -816,7 +814,7 @@ void ReconstructFromNeighbors::estimate_code( &ki, &m1, &one, - tmp1 + d0, + tmp1.get() + d0, &di, codebook.data() + sq * (m1 * k), &m1, @@ -1033,8 +1031,8 @@ void IndexHNSW2Level::search( #pragma omp parallel { VisitedTable vt(ntotal); - DistanceComputer* dis = storage_distance_computer(storage); - ScopeDeleter1 del(dis); + std::unique_ptr dis( + storage_distance_computer(storage)); int candidates_size = hnsw.upper_beam; MinimaxHeap candidates(candidates_size); diff --git a/faiss/IndexIVF.cpp b/faiss/IndexIVF.cpp index c7780575e6..4e0f464811 100644 --- a/faiss/IndexIVF.cpp +++ b/faiss/IndexIVF.cpp @@ -447,9 +447,8 @@ void IndexIVF::search_preassigned( #pragma omp parallel if (do_parallel) reduction(+ : nlistv, ndis, nheap) { - InvertedListScanner* scanner = - get_InvertedListScanner(store_pairs, sel); - ScopeDeleter1 del(scanner); + std::unique_ptr scanner( + get_InvertedListScanner(store_pairs, sel)); /***************************************************** * Depending on parallel_mode, there are two possible ways diff --git a/faiss/IndexIVFPQ.cpp b/faiss/IndexIVFPQ.cpp index 85a3671e26..4a0ec74333 100644 --- a/faiss/IndexIVFPQ.cpp +++ b/faiss/IndexIVFPQ.cpp @@ -139,20 +139,20 @@ void IndexIVFPQ::add_core( add_core_o(n, x, xids, nullptr, coarse_idx); } -static float* compute_residuals( +static std::unique_ptr compute_residuals( const Index* quantizer, idx_t n, const float* x, const idx_t* list_nos) { size_t d = quantizer->d; - float* residuals = new float[n * d]; + std::unique_ptr residuals(new float[n * d]); // TODO: parallelize? for (size_t i = 0; i < n; i++) { if (list_nos[i] < 0) - memset(residuals + i * d, 0, sizeof(*residuals) * d); + memset(residuals.get() + i * d, 0, sizeof(float) * d); else quantizer->compute_residual( - x + i * d, residuals + i * d, list_nos[i]); + x + i * d, residuals.get() + i * d, list_nos[i]); } return residuals; } @@ -164,9 +164,9 @@ void IndexIVFPQ::encode_vectors( uint8_t* codes, bool include_listnos) const { if (by_residual) { - float* to_encode = compute_residuals(quantizer, n, x, list_nos); - ScopeDeleter del(to_encode); - pq.compute_codes(to_encode, codes, n); + std::unique_ptr to_encode = + compute_residuals(quantizer, n, x, list_nos); + pq.compute_codes(to_encode.get(), codes, n); } else { pq.compute_codes(x, codes, n); } @@ -241,31 +241,30 @@ void IndexIVFPQ::add_core_o( FAISS_THROW_IF_NOT(is_trained); double t0 = getmillisecs(); const idx_t* idx; - ScopeDeleter del_idx; + std::unique_ptr del_idx; if (precomputed_idx) { idx = precomputed_idx; } else { idx_t* idx0 = new idx_t[n]; - del_idx.set(idx0); + del_idx.reset(idx0); quantizer->assign(n, x, idx0); idx = idx0; } double t1 = getmillisecs(); - uint8_t* xcodes = new uint8_t[n * code_size]; - ScopeDeleter del_xcodes(xcodes); + std::unique_ptr xcodes(new uint8_t[n * code_size]); const float* to_encode = nullptr; - ScopeDeleter del_to_encode; + std::unique_ptr del_to_encode; if (by_residual) { - to_encode = compute_residuals(quantizer, n, x, idx); - del_to_encode.set(to_encode); + del_to_encode = compute_residuals(quantizer, n, x, idx); + to_encode = del_to_encode.get(); } else { to_encode = x; } - pq.compute_codes(to_encode, xcodes, n); + pq.compute_codes(to_encode, xcodes.get(), n); double t2 = getmillisecs(); // TODO: parallelize? @@ -281,7 +280,7 @@ void IndexIVFPQ::add_core_o( continue; } - uint8_t* code = xcodes + i * code_size; + uint8_t* code = xcodes.get() + i * code_size; size_t offset = invlists->add_entry(key, id, code); if (residuals_2) { diff --git a/faiss/IndexIVFPQR.cpp b/faiss/IndexIVFPQR.cpp index 5a5b88d94d..2dd967e829 100644 --- a/faiss/IndexIVFPQR.cpp +++ b/faiss/IndexIVFPQR.cpp @@ -92,17 +92,16 @@ void IndexIVFPQR::add_core( const float* x, const idx_t* xids, const idx_t* precomputed_idx) { - float* residual_2 = new float[n * d]; - ScopeDeleter del(residual_2); + std::unique_ptr residual_2(new float[n * d]); idx_t n0 = ntotal; - add_core_o(n, x, xids, residual_2, precomputed_idx); + add_core_o(n, x, xids, residual_2.get(), precomputed_idx); refine_codes.resize(ntotal * refine_pq.code_size); refine_pq.compute_codes( - residual_2, &refine_codes[n0 * refine_pq.code_size], n); + residual_2.get(), &refine_codes[n0 * refine_pq.code_size], n); } #define TIC t0 = get_cycles() #define TOC get_cycles() - t0 @@ -121,11 +120,10 @@ void IndexIVFPQR::search_preassigned( uint64_t t0; TIC; size_t k_coarse = long(k * k_factor); - idx_t* coarse_labels = new idx_t[k_coarse * n]; - ScopeDeleter del1(coarse_labels); - { // query with quantizer levels 1 and 2. - float* coarse_distances = new float[k_coarse * n]; - ScopeDeleter del(coarse_distances); + std::unique_ptr coarse_labels(new idx_t[k_coarse * n]); + { + // query with quantizer levels 1 and 2. + std::unique_ptr coarse_distances(new float[k_coarse * n]); IndexIVFPQ::search_preassigned( n, @@ -133,8 +131,8 @@ void IndexIVFPQR::search_preassigned( k_coarse, idx, L1_dis, - coarse_distances, - coarse_labels, + coarse_distances.get(), + coarse_labels.get(), true, params); } @@ -148,13 +146,12 @@ void IndexIVFPQR::search_preassigned( #pragma omp parallel reduction(+ : n_refine) { // tmp buffers - float* residual_1 = new float[2 * d]; - ScopeDeleter del(residual_1); - float* residual_2 = residual_1 + d; + std::unique_ptr residual_1(new float[2 * d]); + float* residual_2 = residual_1.get() + d; #pragma omp for for (idx_t i = 0; i < n; i++) { const float* xq = x + i * d; - const idx_t* shortlist = coarse_labels + k_coarse * i; + const idx_t* shortlist = coarse_labels.get() + k_coarse * i; float* heap_sim = distances + k * i; idx_t* heap_ids = labels + k * i; maxheap_heapify(k, heap_sim, heap_ids); @@ -172,7 +169,7 @@ void IndexIVFPQR::search_preassigned( assert(ofs >= 0 && ofs < invlists->list_size(list_no)); // 1st level residual - quantizer->compute_residual(xq, residual_1, list_no); + quantizer->compute_residual(xq, residual_1.get(), list_no); // 2nd level residual const uint8_t* l2code = invlists->get_single_code(list_no, ofs); @@ -185,9 +182,10 @@ void IndexIVFPQR::search_preassigned( idx_t id = invlists->get_single_id(list_no, ofs); assert(0 <= id && id < ntotal); refine_pq.decode( - &refine_codes[id * refine_pq.code_size], residual_1); + &refine_codes[id * refine_pq.code_size], + residual_1.get()); - float dis = fvec_L2sqr(residual_1, residual_2, d); + float dis = fvec_L2sqr(residual_1.get(), residual_2, d); if (dis < heap_sim[0]) { idx_t id_or_pair = store_pairs ? sl : id; diff --git a/faiss/IndexLSH.cpp b/faiss/IndexLSH.cpp index 3399a8f2d8..c2cbcb8b46 100644 --- a/faiss/IndexLSH.cpp +++ b/faiss/IndexLSH.cpp @@ -11,6 +11,7 @@ #include #include +#include #include #include @@ -75,18 +76,17 @@ void IndexLSH::train(idx_t n, const float* x) { thresholds.resize(nbits); train_thresholds = false; const float* xt = apply_preprocess(n, x); - ScopeDeleter del(xt == x ? nullptr : xt); + std::unique_ptr del(xt == x ? nullptr : xt); train_thresholds = true; - float* transposed_x = new float[n * nbits]; - ScopeDeleter del2(transposed_x); + std::unique_ptr transposed_x(new float[n * nbits]); for (idx_t i = 0; i < n; i++) for (idx_t j = 0; j < nbits; j++) transposed_x[j * n + i] = xt[i * nbits + j]; for (idx_t i = 0; i < nbits; i++) { - float* xi = transposed_x + i * n; + float* xi = transposed_x.get() + i * n; // std::nth_element std::sort(xi, xi + n); if (n % 2 == 1) @@ -110,19 +110,17 @@ void IndexLSH::search( FAISS_THROW_IF_NOT(k > 0); FAISS_THROW_IF_NOT(is_trained); const float* xt = apply_preprocess(n, x); - ScopeDeleter del(xt == x ? nullptr : xt); + std::unique_ptr del(xt == x ? nullptr : xt); - uint8_t* qcodes = new uint8_t[n * code_size]; - ScopeDeleter del2(qcodes); + std::unique_ptr qcodes(new uint8_t[n * code_size]); - fvecs2bitvecs(xt, qcodes, nbits, n); + fvecs2bitvecs(xt, qcodes.get(), nbits, n); - int* idistances = new int[n * k]; - ScopeDeleter del3(idistances); + std::unique_ptr idistances(new int[n * k]); - int_maxheap_array_t res = {size_t(n), size_t(k), labels, idistances}; + int_maxheap_array_t res = {size_t(n), size_t(k), labels, idistances.get()}; - hammings_knn_hc(&res, qcodes, codes.data(), ntotal, code_size, true); + hammings_knn_hc(&res, qcodes.get(), codes.data(), ntotal, code_size, true); // convert distances to floats for (int i = 0; i < k * n; i++) @@ -146,16 +144,16 @@ void IndexLSH::transfer_thresholds(LinearTransform* vt) { void IndexLSH::sa_encode(idx_t n, const float* x, uint8_t* bytes) const { FAISS_THROW_IF_NOT(is_trained); const float* xt = apply_preprocess(n, x); - ScopeDeleter del(xt == x ? nullptr : xt); + std::unique_ptr del(xt == x ? nullptr : xt); fvecs2bitvecs(xt, bytes, nbits, n); } void IndexLSH::sa_decode(idx_t n, const uint8_t* bytes, float* x) const { float* xt = x; - ScopeDeleter del; + std::unique_ptr del; if (rotate_data || nbits != d) { xt = new float[n * nbits]; - del.set(xt); + del.reset(xt); } bitvecs2fvecs(bytes, xt, nbits, n); diff --git a/faiss/IndexNNDescent.cpp b/faiss/IndexNNDescent.cpp index 8cdc0c4ab1..27bd6e33ee 100644 --- a/faiss/IndexNNDescent.cpp +++ b/faiss/IndexNNDescent.cpp @@ -158,8 +158,8 @@ void IndexNNDescent::search( { VisitedTable vt(ntotal); - DistanceComputer* dis = storage_distance_computer(storage); - ScopeDeleter1 del(dis); + std::unique_ptr dis( + storage_distance_computer(storage)); #pragma omp for for (idx_t i = i0; i < i1; i++) { @@ -197,8 +197,7 @@ void IndexNNDescent::add(idx_t n, const float* x) { storage->add(n, x); ntotal = storage->ntotal; - DistanceComputer* dis = storage_distance_computer(storage); - ScopeDeleter1 del(dis); + std::unique_ptr dis(storage_distance_computer(storage)); nndescent.build(*dis, ntotal, verbose); } diff --git a/faiss/IndexNSG.cpp b/faiss/IndexNSG.cpp index 9652266c43..2d1c3d820f 100644 --- a/faiss/IndexNSG.cpp +++ b/faiss/IndexNSG.cpp @@ -79,8 +79,8 @@ void IndexNSG::search( { VisitedTable vt(ntotal); - DistanceComputer* dis = storage_distance_computer(storage); - ScopeDeleter1 del(dis); + std::unique_ptr dis( + storage_distance_computer(storage)); #pragma omp for for (idx_t i = i0; i < i1; i++) { diff --git a/faiss/IndexPQ.cpp b/faiss/IndexPQ.cpp index bdcee56300..71cb855ef5 100644 --- a/faiss/IndexPQ.cpp +++ b/faiss/IndexPQ.cpp @@ -14,6 +14,7 @@ #include #include +#include #include #include @@ -198,17 +199,16 @@ void IndexPQ::search( } else { // code-to-code distances - uint8_t* q_codes = new uint8_t[n * pq.code_size]; - ScopeDeleter del(q_codes); + std::unique_ptr q_codes(new uint8_t[n * pq.code_size]); if (!encode_signs) { - pq.compute_codes(x, q_codes, n); + pq.compute_codes(x, q_codes.get(), n); } else { FAISS_THROW_IF_NOT(d == pq.nbits * pq.M); - memset(q_codes, 0, n * pq.code_size); + memset(q_codes.get(), 0, n * pq.code_size); for (size_t i = 0; i < n; i++) { const float* xi = x + i * d; - uint8_t* code = q_codes + i * pq.code_size; + uint8_t* code = q_codes.get() + i * pq.code_size; for (int j = 0; j < d; j++) if (xi[j] > 0) code[j >> 3] |= 1 << (j & 7); @@ -219,19 +219,18 @@ void IndexPQ::search( float_maxheap_array_t res = { size_t(n), size_t(k), labels, distances}; - pq.search_sdc(q_codes, n, codes.data(), ntotal, &res, true); + pq.search_sdc(q_codes.get(), n, codes.data(), ntotal, &res, true); } else { - int* idistances = new int[n * k]; - ScopeDeleter del(idistances); + std::unique_ptr idistances(new int[n * k]); int_maxheap_array_t res = { - size_t(n), size_t(k), labels, idistances}; + size_t(n), size_t(k), labels, idistances.get()}; if (search_type == ST_HE) { hammings_knn_hc( &res, - q_codes, + q_codes.get(), codes.data(), ntotal, pq.code_size, @@ -240,7 +239,7 @@ void IndexPQ::search( } else if (search_type == ST_generalized_HE) { generalized_hammings_knn_hc( &res, - q_codes, + q_codes.get(), codes.data(), ntotal, pq.code_size, @@ -333,22 +332,20 @@ void IndexPQ::search_core_polysemous( } // PQ distance tables - float* dis_tables = new float[n * pq.ksub * pq.M]; - ScopeDeleter del(dis_tables); - pq.compute_distance_tables(n, x, dis_tables); + std::unique_ptr dis_tables(new float[n * pq.ksub * pq.M]); + pq.compute_distance_tables(n, x, dis_tables.get()); // Hamming embedding queries - uint8_t* q_codes = new uint8_t[n * pq.code_size]; - ScopeDeleter del2(q_codes); + std::unique_ptr q_codes(new uint8_t[n * pq.code_size]); if (false) { - pq.compute_codes(x, q_codes, n); + pq.compute_codes(x, q_codes.get(), n); } else { #pragma omp parallel for for (idx_t qi = 0; qi < n; qi++) { pq.compute_code_from_distance_table( - dis_tables + qi * pq.M * pq.ksub, - q_codes + qi * pq.code_size); + dis_tables.get() + qi * pq.M * pq.ksub, + q_codes.get() + qi * pq.code_size); } } @@ -358,9 +355,9 @@ void IndexPQ::search_core_polysemous( #pragma omp parallel for reduction(+ : n_pass, bad_code_size) for (idx_t qi = 0; qi < n; qi++) { - const uint8_t* q_code = q_codes + qi * pq.code_size; + const uint8_t* q_code = q_codes.get() + qi * pq.code_size; - const float* dis_table_qi = dis_tables + qi * pq.M * pq.ksub; + const float* dis_table_qi = dis_tables.get() + qi * pq.M * pq.ksub; int64_t* heap_ids = labels + qi * k; float* heap_dis = distances + qi * k; @@ -441,12 +438,11 @@ void IndexPQ::sa_decode(idx_t n, const uint8_t* bytes, float* x) const { void IndexPQ::hamming_distance_table(idx_t n, const float* x, int32_t* dis) const { - uint8_t* q_codes = new uint8_t[n * pq.code_size]; - ScopeDeleter del(q_codes); + std::unique_ptr q_codes(new uint8_t[n * pq.code_size]); - pq.compute_codes(x, q_codes, n); + pq.compute_codes(x, q_codes.get(), n); - hammings(q_codes, codes.data(), n, ntotal, pq.code_size, dis); + hammings(q_codes.get(), codes.data(), n, ntotal, pq.code_size, dis); } void IndexPQ::hamming_distance_histogram( @@ -460,16 +456,15 @@ void IndexPQ::hamming_distance_histogram( FAISS_THROW_IF_NOT(pq.nbits == 8); // Hamming embedding queries - uint8_t* q_codes = new uint8_t[n * pq.code_size]; - ScopeDeleter del(q_codes); - pq.compute_codes(x, q_codes, n); + std::unique_ptr q_codes(new uint8_t[n * pq.code_size]); + pq.compute_codes(x, q_codes.get(), n); uint8_t* b_codes; - ScopeDeleter del_b_codes; + std::unique_ptr del_b_codes; if (xb) { b_codes = new uint8_t[nb * pq.code_size]; - del_b_codes.set(b_codes); + del_b_codes.reset(b_codes); pq.compute_codes(xb, b_codes, nb); } else { nb = ntotal; @@ -482,8 +477,7 @@ void IndexPQ::hamming_distance_histogram( #pragma omp parallel { std::vector histi(nbits + 1); - hamdis_t* distances = new hamdis_t[nb * bs]; - ScopeDeleter del(distances); + std::unique_ptr distances(new hamdis_t[nb * bs]); #pragma omp for for (idx_t q0 = 0; q0 < n; q0 += bs) { // printf ("dis stats: %zd/%zd\n", q0, n); @@ -492,12 +486,12 @@ void IndexPQ::hamming_distance_histogram( q1 = n; hammings( - q_codes + q0 * pq.code_size, + q_codes.get() + q0 * pq.code_size, b_codes, q1 - q0, nb, pq.code_size, - distances); + distances.get()); for (size_t i = 0; i < nb * (q1 - q0); i++) histi[distances[i]]++; @@ -914,17 +908,16 @@ void MultiIndexQuantizer::search( return; } - float* dis_tables = new float[n * pq.ksub * pq.M]; - ScopeDeleter del(dis_tables); + std::unique_ptr dis_tables(new float[n * pq.ksub * pq.M]); - pq.compute_distance_tables(n, x, dis_tables); + pq.compute_distance_tables(n, x, dis_tables.get()); if (k == 1) { // simple version that just finds the min in each table #pragma omp parallel for for (int i = 0; i < n; i++) { - const float* dis_table = dis_tables + i * pq.ksub * pq.M; + const float* dis_table = dis_tables.get() + i * pq.ksub * pq.M; float dis = 0; idx_t label = 0; @@ -954,7 +947,7 @@ void MultiIndexQuantizer::search( k, pq.M, pq.nbits, pq.ksub); #pragma omp for for (int i = 0; i < n; i++) { - msk.run(dis_tables + i * pq.ksub * pq.M, + msk.run(dis_tables.get() + i * pq.ksub * pq.M, pq.ksub, distances + i * k, labels + i * k); diff --git a/faiss/IndexPreTransform.cpp b/faiss/IndexPreTransform.cpp index cde857c8ed..8bc847a957 100644 --- a/faiss/IndexPreTransform.cpp +++ b/faiss/IndexPreTransform.cpp @@ -67,7 +67,7 @@ void IndexPreTransform::train(idx_t n, const float* x) { } } const float* prev_x = x; - ScopeDeleter del; + std::unique_ptr del; if (verbose) { printf("IndexPreTransform::train: training chain 0 to %d\n", @@ -102,10 +102,12 @@ void IndexPreTransform::train(idx_t n, const float* x) { float* xt = chain[i]->apply(n, prev_x); - if (prev_x != x) - delete[] prev_x; + if (prev_x != x) { + del.reset(); + } + prev_x = xt; - del.set(xt); + del.reset(xt); } is_trained = true; @@ -113,11 +115,11 @@ void IndexPreTransform::train(idx_t n, const float* x) { const float* IndexPreTransform::apply_chain(idx_t n, const float* x) const { const float* prev_x = x; - ScopeDeleter del; + std::unique_ptr del; for (int i = 0; i < chain.size(); i++) { float* xt = chain[i]->apply(n, prev_x); - ScopeDeleter del2(xt); + std::unique_ptr del2(xt); del2.swap(del); prev_x = xt; } @@ -128,11 +130,11 @@ const float* IndexPreTransform::apply_chain(idx_t n, const float* x) const { void IndexPreTransform::reverse_chain(idx_t n, const float* xt, float* x) const { const float* next_x = xt; - ScopeDeleter del; + std::unique_ptr del; for (int i = chain.size() - 1; i >= 0; i--) { float* prev_x = (i == 0) ? x : new float[n * chain[i]->d_in]; - ScopeDeleter del2((prev_x == x) ? nullptr : prev_x); + std::unique_ptr del2((prev_x == x) ? nullptr : prev_x); chain[i]->reverse_transform(n, next_x, prev_x); del2.swap(del); next_x = prev_x; @@ -176,7 +178,7 @@ void IndexPreTransform::search( FAISS_THROW_IF_NOT(k > 0); FAISS_THROW_IF_NOT(is_trained); const float* xt = apply_chain(n, x); - ScopeDeleter del(xt == x ? nullptr : xt); + std::unique_ptr del(xt == x ? nullptr : xt); index->search( n, xt, k, distances, labels, extract_index_search_params(params)); } @@ -206,7 +208,7 @@ size_t IndexPreTransform::remove_ids(const IDSelector& sel) { void IndexPreTransform::reconstruct(idx_t key, float* recons) const { float* x = chain.empty() ? recons : new float[index->d]; - ScopeDeleter del(recons == x ? nullptr : x); + std::unique_ptr del(recons == x ? nullptr : x); // Initial reconstruction index->reconstruct(key, x); @@ -216,7 +218,7 @@ void IndexPreTransform::reconstruct(idx_t key, float* recons) const { void IndexPreTransform::reconstruct_n(idx_t i0, idx_t ni, float* recons) const { float* x = chain.empty() ? recons : new float[ni * index->d]; - ScopeDeleter del(recons == x ? nullptr : x); + std::unique_ptr del(recons == x ? nullptr : x); // Initial reconstruction index->reconstruct_n(i0, ni, x); @@ -238,7 +240,8 @@ void IndexPreTransform::search_and_reconstruct( TransformedVectors trans(x, apply_chain(n, x)); float* recons_temp = chain.empty() ? recons : new float[n * k * index->d]; - ScopeDeleter del2((recons_temp == recons) ? nullptr : recons_temp); + std::unique_ptr del2( + (recons_temp == recons) ? nullptr : recons_temp); index->search_and_reconstruct( n, trans.x, diff --git a/faiss/IndexRefine.cpp b/faiss/IndexRefine.cpp index 9aade766a6..8fb0ea80bb 100644 --- a/faiss/IndexRefine.cpp +++ b/faiss/IndexRefine.cpp @@ -118,14 +118,14 @@ void IndexRefine::search( FAISS_THROW_IF_NOT(is_trained); idx_t* base_labels = labels; float* base_distances = distances; - ScopeDeleter del1; - ScopeDeleter del2; + std::unique_ptr del1; + std::unique_ptr del2; if (k != k_base) { base_labels = new idx_t[n * k_base]; - del1.set(base_labels); + del1.reset(base_labels); base_distances = new float[n * k_base]; - del2.set(base_distances); + del2.reset(base_distances); } base_index->search( @@ -262,14 +262,14 @@ void IndexRefineFlat::search( FAISS_THROW_IF_NOT(is_trained); idx_t* base_labels = labels; float* base_distances = distances; - ScopeDeleter del1; - ScopeDeleter del2; + std::unique_ptr del1; + std::unique_ptr del2; if (k != k_base) { base_labels = new idx_t[n * k_base]; - del1.set(base_labels); + del1.reset(base_labels); base_distances = new float[n * k_base]; - del2.set(base_distances); + del2.reset(base_distances); } base_index->search( diff --git a/faiss/IndexScalarQuantizer.cpp b/faiss/IndexScalarQuantizer.cpp index acd3592bf9..6b23315735 100644 --- a/faiss/IndexScalarQuantizer.cpp +++ b/faiss/IndexScalarQuantizer.cpp @@ -60,10 +60,9 @@ void IndexScalarQuantizer::search( #pragma omp parallel { - InvertedListScanner* scanner = - sq.select_InvertedListScanner(metric_type, nullptr, true, sel); + std::unique_ptr scanner( + sq.select_InvertedListScanner(metric_type, nullptr, true, sel)); - ScopeDeleter1 del(scanner); scanner->list_no = 0; // directly the list number #pragma omp for diff --git a/faiss/MetaIndexes.cpp b/faiss/MetaIndexes.cpp index d3dd5b0fc6..1adc88fb6f 100644 --- a/faiss/MetaIndexes.cpp +++ b/faiss/MetaIndexes.cpp @@ -70,37 +70,37 @@ void IndexSplitVectors::search( sum_d == d, "not enough indexes compared to # dimensions"); int64_t nshard = sub_indexes.size(); - float* all_distances = new float[nshard * k * n]; - idx_t* all_labels = new idx_t[nshard * k * n]; - ScopeDeleter del(all_distances); - ScopeDeleter del2(all_labels); - - auto query_func = [n, - x, - k, - distances, - labels, - all_distances, - all_labels, - this](int no) { - const IndexSplitVectors* index = this; - float* distances1 = no == 0 ? distances : all_distances + no * k * n; - idx_t* labels1 = no == 0 ? labels : all_labels + no * k * n; - if (index->verbose) - printf("begin query shard %d on %" PRId64 " points\n", no, n); - const Index* sub_index = index->sub_indexes[no]; - int64_t sub_d = sub_index->d, d = index->d; - idx_t ofs = 0; - for (int i = 0; i < no; i++) - ofs += index->sub_indexes[i]->d; - float* sub_x = new float[sub_d * n]; - ScopeDeleter del1(sub_x); - for (idx_t i = 0; i < n; i++) - memcpy(sub_x + i * sub_d, x + ofs + i * d, sub_d * sizeof(sub_x)); - sub_index->search(n, sub_x, k, distances1, labels1); - if (index->verbose) - printf("end query shard %d\n", no); - }; + + std::unique_ptr all_distances(new float[nshard * k * n]); + std::unique_ptr all_labels(new idx_t[nshard * k * n]); + + auto query_func = + [n, x, k, distances, labels, &all_distances, &all_labels, this]( + int no) { + const IndexSplitVectors* index = this; + float* distances1 = + no == 0 ? distances : all_distances.get() + no * k * n; + idx_t* labels1 = + no == 0 ? labels : all_labels.get() + no * k * n; + if (index->verbose) + printf("begin query shard %d on %" PRId64 " points\n", + no, + n); + const Index* sub_index = index->sub_indexes[no]; + int64_t sub_d = sub_index->d, d = index->d; + idx_t ofs = 0; + for (int i = 0; i < no; i++) + ofs += index->sub_indexes[i]->d; + + std::unique_ptr sub_x(new float[sub_d * n]); + for (idx_t i = 0; i < n; i++) + memcpy(sub_x.get() + i * sub_d, + x + ofs + i * d, + sub_d * sizeof(sub_x)); + sub_index->search(n, sub_x.get(), k, distances1, labels1); + if (index->verbose) + printf("end query shard %d\n", no); + }; if (!threaded) { for (int i = 0; i < nshard; i++) { @@ -125,8 +125,8 @@ void IndexSplitVectors::search( int64_t factor = 1; for (int i = 0; i < nshard; i++) { if (i > 0) { // results of 0 are already in the table - const float* distances_i = all_distances + i * k * n; - const idx_t* labels_i = all_labels + i * k * n; + const float* distances_i = all_distances.get() + i * k * n; + const idx_t* labels_i = all_labels.get() + i * k * n; for (int64_t j = 0; j < n; j++) { if (labels[j] >= 0 && labels_i[j] >= 0) { labels[j] += labels_i[j] * factor; diff --git a/faiss/gpu/GpuIndexIVFPQ.cu b/faiss/gpu/GpuIndexIVFPQ.cu index 8cb84c2ee0..69c4cf0556 100644 --- a/faiss/gpu/GpuIndexIVFPQ.cu +++ b/faiss/gpu/GpuIndexIVFPQ.cu @@ -282,7 +282,7 @@ void GpuIndexIVFPQ::trainResidualQuantizer_(idx_t n, const float* x) { verbose, pq.cp.seed); - ScopeDeleter del_x(x_in == x ? nullptr : x); + std::unique_ptr del_x(x_in == x ? nullptr : x); if (this->verbose) { printf("computing residuals\n"); diff --git a/faiss/impl/FaissException.h b/faiss/impl/FaissException.h index 5e5bcf1a30..b2774e8515 100644 --- a/faiss/impl/FaissException.h +++ b/faiss/impl/FaissException.h @@ -41,45 +41,6 @@ class FaissException : public std::exception { void handleExceptions( std::vector>& exceptions); -/** bare-bones unique_ptr - * this one deletes with delete [] */ -template -struct ScopeDeleter { - const T* ptr; - explicit ScopeDeleter(const T* ptr = nullptr) : ptr(ptr) {} - void release() { - ptr = nullptr; - } - void set(const T* ptr_in) { - ptr = ptr_in; - } - void swap(ScopeDeleter& other) { - std::swap(ptr, other.ptr); - } - ~ScopeDeleter() { - delete[] ptr; - } -}; - -/** same but deletes with the simple delete (least common case) */ -template -struct ScopeDeleter1 { - const T* ptr; - explicit ScopeDeleter1(const T* ptr = nullptr) : ptr(ptr) {} - void release() { - ptr = nullptr; - } - void set(const T* ptr_in) { - ptr = ptr_in; - } - void swap(ScopeDeleter1& other) { - std::swap(ptr, other.ptr); - } - ~ScopeDeleter1() { - delete ptr; - } -}; - /** RAII object for a set of possibly transformed vectors (deallocated only if * they are indeed transformed) */ diff --git a/faiss/impl/PolysemousTraining.cpp b/faiss/impl/PolysemousTraining.cpp index e524724113..1f01fc9dcf 100644 --- a/faiss/impl/PolysemousTraining.cpp +++ b/faiss/impl/PolysemousTraining.cpp @@ -12,11 +12,11 @@ #include #include +#include #include #include #include - -#include +#include #include #include @@ -882,14 +882,13 @@ void PolysemousTraining::optimize_ranking( double t0 = getmillisecs(); - PermutationObjective* obj = new RankingScore2( + std::unique_ptr obj(new RankingScore2( nbits, nq, nb, codes.data(), codes.data() + nq, - gt_distances.data()); - ScopeDeleter1 del(obj); + gt_distances.data())); if (verbose > 0) { printf(" m=%d, nq=%zd, nb=%zd, initialize RankingScore " @@ -900,7 +899,7 @@ void PolysemousTraining::optimize_ranking( getmillisecs() - t0); } - SimulatedAnnealingOptimizer optim(obj, *this); + SimulatedAnnealingOptimizer optim(obj.get(), *this); if (log_pattern.size()) { char fname[256]; diff --git a/faiss/impl/ProductQuantizer.cpp b/faiss/impl/ProductQuantizer.cpp index 342a6dc8d1..8ae033ca8f 100644 --- a/faiss/impl/ProductQuantizer.cpp +++ b/faiss/impl/ProductQuantizer.cpp @@ -135,11 +135,10 @@ void ProductQuantizer::train(size_t n, const float* x) { } } - float* xslice = new float[n * dsub]; - ScopeDeleter del(xslice); + std::unique_ptr xslice(new float[n * dsub]); for (int m = 0; m < M; m++) { for (int j = 0; j < n; j++) - memcpy(xslice + j * dsub, + memcpy(xslice.get() + j * dsub, x + j * d + m * dsub, dsub * sizeof(float)); @@ -153,11 +152,19 @@ void ProductQuantizer::train(size_t n, const float* x) { switch (final_train_type) { case Train_hypercube: init_hypercube( - dsub, nbits, n, xslice, clus.centroids.data()); + dsub, + nbits, + n, + xslice.get(), + clus.centroids.data()); break; case Train_hypercube_pca: init_hypercube_pca( - dsub, nbits, n, xslice, clus.centroids.data()); + dsub, + nbits, + n, + xslice.get(), + clus.centroids.data()); break; case Train_hot_start: memcpy(clus.centroids.data(), @@ -172,7 +179,7 @@ void ProductQuantizer::train(size_t n, const float* x) { printf("Training PQ slice %d/%zd\n", m, M); } IndexFlatL2 index(dsub); - clus.train(n, xslice, assign_index ? *assign_index : index); + clus.train(n, xslice.get(), assign_index ? *assign_index : index); set_params(clus.centroids.data(), m); } @@ -343,21 +350,20 @@ void ProductQuantizer::compute_codes_with_assign_index( assign_index->reset(); assign_index->add(ksub, get_centroids(m, 0)); size_t bs = 65536; - float* xslice = new float[bs * dsub]; - ScopeDeleter del(xslice); - idx_t* assign = new idx_t[bs]; - ScopeDeleter del2(assign); + + std::unique_ptr xslice(new float[bs * dsub]); + std::unique_ptr assign(new idx_t[bs]); for (size_t i0 = 0; i0 < n; i0 += bs) { size_t i1 = std::min(i0 + bs, n); for (size_t i = i0; i < i1; i++) { - memcpy(xslice + (i - i0) * dsub, + memcpy(xslice.get() + (i - i0) * dsub, x + i * d + m * dsub, dsub * sizeof(float)); } - assign_index->assign(i1 - i0, xslice, assign); + assign_index->assign(i1 - i0, xslice.get(), assign.get()); if (nbits == 8) { uint8_t* c = codes + code_size * i0 + m; @@ -407,14 +413,13 @@ void ProductQuantizer::compute_codes(const float* x, uint8_t* codes, size_t n) compute_code(x + i * d, codes + i * code_size); } else { // worthwhile to use BLAS - float* dis_tables = new float[n * ksub * M]; - ScopeDeleter del(dis_tables); - compute_distance_tables(n, x, dis_tables); + std::unique_ptr dis_tables(new float[n * ksub * M]); + compute_distance_tables(n, x, dis_tables.get()); #pragma omp parallel for for (int64_t i = 0; i < n; i++) { uint8_t* code = codes + i * code_size; - const float* tab = dis_tables + i * ksub * M; + const float* tab = dis_tables.get() + i * ksub * M; compute_code_from_distance_table(tab, code); } } diff --git a/faiss/impl/index_read.cpp b/faiss/impl/index_read.cpp index 4802e60ede..3253100369 100644 --- a/faiss/impl/index_read.cpp +++ b/faiss/impl/index_read.cpp @@ -436,7 +436,7 @@ ProductQuantizer* read_ProductQuantizer(const char* fname) { ProductQuantizer* read_ProductQuantizer(IOReader* reader) { ProductQuantizer* pq = new ProductQuantizer(); - ScopeDeleter1 del(pq); + std::unique_ptr del(pq); read_ProductQuantizer(pq, reader); del.release(); @@ -595,7 +595,7 @@ Index* read_index(IOReader* f, int io_flags) { READ1(idxp->encode_signs); READ1(idxp->polysemous_ht); } - // Old versoins of PQ all had metric_type set to INNER_PRODUCT + // Old versions of PQ all had metric_type set to INNER_PRODUCT // when they were in fact using L2. Therefore, we force metric type // to L2 when the old format is detected if (h == fourcc("IxPQ") || h == fourcc("IxPo")) { diff --git a/faiss/impl/platform_macros.h b/faiss/impl/platform_macros.h index 3315d0405e..5b38ff2a68 100644 --- a/faiss/impl/platform_macros.h +++ b/faiss/impl/platform_macros.h @@ -151,4 +151,11 @@ inline int __builtin_clzll(uint64_t x) { #define FAISS_PRAGMA_IMPRECISE_FUNCTION_END #endif +#if defined(_MSC_VER) && defined(_MSVC_LANG) +static_assert(_MSVC_LANG >= 201703L, "C++17 is expected"); +#else +// make sure that at least C++17 is used +static_assert(__cplusplus >= 201703L, "C++17 is expected"); +#endif + // clang-format on