Skip to content

Commit

Permalink
Fix IVFPQ ProductQuantizer::set_derived_values function bug. if nbits…
Browse files Browse the repository at this point in the history
… ==0 and then search. crash.

for example:

int d = 64; // dimension;
    int nb = 1; // database size
    int nq = 1; // nb of queries

    std::mt19937 rng;
    std::uniform_real_distribution<> distrib;

    float* xb = new float[d * nb];
    float* xq = new float[d * nq];

    for (int i = 0; i < nb; i++) {
        for (int j = 0; j < d; j++)
            xb[d * i + j] = distrib(rng);
        xb[d * i] += i / 1000.;
    }

    for (int i = 0; i < nq; i++) {
        for (int j = 0; j < d; j++)
            xq[d * i + j] = distrib(rng);
        xq[d * i] += i / 1000.;
    }

    int nlist = 1;
    int k = 1;
    int m = 8;
    size_t nbits_per_idx = 0;
    faiss::IndexFlatL2 quantizer(d);
    faiss::IndexIVFPQ index(&quantizer, d, nlist, m, nbits_per_idx);

    index.train(nb, xb);
    index.add(nb, xb);

    { // with param
        int query_count = 1;
        // sanity check
        idx_t* I = new idx_t[k * query_count];
        float* D = new float[k * query_count];

        index.search(query_count, xq, k, D, I);  // crash. Segmentation fault (core dumped)

        printf("I=\n");
        for (int i = 0; i < query_count; i++) {
            for (int j = 0; j < k; j++)
                printf("%5zd ", I[i * k + j]);
            printf("\n");
        }

        printf("D=\n");
        for (int i = 0; i < query_count; i++) {
            for (int j = 0; j < k; j++)
                printf("%7g ", D[i * k + j]);
            printf("\n");
        }

        delete[] I;
        delete[] D;
    }

./14-IVFPQ_crash
WARNING clustering 1 points to 1 centroids: please provide at least 39 training points
WARNING clustering 1 points to 1 centroids: please provide at least 39 training points
WARNING clustering 1 points to 1 centroids: please provide at least 39 training points
WARNING clustering 1 points to 1 centroids: please provide at least 39 training points
WARNING clustering 1 points to 1 centroids: please provide at least 39 training points
WARNING clustering 1 points to 1 centroids: please provide at least 39 training points
WARNING clustering 1 points to 1 centroids: please provide at least 39 training points
WARNING clustering 1 points to 1 centroids: please provide at least 39 training points
WARNING clustering 1 points to 1 centroids: please provide at least 39 training points
Segmentation fault (core dumped)

faiss/faiss/impl/ProductQuantizer-inl.h
 72│ inline uint64_t PQDecoderGeneric::decode() {
 73│     if (offset == 0) {
 74├───────> reg = *code;
 75│     }
 76│     uint64_t c = (reg >> offset);
 77│
 78│     if (offset + nbits >= 8) {
 79│         uint64_t e = 8 - offset;
 80│         ++code;
 81│         for (int i = 0; i < (nbits - (8 - offset)) / 8; ++i) {
 82│             c |= ((uint64_t)(*code++) << e);
 83│             e += 8;
 84│         }
 85│
 86│         offset += nbits;
 87│         offset &= 7;

 (gdb) bt
    ncode=1, codes=0x0, res=...) at /home/user/source2/faiss/faiss/IndexIVFPQ.cpp:908
    k=1) at /home/user/source2/faiss/faiss/IndexIVFPQ.cpp:1242
    at /home/user/source2/faiss/faiss/IndexIVF.cpp:448

faiss/faiss/IndexIVF.cpp
561├───────────────────> nheap += scanner->scan_codes(
 562│                             list_size, codes, ids, simi, idxi, k);  // codes is empty. not alloc any memmory.
 563│
 564│                     return list_size;
 565│                 }
 566│             } catch (const std::exception& e) {
 567│                 std::lock_guard<std::mutex> lock(exception_mutex);
 568│                 exception_string =
 569│                         demangle_cpp_symbol(typeid(e).name()) + "  " + e.what();
 570│                 interrupt = true;
 571│                 return size_t(0);
 572│             }
 573│         };
 574│

 facebookresearch#5  0x0000000000411d96 in operator() (__closure=0x7fffffffa5c0, key=0, coarse_dis_i=11.6475525, simi=0x584510, idxi=0x5844f0, list_size_max=9223372036854775807) at /home/user/source2/faiss/faiss/IndexIVF.cpp:561
  • Loading branch information
Haijun Yu committed Oct 25, 2023
1 parent c3b9374 commit 64b064b
Showing 1 changed file with 5 additions and 1 deletion.
6 changes: 5 additions & 1 deletion faiss/impl/ProductQuantizer.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,11 @@ void ProductQuantizer::set_derived_values() {
d % M == 0,
"The dimension of the vector (d) should be a multiple of the number of subquantizers (M)");
dsub = d / M;
code_size = (nbits * M + 7) / 8;
if (nbits > 0)
code_size = (nbits * M + 7) / 8;
else // nbits = 0
code_size = (1 * M + 7) / 8;

ksub = 1 << nbits;
centroids.resize(d * ksub);
verbose = false;
Expand Down

0 comments on commit 64b064b

Please sign in to comment.