Merge branch 'main' into convert-circleci-to-github-actions

.circleci/config.yml

@@ -38,12 +38,17 @@ jobs:
  - run:
  name: Install clang-format
  command: |
- apt-get update
- apt-get install -y git-core clang-format-11
+ apt-get update -y
+ apt-get install -y wget
+ apt install -y lsb-release wget software-properties-common gnupg
+ wget https://apt.llvm.org/llvm.sh
+ chmod u+x llvm.sh
+ ./llvm.sh 18
+ apt-get install -y git-core clang-format-18
  - run:
  name: Verify clang-format
  command: |
- git ls-files | grep -E '\.(cpp|h|cu|cuh)$' | xargs clang-format-11 -i
+ git ls-files | grep -E '\.(cpp|h|cu|cuh)$' | xargs clang-format-18 -i
  if git diff --quiet; then
  echo "Formatting OK!"
  else
@@ -224,15 +229,15 @@ jobs:
  - run:
  name: Install env using main channel
  command: |
- conda install -y -q python=3.11 cmake make swig=4.0.2 mkl=2023 mkl-devel=2023 numpy scipy pytest gxx_linux-64 sysroot_linux-64
+ conda install -y -q python=3.11 cmake make swig mkl=2023 mkl-devel=2023 numpy scipy pytest gxx_linux-64=11.2 sysroot_linux-64
  - when:
  condition:
  equal: [ "ON", << parameters.raft >> ]
  steps:
  - run:
  name: Install env using conda-forge channel
  command: |
- conda install -y -q python=3.11 cmake make swig=4.0.2 mkl=2023 mkl-devel=2023 numpy scipy pytest gxx_linux-64 sysroot_linux-64=2.28 libraft cuda-version=11.8 cuda-toolkit -c rapidsai-nightly -c "nvidia/label/cuda-11.8.0" -c conda-forge
+ conda install -y -q python=3.11 cmake make swig mkl=2023 mkl-devel=2023 numpy scipy pytest gxx_linux-64=11.2 sysroot_linux-64=2.28 libraft cuda-version=11.8 cuda-toolkit -c rapidsai-nightly -c "nvidia/label/cuda-11.8.0" -c conda-forge
  - when:
  condition:
  and:

CHANGELOG.md

@@ -9,6 +9,8 @@ We try to indicate most contributions here with the contributor names who are no
 the Facebook Faiss team. Feel free to add entries here if you submit a PR.
 
 ## [Unreleased]
+### Changed
+- Previously, when moving indices to GPU with coarse quantizers that were not implemented on GPU, the cloner would silently fallback to CPU. This version will now throw an exception instead and the calling code would need to explicitly allow fallback to CPU by setting a flag in cloner config.
 
 ## [1.8.0] - 2024-02-27
 ### Added

CMakeLists.txt

@@ -18,7 +18,7 @@
 # the License.
 # =============================================================================
 
-cmake_minimum_required(VERSION 3.23.1 FATAL_ERROR)
+cmake_minimum_required(VERSION 3.24.0 FATAL_ERROR)
 
 set(FAISS_LANGUAGES CXX)
 

benchs/bench_fw/benchmark.py

@@ -208,9 +208,11 @@ def set_io(self, benchmark_io):
  self.io.distance_metric = self.distance_metric
  self.io.distance_metric_type = self.distance_metric_type
 
- def get_index_desc(self, factory: str) -> Optional[IndexDescriptor]:
+ def get_index_desc(self, factory_or_codec: str) -> Optional[IndexDescriptor]:
  for desc in self.index_descs:
- if desc.factory == factory:
+ if desc.factory == factory_or_codec:
+ return desc
+ if desc.codec_alias == factory_or_codec:
  return desc
  return None
 
@@ -232,7 +234,7 @@ def range_search_reference(self, index, parameters, range_metric):
  parameters,
  radius=m_radius,
  )
- flat = index.factory == "Flat"
+ flat = index.is_flat_index()
  (
  gt_radius,
  range_search_metric_function,
@@ -650,6 +652,7 @@ def benchmark(
  f"Range index {index_desc.factory} has no radius_score"
  )
  results["metrics"] = {}
+ self.build_index_wrapper(index_desc)
  for metric_key, range_metric in index_desc.range_metrics.items():
  (
  gt_radius,

benchs/bench_fw/descriptors.py

@@ -20,6 +20,7 @@ class IndexDescriptor:
  # but not both at the same time.
  path: Optional[str] = None
  factory: Optional[str] = None
+ codec_alias: Optional[str] = None
  construction_params: Optional[List[Dict[str, int]]] = None
  search_params: Optional[Dict[str, int]] = None
  # range metric definitions

benchs/bench_fw/index.py

@@ -495,7 +495,7 @@ def range_search(
  radius: Optional[float] = None,
  ):
  logger.info("range_search: begin")
- if search_parameters is not None and search_parameters["snap"] == 1:
+ if search_parameters is not None and search_parameters.get("snap") == 1:
  query_vectors = self.snap(query_vectors)
  filename = (
  self.get_range_search_name(
@@ -776,6 +776,9 @@ def add_range_or_val(name, range):
  )
  return op
 
+ def is_flat_index(self):
+ return self.get_index_name().startswith("Flat")
+
 
 # IndexFromCodec, IndexFromQuantizer and IndexFromPreTransform
 # are used to wrap pre-trained Faiss indices (codecs)
@@ -807,6 +810,9 @@ def get_codec_name(self):
  name += Index.param_dict_list_to_name(self.construction_params)
  return name
 
+ def fetch_meta(self, dry_run=False):
+ return None, None
+
  def fetch_codec(self):
  codec = self.io.read_index(
  os.path.basename(self.path),
@@ -911,7 +917,7 @@ def fetch_codec(self, dry_run=False):
  assert codec_size is not None
  meta = {
  "training_time": training_time,
- "training_size": self.training_vectors.num_vectors,
+ "training_size": self.training_vectors.num_vectors if self.training_vectors else 0,
  "codec_size": codec_size,
  "sa_code_size": self.get_sa_code_size(codec),
  "code_size": self.get_code_size(codec),

c_api/clone_index_c.cpp

@@ -14,6 +14,7 @@
 #include "macros_impl.h"
 
 using faiss::Index;
+using faiss::IndexBinary;
 
 int faiss_clone_index(const FaissIndex* idx, FaissIndex** p_out) {
  try {
@@ -22,3 +23,14 @@ int faiss_clone_index(const FaissIndex* idx, FaissIndex** p_out) {
  }
  CATCH_AND_HANDLE
 }
+
+int faiss_clone_index_binary(
+ const FaissIndexBinary* idx,
+ FaissIndexBinary** p_out) {
+ try {
+ auto out = faiss::clone_binary_index(
+ reinterpret_cast<const IndexBinary*>(idx));
+ *p_out = reinterpret_cast<FaissIndexBinary*>(out);
+ }
+ CATCH_AND_HANDLE
+}

c_api/clone_index_c.h

@@ -13,6 +13,7 @@
 #define FAISS_CLONE_INDEX_C_H
 
 #include <stdio.h>
+#include "IndexBinary_c.h"
 #include "Index_c.h"
 #include "faiss_c.h"
 
@@ -25,6 +26,9 @@ extern "C" {
 /** Clone an index. This is equivalent to `faiss::clone_index` */
 int faiss_clone_index(const FaissIndex*, FaissIndex** p_out);
 
+/** Clone a binary index. This is equivalent to `faiss::clone_index_binary` */
+int faiss_clone_index_binary(const FaissIndexBinary*, FaissIndexBinary** p_out);
+
 #ifdef __cplusplus
 }
 #endif

c_api/index_factory_c.cpp

@@ -15,7 +15,7 @@
 
 using faiss::Index;
 
-/** Build and index with the sequence of processing steps described in
+/** Build an index with the sequence of processing steps described in
  * the string.
  */
 int faiss_index_factory(
@@ -29,3 +29,17 @@ int faiss_index_factory(
  }
  CATCH_AND_HANDLE
 }
+
+/** Build an index with the sequence of processing steps described in
+ * the string.
+ */
+int faiss_index_binary_factory(
+ FaissIndexBinary** p_index,
+ int d,
+ const char* description) {
+ try {
+ *p_index = reinterpret_cast<FaissIndexBinary*>(
+ faiss::index_binary_factory(d, description));
+ }
+ CATCH_AND_HANDLE
+}

c_api/index_factory_c.h

@@ -11,14 +11,15 @@
 #ifndef FAISS_INDEX_FACTORY_C_H
 #define FAISS_INDEX_FACTORY_C_H
 
+#include "IndexBinary_c.h"
 #include "Index_c.h"
 #include "faiss_c.h"
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-/** Build and index with the sequence of processing steps described in
+/** Build an index with the sequence of processing steps described in
  * the string.
  */
 int faiss_index_factory(
@@ -27,6 +28,14 @@ int faiss_index_factory(
  const char* description,
  FaissMetricType metric);
 
+/** Build a binary index with the sequence of processing steps described in
+ * the string.
+ */
+int faiss_index_binary_factory(
+ FaissIndexBinary** p_index,
+ int d,
+ const char* description);
+
 #ifdef __cplusplus
 }
 #endif

conda/faiss-gpu-raft/meta.yaml

@@ -48,7 +48,7 @@ outputs:
  - {{ compiler('cxx') }}
  - sysroot_linux-64 # [linux64]
  - llvm-openmp # [osx]
- - cmake >=3.23.1
+ - cmake >=3.24.0
  - make # [not win]
  - mkl-devel =2023 # [x86_64]
  - cuda-toolkit {{ cudatoolkit }}
@@ -84,8 +84,8 @@ outputs:
  build:
  - {{ compiler('cxx') }}
  - sysroot_linux-64 =2.17 # [linux64]
- - swig =4.0.2
- - cmake >=3.23.1
+ - swig
+ - cmake >=3.24.0
  - make # [not win]
  host:
  - python {{ python }}

conda/faiss-gpu/meta.yaml

@@ -48,7 +48,7 @@ outputs:
  - {{ compiler('cxx') }}
  - sysroot_linux-64 # [linux64]
  - llvm-openmp # [osx]
- - cmake >=3.23.1
+ - cmake >=3.24.0
  - make # [not win]
  - mkl-devel =2023 # [x86_64]
  - cuda-toolkit {{ cudatoolkit }}
@@ -81,7 +81,7 @@ outputs:
  - {{ compiler('cxx') }}
  - sysroot_linux-64 =2.17 # [linux64]
  - swig
- - cmake >=3.23.1
+ - cmake >=3.24.0
  - make # [not win]
  host:
  - python {{ python }}

conda/faiss/meta.yaml

@@ -39,7 +39,7 @@ outputs:
  - {{ compiler('cxx') }}
  - sysroot_linux-64 # [linux64]
  - llvm-openmp # [osx]
- - cmake >=3.23.1
+ - cmake >=3.24.0
  - make # [not win]
  - mkl-devel =2023 # [x86_64]
  host:
@@ -69,7 +69,7 @@ outputs:
  - {{ compiler('cxx') }}
  - sysroot_linux-64 =2.17 # [linux64]
  - swig
- - cmake >=3.23.1
+ - cmake >=3.24.0
  - make # [not win]
  host:
  - python {{ python }}

contrib/ondisk.py

@@ -11,7 +11,7 @@
 
 
 def merge_ondisk(
- trained_index: faiss.Index, shard_fnames: List[str], ivfdata_fname: str
+ trained_index: faiss.Index, shard_fnames: List[str], ivfdata_fname: str, shift_ids=False
 ) -> None:
  """Add the contents of the indexes stored in shard_fnames into the index
  trained_index. The on-disk data is stored in ivfdata_fname"""
@@ -51,7 +51,7 @@ def merge_ondisk(
  ivf_vector.push_back(ivf)
 
  LOG.info("merge %d inverted lists " % ivf_vector.size())
- ntotal = invlists.merge_from(ivf_vector.data(), ivf_vector.size())
+ ntotal = invlists.merge_from_multiple(ivf_vector.data(), ivf_vector.size(), shift_ids)
 
  # now replace the inverted lists in the output index
  index.ntotal = index_ivf.ntotal = ntotal

contrib/vecs_io.py

@@ -3,6 +3,7 @@
 # This source code is licensed under the MIT license found in the
 # LICENSE file in the root directory of this source tree.
 
+import sys
 import numpy as np
 
 """
@@ -13,6 +14,8 @@
 
 def ivecs_read(fname):
  a = np.fromfile(fname, dtype='int32')
+ if sys.big_endian:
+ a.byteswap(inplace=True)
  d = a[0]
  return a.reshape(-1, d + 1)[:, 1:].copy()
 
@@ -22,6 +25,7 @@ def fvecs_read(fname):
 
 
 def ivecs_mmap(fname):
+ assert not sys.big_endian
  a = np.memmap(fname, dtype='int32', mode='r')
  d = a[0]
  return a.reshape(-1, d + 1)[:, 1:]
@@ -33,7 +37,11 @@ def fvecs_mmap(fname):
 
 def bvecs_mmap(fname):
  x = np.memmap(fname, dtype='uint8', mode='r')
- d = x[:4].view('int32')[0]
+ if sys.big_endian:
+ da = x[:4][::-1].copy()
+ d = da.view('int32')[0]
+ else:
+ d = x[:4].view('int32')[0]
  return x.reshape(-1, d + 4)[:, 4:]
 
 
@@ -42,6 +50,8 @@ def ivecs_write(fname, m):
  m1 = np.empty((n, d + 1), dtype='int32')
  m1[:, 0] = d
  m1[:, 1:] = m
+ if sys.big_endian:
+ m1.byteswap(inplace=True)
  m1.tofile(fname)
 
 

demos/demo_imi_pq.cpp

@@ -77,7 +77,6 @@ int main() {
  // the coarse quantizer should not be dealloced before the index
  // 4 = nb of bytes per code (d must be a multiple of this)
  // 8 = nb of bits per sub-code (almost always 8)
- faiss::MetricType metric = faiss::METRIC_L2; // can be METRIC_INNER_PRODUCT
  faiss::IndexIVFPQ index(
  &coarse_quantizer, d, ncentroids, bytes_per_code, 8);
  index.quantizer_trains_alone = true;

demos/rocksdb_ivf/RocksDBInvertedLists.cpp

@@ -101,7 +101,8 @@ void RocksDBInvertedLists::resize(size_t /*list_no*/, size_t /*new_size*/) {
 }
 
 InvertedListsIterator* RocksDBInvertedLists::get_iterator(
- size_t list_no) const {
+ size_t list_no,
+ void* inverted_list_context) const {
  return new RocksDBInvertedListsIterator(db_.get(), list_no, code_size);
 }
 

demos/rocksdb_ivf/RocksDBInvertedLists.h

@@ -49,7 +49,9 @@ struct RocksDBInvertedLists : faiss::InvertedLists {
 
  void resize(size_t list_no, size_t new_size) override;
 
- faiss::InvertedListsIterator* get_iterator(size_t list_no) const override;
+ faiss::InvertedListsIterator* get_iterator(
+ size_t list_no,
+ void* inverted_list_context) const override;
 
  private:
  std::unique_ptr<rocksdb::DB> db_;

faiss/CMakeLists.txt

@@ -290,7 +290,10 @@ if(WIN32)
  target_compile_definitions(faiss_avx512 PRIVATE FAISS_MAIN_LIB)
 endif()
 
-target_compile_definitions(faiss PRIVATE FINTEGER=int)
+string(FIND "${CMAKE_CXX_FLAGS}" "FINTEGER" finteger_idx)
+if (${finteger_idx} EQUAL -1)
+ target_compile_definitions(faiss PRIVATE FINTEGER=int)
+endif()
 target_compile_definitions(faiss_avx2 PRIVATE FINTEGER=int)
 target_compile_definitions(faiss_avx512 PRIVATE FINTEGER=int)