From 0013c702f47bedbf6159ac356e61f378ccd12ac8 Mon Sep 17 00:00:00 2001 From: Gergely Szilvasy Date: Thu, 11 Jan 2024 08:26:33 -0800 Subject: [PATCH] avx512 CI + conda packages (#3197) Summary: Pull Request resolved: https://github.com/facebookresearch/faiss/pull/3197 Reviewed By: mlomeli1 Differential Revision: D52689379 Pulled By: algoriddle fbshipit-source-id: 54e27c6d310d6da14777ae10ae62f46e7076cacf --- .circleci/config.yml | 4 ++++ .gitignore | 1 + conda/faiss-gpu-raft/build-lib.sh | 6 +++--- conda/faiss-gpu-raft/build-pkg.sh | 6 +++--- conda/faiss-gpu-raft/test_cpu_dispatch.sh | 5 +++-- conda/faiss-gpu/build-lib.sh | 6 +++--- conda/faiss-gpu/build-pkg.sh | 6 +++--- conda/faiss-gpu/test_cpu_dispatch.sh | 5 +++-- conda/faiss/build-lib-osx.sh | 6 +++--- conda/faiss/build-lib.sh | 6 +++--- conda/faiss/build-pkg-arm64.sh | 2 +- conda/faiss/build-pkg-osx.sh | 6 +++--- conda/faiss/build-pkg.sh | 6 +++--- conda/faiss/test_cpu_dispatch.sh | 5 +++-- faiss/CMakeLists.txt | 2 +- faiss/utils/utils.cpp | 2 +- tests/test_index_accuracy.py | 4 ++-- 17 files changed, 43 insertions(+), 35 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9ddcb5ba83..406817e09d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -314,6 +314,10 @@ workflows: name: Linux x86_64 AVX2 (cmake) exec: linux-x86_64-cpu opt_level: "avx2" + - build_cmake: + name: Linux x86_64 AVX512 (cmake) + exec: linux-x86_64-cpu + opt_level: "avx512" - build_cmake: name: Linux x86_64 GPU (cmake) exec: linux-x86_64-gpu diff --git a/.gitignore b/.gitignore index 183e2066e1..caab1304c8 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,4 @@ /tests/test /tests/gtest/ faiss/python/swigfaiss_avx2.swig +faiss/python/swigfaiss_avx512.swig diff --git a/conda/faiss-gpu-raft/build-lib.sh b/conda/faiss-gpu-raft/build-lib.sh index 7ca17180a4..79ca8da2cd 100644 --- a/conda/faiss-gpu-raft/build-lib.sh +++ b/conda/faiss-gpu-raft/build-lib.sh @@ -7,11 +7,11 @@ set -e -# Build libfaiss.so/libfaiss_avx2.so. +# Build libfaiss.so/libfaiss_avx2.so/libfaiss_avx512.so cmake -B _build \ -DBUILD_SHARED_LIBS=ON \ -DBUILD_TESTING=OFF \ - -DFAISS_OPT_LEVEL=avx2 \ + -DFAISS_OPT_LEVEL=avx512 \ -DFAISS_ENABLE_GPU=ON \ -DFAISS_ENABLE_RAFT=ON \ -DCMAKE_CUDA_ARCHITECTURES="${CUDA_ARCHS}" \ @@ -20,7 +20,7 @@ cmake -B _build \ -DCMAKE_INSTALL_LIBDIR=lib \ -DCMAKE_BUILD_TYPE=Release . -make -C _build -j$(nproc) faiss faiss_avx2 +make -C _build -j$(nproc) faiss faiss_avx2 faiss_avx512 cmake --install _build --prefix $PREFIX cmake --install _build --prefix _libfaiss_stage/ diff --git a/conda/faiss-gpu-raft/build-pkg.sh b/conda/faiss-gpu-raft/build-pkg.sh index 3bb61588e5..da5fdefca9 100644 --- a/conda/faiss-gpu-raft/build-pkg.sh +++ b/conda/faiss-gpu-raft/build-pkg.sh @@ -7,17 +7,17 @@ set -e -# Build swigfaiss.so/swigfaiss_avx2.so. +# Build swigfaiss.so/swigfaiss_avx2.so/swigfaiss_avx512.so cmake -B _build_python_${PY_VER} \ -Dfaiss_ROOT=_libfaiss_stage/ \ - -DFAISS_OPT_LEVEL=avx2 \ + -DFAISS_OPT_LEVEL=avx512 \ -DFAISS_ENABLE_GPU=ON \ -DFAISS_ENABLE_RAFT=ON \ -DCMAKE_BUILD_TYPE=Release \ -DPython_EXECUTABLE=$PYTHON \ faiss/python -make -C _build_python_${PY_VER} -j$(nproc) swigfaiss swigfaiss_avx2 +make -C _build_python_${PY_VER} -j$(nproc) swigfaiss swigfaiss_avx2 swigfaiss_avx512 # Build actual python module. cd _build_python_${PY_VER}/ diff --git a/conda/faiss-gpu-raft/test_cpu_dispatch.sh b/conda/faiss-gpu-raft/test_cpu_dispatch.sh index b2891919d5..a7c1b2da72 100755 --- a/conda/faiss-gpu-raft/test_cpu_dispatch.sh +++ b/conda/faiss-gpu-raft/test_cpu_dispatch.sh @@ -6,5 +6,6 @@ set -e -FAISS_DISABLE_CPU_FEATURES=AVX2 LD_DEBUG=libs python -c "import faiss" 2>&1 | grep libfaiss.so -LD_DEBUG=libs python -c "import faiss" 2>&1 | grep libfaiss_avx2.so +FAISS_OPT_LEVEL= LD_DEBUG=libs python -c "import faiss" 2>&1 | grep libfaiss.so +FAISS_OPT_LEVEL=AVX2 LD_DEBUG=libs python -c "import faiss" 2>&1 | grep libfaiss_avx2.so +FAISS_OPT_LEVEL=AVX512 LD_DEBUG=libs python -c "import faiss" 2>&1 | grep libfaiss_avx512.so diff --git a/conda/faiss-gpu/build-lib.sh b/conda/faiss-gpu/build-lib.sh index 6b6b1c28d0..2d25e9c5e6 100755 --- a/conda/faiss-gpu/build-lib.sh +++ b/conda/faiss-gpu/build-lib.sh @@ -7,11 +7,11 @@ set -e -# Build libfaiss.so/libfaiss_avx2.so. +# Build libfaiss.so/libfaiss_avx2.so/libfaiss_avx512.so cmake -B _build \ -DBUILD_SHARED_LIBS=ON \ -DBUILD_TESTING=OFF \ - -DFAISS_OPT_LEVEL=avx2 \ + -DFAISS_OPT_LEVEL=avx512 \ -DFAISS_ENABLE_GPU=ON \ -DFAISS_ENABLE_RAFT=OFF \ -DCMAKE_CUDA_ARCHITECTURES="${CUDA_ARCHS}" \ @@ -20,7 +20,7 @@ cmake -B _build \ -DCMAKE_INSTALL_LIBDIR=lib \ -DCMAKE_BUILD_TYPE=Release . -make -C _build -j$(nproc) faiss faiss_avx2 +make -C _build -j$(nproc) faiss faiss_avx2 faiss_avx512 cmake --install _build --prefix $PREFIX cmake --install _build --prefix _libfaiss_stage/ diff --git a/conda/faiss-gpu/build-pkg.sh b/conda/faiss-gpu/build-pkg.sh index 3a41511921..e529a83d80 100755 --- a/conda/faiss-gpu/build-pkg.sh +++ b/conda/faiss-gpu/build-pkg.sh @@ -7,17 +7,17 @@ set -e -# Build swigfaiss.so/swigfaiss_avx2.so. +# Build swigfaiss.so/swigfaiss_avx2.so/swigfaiss_avx512.so cmake -B _build_python_${PY_VER} \ -Dfaiss_ROOT=_libfaiss_stage/ \ - -DFAISS_OPT_LEVEL=avx2 \ + -DFAISS_OPT_LEVEL=avx512 \ -DFAISS_ENABLE_GPU=ON \ -DFAISS_ENABLE_RAFT=OFF \ -DCMAKE_BUILD_TYPE=Release \ -DPython_EXECUTABLE=$PYTHON \ faiss/python -make -C _build_python_${PY_VER} -j$(nproc) swigfaiss swigfaiss_avx2 +make -C _build_python_${PY_VER} -j$(nproc) swigfaiss swigfaiss_avx2 swigfaiss_avx512 # Build actual python module. cd _build_python_${PY_VER}/ diff --git a/conda/faiss-gpu/test_cpu_dispatch.sh b/conda/faiss-gpu/test_cpu_dispatch.sh index b2891919d5..a7c1b2da72 100755 --- a/conda/faiss-gpu/test_cpu_dispatch.sh +++ b/conda/faiss-gpu/test_cpu_dispatch.sh @@ -6,5 +6,6 @@ set -e -FAISS_DISABLE_CPU_FEATURES=AVX2 LD_DEBUG=libs python -c "import faiss" 2>&1 | grep libfaiss.so -LD_DEBUG=libs python -c "import faiss" 2>&1 | grep libfaiss_avx2.so +FAISS_OPT_LEVEL= LD_DEBUG=libs python -c "import faiss" 2>&1 | grep libfaiss.so +FAISS_OPT_LEVEL=AVX2 LD_DEBUG=libs python -c "import faiss" 2>&1 | grep libfaiss_avx2.so +FAISS_OPT_LEVEL=AVX512 LD_DEBUG=libs python -c "import faiss" 2>&1 | grep libfaiss_avx512.so diff --git a/conda/faiss/build-lib-osx.sh b/conda/faiss/build-lib-osx.sh index a30de2d000..e858106bf2 100755 --- a/conda/faiss/build-lib-osx.sh +++ b/conda/faiss/build-lib-osx.sh @@ -7,11 +7,11 @@ set -e -# Build libfaiss.so/libfaiss_avx2.so. +# Build libfaiss.so/libfaiss_avx2.so/libfaiss_avx512.so cmake -B _build \ -DBUILD_SHARED_LIBS=ON \ -DBUILD_TESTING=OFF \ - -DFAISS_OPT_LEVEL=avx2 \ + -DFAISS_OPT_LEVEL=avx512 \ -DFAISS_ENABLE_GPU=OFF \ -DFAISS_ENABLE_PYTHON=OFF \ -DBLA_VENDOR=Intel10_64lp \ @@ -21,7 +21,7 @@ cmake -B _build \ -DCMAKE_INSTALL_LIBDIR=lib \ -DCMAKE_BUILD_TYPE=Release . -make -C _build -j$(nproc) faiss faiss_avx2 +make -C _build -j$(nproc) faiss faiss_avx2 faiss_avx512 cmake --install _build --prefix $PREFIX cmake --install _build --prefix _libfaiss_stage/ diff --git a/conda/faiss/build-lib.sh b/conda/faiss/build-lib.sh index 8aed84ba41..5028891d93 100755 --- a/conda/faiss/build-lib.sh +++ b/conda/faiss/build-lib.sh @@ -7,18 +7,18 @@ set -e -# Build libfaiss.so/libfaiss_avx2.so. +# Build libfaiss.so/libfaiss_avx2.so/libfaiss_avx512.so cmake -B _build \ -DBUILD_SHARED_LIBS=ON \ -DBUILD_TESTING=OFF \ - -DFAISS_OPT_LEVEL=avx2 \ + -DFAISS_OPT_LEVEL=avx512 \ -DFAISS_ENABLE_GPU=OFF \ -DFAISS_ENABLE_PYTHON=OFF \ -DBLA_VENDOR=Intel10_64lp \ -DCMAKE_INSTALL_LIBDIR=lib \ -DCMAKE_BUILD_TYPE=Release . -make -C _build -j$(nproc) faiss faiss_avx2 +make -C _build -j$(nproc) faiss faiss_avx2 faiss_avx512 cmake --install _build --prefix $PREFIX cmake --install _build --prefix _libfaiss_stage/ diff --git a/conda/faiss/build-pkg-arm64.sh b/conda/faiss/build-pkg-arm64.sh index c63380ab01..70fc7312e5 100755 --- a/conda/faiss/build-pkg-arm64.sh +++ b/conda/faiss/build-pkg-arm64.sh @@ -7,7 +7,7 @@ set -e -# Build swigfaiss.so/swigfaiss_avx2.so. +# Build swigfaiss.so cmake -B _build_python_${PY_VER} \ -Dfaiss_ROOT=_libfaiss_stage/ \ -DFAISS_ENABLE_GPU=OFF \ diff --git a/conda/faiss/build-pkg-osx.sh b/conda/faiss/build-pkg-osx.sh index 15016face9..914aed174d 100755 --- a/conda/faiss/build-pkg-osx.sh +++ b/conda/faiss/build-pkg-osx.sh @@ -7,10 +7,10 @@ set -e -# Build swigfaiss.so/swigfaiss_avx2.so. +# Build swigfaiss.so/swigfaiss_avx2.so/swigfaiss_avx512 cmake -B _build_python_${PY_VER} \ -Dfaiss_ROOT=_libfaiss_stage/ \ - -DFAISS_OPT_LEVEL=avx2 \ + -DFAISS_OPT_LEVEL=avx512 \ -DFAISS_ENABLE_GPU=OFF \ -DOpenMP_CXX_FLAGS=-fopenmp=libiomp5 \ -DOpenMP_CXX_LIB_NAMES=libiomp5 \ @@ -19,7 +19,7 @@ cmake -B _build_python_${PY_VER} \ -DPython_EXECUTABLE=$PYTHON \ faiss/python -make -C _build_python_${PY_VER} -j$(nproc) swigfaiss swigfaiss_avx2 +make -C _build_python_${PY_VER} -j$(nproc) swigfaiss swigfaiss_avx2 swigfaiss_avx512 # Build actual python module. cd _build_python_${PY_VER}/ diff --git a/conda/faiss/build-pkg.sh b/conda/faiss/build-pkg.sh index 005aec2fcc..92e0febfa8 100755 --- a/conda/faiss/build-pkg.sh +++ b/conda/faiss/build-pkg.sh @@ -7,16 +7,16 @@ set -e -# Build swigfaiss.so/swigfaiss_avx2.so. +# Build swigfaiss.so/swigfaiss_avx2.so/swigfaiss_avx512.so cmake -B _build_python_${PY_VER} \ -Dfaiss_ROOT=_libfaiss_stage/ \ - -DFAISS_OPT_LEVEL=avx2 \ + -DFAISS_OPT_LEVEL=avx512 \ -DFAISS_ENABLE_GPU=OFF \ -DCMAKE_BUILD_TYPE=Release \ -DPython_EXECUTABLE=$PYTHON \ faiss/python -make -C _build_python_${PY_VER} -j$(nproc) swigfaiss swigfaiss_avx2 +make -C _build_python_${PY_VER} -j$(nproc) swigfaiss swigfaiss_avx2 swigfaiss_avx512 # Build actual python module. cd _build_python_${PY_VER}/ diff --git a/conda/faiss/test_cpu_dispatch.sh b/conda/faiss/test_cpu_dispatch.sh index b2891919d5..a7c1b2da72 100755 --- a/conda/faiss/test_cpu_dispatch.sh +++ b/conda/faiss/test_cpu_dispatch.sh @@ -6,5 +6,6 @@ set -e -FAISS_DISABLE_CPU_FEATURES=AVX2 LD_DEBUG=libs python -c "import faiss" 2>&1 | grep libfaiss.so -LD_DEBUG=libs python -c "import faiss" 2>&1 | grep libfaiss_avx2.so +FAISS_OPT_LEVEL= LD_DEBUG=libs python -c "import faiss" 2>&1 | grep libfaiss.so +FAISS_OPT_LEVEL=AVX2 LD_DEBUG=libs python -c "import faiss" 2>&1 | grep libfaiss_avx2.so +FAISS_OPT_LEVEL=AVX512 LD_DEBUG=libs python -c "import faiss" 2>&1 | grep libfaiss_avx512.so diff --git a/faiss/CMakeLists.txt b/faiss/CMakeLists.txt index 27701586c8..db5133d68b 100644 --- a/faiss/CMakeLists.txt +++ b/faiss/CMakeLists.txt @@ -328,7 +328,7 @@ if(FAISS_OPT_LEVEL STREQUAL "avx2") ) endif() if(FAISS_OPT_LEVEL STREQUAL "avx512") - install(TARGETS faiss_avx512 + install(TARGETS faiss_avx2 faiss_avx512 EXPORT faiss-targets ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} diff --git a/faiss/utils/utils.cpp b/faiss/utils/utils.cpp index efbff502b0..c39a1bf5eb 100644 --- a/faiss/utils/utils.cpp +++ b/faiss/utils/utils.cpp @@ -117,7 +117,7 @@ std::string get_compile_options() { #ifdef __AVX2__ options += "AVX2 "; #elif __AVX512F__ - options += "AVX512"; + options += "AVX512 "; #elif defined(__aarch64__) options += "NEON "; #else diff --git a/tests/test_index_accuracy.py b/tests/test_index_accuracy.py index 44b4ca365f..3f7bfbd303 100644 --- a/tests/test_index_accuracy.py +++ b/tests/test_index_accuracy.py @@ -639,14 +639,14 @@ def test_OIVFPQ(self): d = ev.d quantizer = faiss.IndexFlatL2(d) index = faiss.IndexIVFPQ(quantizer, d, ncentroids, M, 8) - index.nprobe = 5 + index.nprobe = 12 res = ev.launch("IVFPQ", index) e_ivfpq = ev.evalres(res) quantizer = faiss.IndexFlatL2(d) index_ivfpq = faiss.IndexIVFPQ(quantizer, d, ncentroids, M, 8) - index_ivfpq.nprobe = 5 + index_ivfpq.nprobe = 12 opq_matrix = faiss.OPQMatrix(d, M) opq_matrix.niter = 10 index = faiss.IndexPreTransform(opq_matrix, index_ivfpq)