From 636d95e8a4f635bd91a0ae8ff5ac629c32153205 Mon Sep 17 00:00:00 2001 From: Tarang Jain Date: Tue, 1 Apr 2025 10:28:02 -0700 Subject: [PATCH] Upgrade to libcuvs=25.04 (#4164) Summary: - [x] Upgrade cuVS version to 25.04 (nightly) - [x] Update install docs; deprecate faiss-gpu-raft - [x] CAGRA IVF-PQ Params as shared_ptr Pull Request resolved: https://github.com/facebookresearch/faiss/pull/4164 Reviewed By: bshethmeta, gtwang01 Differential Revision: D72194928 Pulled By: mnorris11 fbshipit-source-id: ef5143760bebc2fcb2a3dc20ddc26b5d02a5c21d --- .github/actions/build_cmake/action.yml | 4 ++-- CMakeLists.txt | 18 +++++++++--------- INSTALL.md | 8 +++----- cmake/thirdparty/fetch_rapids.cmake | 2 +- conda/faiss-gpu-cuvs/meta.yaml | 6 +++--- faiss/gpu/GpuIndexCagra.h | 4 ++-- 6 files changed, 20 insertions(+), 22 deletions(-) diff --git a/.github/actions/build_cmake/action.yml b/.github/actions/build_cmake/action.yml index 6251519fe..54f0a42f5 100644 --- a/.github/actions/build_cmake/action.yml +++ b/.github/actions/build_cmake/action.yml @@ -39,7 +39,7 @@ runs: conda update -y -q conda echo "$CONDA/bin" >> $GITHUB_PATH - conda install -y -q python=3.11 cmake=3.26 make=4.2 swig=4.0 "numpy<2" scipy=1.14 pytest=7.4 gflags=2.2 + conda install -y -q python=3.11 cmake=3.30.4 make=4.2 swig=4.0 "numpy<2" scipy=1.14 pytest=7.4 gflags=2.2 # install base packages for ARM64 if [ "${{ runner.arch }}" = "ARM64" ]; then @@ -61,7 +61,7 @@ runs: conda install -y -q cuda-toolkit=12.4 -c "nvidia/label/cuda-12.4.0" # and CUDA from cuVS channel for cuVS builds elif [ "${{ inputs.cuvs }}" = "ON" ]; then - conda install -y -q libcuvs=24.12 'cuda-version>=12.0,<=12.5' cuda-toolkit=12.4.1 gxx_linux-64=12.4 -c rapidsai -c conda-forge + conda install -y -q libcuvs=25.04 'cuda-version>=12.0,<=12.5' cuda-toolkit=12.4.1 gxx_linux-64=12.4 -c rapidsai -c rapidsai-nightly -c conda-forge fi # install test packages diff --git a/CMakeLists.txt b/CMakeLists.txt index 4a70aafab..cc071e6e8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -34,16 +34,16 @@ if(FAISS_ENABLE_GPU) endif() if(FAISS_ENABLE_CUVS) -include(cmake/thirdparty/fetch_rapids.cmake) -include(rapids-cmake) -include(rapids-cpm) -include(rapids-cuda) -include(rapids-export) -include(rapids-find) + include(cmake/thirdparty/fetch_rapids.cmake) + include(rapids-cmake) + include(rapids-cpm) + include(rapids-cuda) + include(rapids-export) + include(rapids-find) -rapids_cuda_init_architectures(faiss) -rapids_cuda_init_architectures(pyfaiss) -rapids_cuda_init_architectures(faiss_c_library) + rapids_cuda_init_architectures(faiss) + rapids_cuda_init_architectures(pyfaiss) + rapids_cuda_init_architectures(faiss_c_library) endif() project(faiss diff --git a/INSTALL.md b/INSTALL.md index ca491a6a6..26077224d 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -6,7 +6,7 @@ pre-release nightly builds. - The CPU-only faiss-cpu conda package is currently available on Linux (x86-64 and aarch64), OSX (arm64 only), and Windows (x86-64) - faiss-gpu, containing both CPU and GPU indices, is available on Linux (x86-64 only) for CUDA 11.4 and 12.1 -- faiss-gpu-cuvs [^1] package containing GPU indices provided by [NVIDIA cuVS](https://github.com/rapidsai/cuvs/) version 24.12, is available on Linux (x86-64 only) for CUDA 11.8 and 12.4. +- faiss-gpu-cuvs package containing GPU indices provided by [NVIDIA cuVS](https://github.com/rapidsai/cuvs/) version 24.12, is available on Linux (x86-64 only) for CUDA 11.8 and 12.4. To install the latest stable release: @@ -37,10 +37,10 @@ $ conda install -c pytorch/label/nightly faiss-cpu $ conda install -c pytorch/label/nightly -c nvidia faiss-gpu=1.10.0 # GPU(+CPU) version with NVIDIA cuVS (package built with CUDA 12.4) -conda install -c pytorch -c rapidsai -c conda-forge -c nvidia pytorch/label/nightly::faiss-gpu-cuvs 'cuda-version>=12.0,<=12.5' +conda install -c pytorch -c rapidsai -c rapidsai-nightly -c conda-forge -c nvidia pytorch/label/nightly::faiss-gpu-cuvs 'cuda-version>=12.0,<=12.5' # GPU(+CPU) version with NVIDIA cuVS (package built with CUDA 11.8) -conda install -c pytorch -c rapidsai -c conda-forge -c nvidia pytorch/label/nightly::faiss-gpu-cuvs 'cuda-version>=11.4,<=11.8' +conda install -c pytorch -c rapidsai -c rapidsai-nightly -c conda-forge -c nvidia pytorch/label/nightly::faiss-gpu-cuvs 'cuda-version>=11.4,<=11.8' # GPU(+CPU) version using AMD ROCm not yet available ``` @@ -321,5 +321,3 @@ and you can run $ python demos/demo_auto_tune.py ``` to test the GPU code. - -[^1]: The vector search and clustering algorithms in NVIDIA RAFT have been formally migrated to [NVIDIA cuVS](https://github.com/rapidsai/cuvs). This package is being renamed to `faiss-gpu-cuvs` in the next stable release, which will use these GPU implementations from the pre-compiled `libcuvs=24.12` binary. diff --git a/cmake/thirdparty/fetch_rapids.cmake b/cmake/thirdparty/fetch_rapids.cmake index 2ed6a78cf..ba0dac02c 100644 --- a/cmake/thirdparty/fetch_rapids.cmake +++ b/cmake/thirdparty/fetch_rapids.cmake @@ -15,7 +15,7 @@ # or implied. See the License for the specific language governing permissions and limitations under # the License. # ============================================================================= -set(RAPIDS_VERSION "24.12") +set(RAPIDS_VERSION "25.04") if(NOT EXISTS ${CMAKE_CURRENT_BINARY_DIR}/FAISS_RAPIDS.cmake) file(DOWNLOAD https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-${RAPIDS_VERSION}/RAPIDS.cmake diff --git a/conda/faiss-gpu-cuvs/meta.yaml b/conda/faiss-gpu-cuvs/meta.yaml index 76d535015..c4de80bc3 100644 --- a/conda/faiss-gpu-cuvs/meta.yaml +++ b/conda/faiss-gpu-cuvs/meta.yaml @@ -50,7 +50,7 @@ outputs: - {{ compiler('cxx') }} =12.4 - sysroot_linux-64 =2.17 # [linux64] - llvm-openmp # [osx] - - cmake >=3.26.4 + - cmake >=3.30.4 - make =4.2 # [not win] - _openmp_mutex =4.5=2_kmp_llvm # [x86_64] - mkl =2023 # [x86_64] @@ -66,7 +66,7 @@ outputs: - _openmp_mutex =4.5=2_kmp_llvm # [x86_64] - mkl =2023 # [x86_64] - openblas =0.3.29 # [not x86_64] - - libcuvs =24.12 + - libcuvs =25.04 - cuda-version {{ cuda_constraints }} run: - _openmp_mutex =4.5=2_kmp_llvm # [x86_64] @@ -74,7 +74,7 @@ outputs: - openblas =0.3.29 # [not x86_64] - cuda-cudart {{ cuda_constraints }} - libcublas {{ libcublas_constraints }} - - libcuvs =24.12 + - libcuvs =25.04 - cuda-version {{ cuda_constraints }} - libnvjitlink test: diff --git a/faiss/gpu/GpuIndexCagra.h b/faiss/gpu/GpuIndexCagra.h index c654e1b46..a0cac805a 100644 --- a/faiss/gpu/GpuIndexCagra.h +++ b/faiss/gpu/GpuIndexCagra.h @@ -173,8 +173,8 @@ struct GpuIndexCagraConfig : public GpuIndexConfig { /// Number of Iterations to run if building with NN_DESCENT size_t nn_descent_niter = 20; - IVFPQBuildCagraConfig* ivf_pq_params = nullptr; - IVFPQSearchCagraConfig* ivf_pq_search_params = nullptr; + std::shared_ptr ivf_pq_params{nullptr}; + std::shared_ptr ivf_pq_search_params{nullptr}; float refine_rate = 2.0f; bool store_dataset = true; };