mirrors
/
faiss
mirror of https://github.com/facebookresearch/faiss.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Add comments for vector decode kernels about IVF+PQ with PQ bits > 8 (#2737) 

Summary:
Pull Request resolved: https://github.com/facebookresearch/faiss/pull/2737

IVFPQ with more than 8 bits per subquantizer seem to be acceptable in Faiss. So, comments were altered, additional unit tests were added.

Reviewed By: mdouze

Differential Revision: D43706459

fbshipit-source-id: 45d0cc6f43ec0198aa95d025f07b75a9c33e4db7
pull/2728/head^2
Alexandr Guzhva 2023-03-06 09:27:42 -08:00 committed by Facebook GitHub Bot
parent 0815970581
commit da09d22d5d
4 changed files with 45 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
faiss/cppcontrib/SaDecodeKernels.h
View File

@ -27,8 +27,10 @@
// * * (use with COARSE_BITS=16)
// * PQ[1]x10
// * PQ[1]x16
// Unfortunately, currently Faiss does not support something like
// IVF256,PQ16x10np
// * IVF256,PQ[1]x10 (such as IVF256,PQ16x10np)
// * IVF256,PQ[1]x16 (such as IVF256,PQ16x16np)
// * IVF[2^9-2^16 bit],PQ[1]x10 (such as IVF1024,PQ16x10np)
// * IVF[2^9-2^16 bit],PQ[1]x16 (such as IVF1024,PQ16x16np)
//
// The goal was to achieve the maximum performance, so the template version it
// is. The provided index families share the same code for sa_decode.
@ -62,6 +64,10 @@
// decoder.
// For example, "Residual4x10,PQ16x10np" for 256-dim data translates into
// Index2LevelDecoder<256,64,16,10,10>
// For example, "IVF1024,PQ16x10np" for 256-dim data translates into
// Index2LevelDecoder<256,256,16,10,10>. But as there are only 1 coarse code
// element, Index2LevelDecoder<256,256,16,16,10> can be used as a faster
// decoder.
//
// Additional supported values for COARSE_BITS and FINE_BITS may be added later.
//

6
faiss/cppcontrib/sa_decode/Level2-avx2-inl.h
View File

@ -1857,8 +1857,12 @@ struct Index2LevelDecoderImpl<
} // namespace
// Suitable for IVF256,PQ[1]x8
// Subtable for IVF256,PQ[1]x10 (such as IVF256,PQ16x10np)
// Suitable for IVF256,PQ[1]x16 (such as IVF256,PQ16x16np)
// Suitable for Residual[1]x8,PQ[2]x8
// Suitable for IVF[9-16 bit],PQ[1]x8 (such as IVF1024,PQ16np)
// Suitable for IVF[2^9-2^16 bit],PQ[1]x8 (such as IVF1024,PQ16np)
// Suitable for IVF[2^9-2^16 bit],PQ[1]x10 (such as IVF1024,PQ16x10np)
// Suitable for IVF[2^9-2^16 bit],PQ[1]x16 (such as IVF1024,PQ16x16np)
// Suitable for Residual[1]x[9-16 bit],PQ[2]x[3] (such as Residual2x9,PQ8)
template <
intptr_t DIM,

8
faiss/cppcontrib/sa_decode/Level2-neon-inl.h
View File

@ -1946,9 +1946,13 @@ struct Index2LevelDecoderImpl<
} // namespace
// Suitable for IVF256,PQ[1]x8
// Subtable for IVF256,PQ[1]x10 (such as IVF256,PQ16x10np)
// Suitable for IVF256,PQ[1]x16 (such as IVF256,PQ16x16np)
// Suitable for Residual[1]x8,PQ[2]x8
// Suitable for IVF[9-16 bit],PQ[1]x8 (such as IVF1024,PQ16np)
// Suitable for Residual1x[9-16 bit],PQ[1]x8 (such as Residual1x9,PQ8)
// Suitable for IVF[2^9-2^16 bit],PQ[1]x8 (such as IVF1024,PQ16np)
// Suitable for IVF[2^9-2^16 bit],PQ[1]x10 (such as IVF1024,PQ16x10np)
// Suitable for IVF[2^9-2^16 bit],PQ[1]x16 (such as IVF1024,PQ16x16np)
// Suitable for Residual[1]x[9-16 bit],PQ[2]x[3] (such as Residual2x9,PQ8)
template <
intptr_t DIM,
intptr_t COARSE_SIZE,

26
tests/test_cppcontrib_sa_decode.cpp
View File

@ -1239,6 +1239,32 @@ TEST(TEST_CPPCONTRIB_SA_DECODE, D160_PQ20x10) {
testIndexPQDecoder<T>(NSAMPLES * 4, 160, "PQ20x10np");
}
TEST(TEST_CPPCONTRIB_SA_DECODE, D256_IVF256_PQ16x10) {
using T = faiss::cppcontrib::Index2LevelDecoder<256, 256, 16, 8, 10>;
testIndex2LevelDecoder<T>(NSAMPLES * 4, 256, "IVF256,PQ16x10np");
}
TEST(TEST_CPPCONTRIB_SA_DECODE, D256_MINMAXFP16_IVF256_PQ16x10) {
using SubT = faiss::cppcontrib::Index2LevelDecoder<256, 256, 16, 8, 10>;
using T = faiss::cppcontrib::IndexMinMaxFP16Decoder<SubT>;
testMinMaxIndex2LevelDecoder<T>(
NSAMPLES * 4, 256, "MinMaxFP16,IVF256,PQ16x10np");
}
TEST(TEST_CPPCONTRIB_SA_DECODE, D256_MINMAXFP16_IVF1024_PQ16x10) {
using SubT = faiss::cppcontrib::Index2LevelDecoder<256, 256, 16, 10, 10>;
using T = faiss::cppcontrib::IndexMinMaxFP16Decoder<SubT>;
testMinMaxIndex2LevelDecoder<T>(
NSAMPLES * 4, 256, "MinMaxFP16,IVF1024,PQ16x10np");
}
TEST(TEST_CPPCONTRIB_SA_DECODE, D256_MINMAXFP16_IVF1024_PQ16x10_ALTERNATIVE) {
using SubT = faiss::cppcontrib::Index2LevelDecoder<256, 256, 16, 16, 10>;
using T = faiss::cppcontrib::IndexMinMaxFP16Decoder<SubT>;
testMinMaxIndex2LevelDecoder<T>(
NSAMPLES * 4, 256, "MinMaxFP16,IVF1024,PQ16x10np");
}
TEST(TEST_CPPCONTRIB_SA_DECODE, D160_Residual4x8_PQ8x10) {
using T = faiss::cppcontrib::Index2LevelDecoder<160, 40, 20, 8, 10>;
testIndex2LevelDecoder<T>(NSAMPLES * 4, 160, "Residual4x8,PQ8x10");