mirror of
https://github.com/facebookresearch/faiss.git
synced 2025-06-03 21:54:02 +08:00
7d21c92fc1
Summary: This PR adds support for dimensionality reduction in OIVFBBS. I tested the code with an index `OPQ64_128,IVF4096,PQ64` using the ssnpp embeddings - this index string is added to the config_ssnpp.yaml to showcase this functionality. Pull Request resolved: https://github.com/facebookresearch/faiss/pull/3290 Reviewed By: junjieqi Differential Revision: D54878345 Pulled By: mlomeli1 fbshipit-source-id: 98ecdeb2224ce0325e37720cc113d82f9c6c75d6
111 lines
2.3 KiB
YAML
111 lines
2.3 KiB
YAML
d: 256
|
|
output: /checkpoint/marialomeli/offline_faiss/ssnpp
|
|
index:
|
|
prod:
|
|
- 'IVF8192,PQ128'
|
|
non-prod:
|
|
- 'IVF16384,PQ128'
|
|
- 'IVF32768,PQ128'
|
|
- 'OPQ64_128,IVF4096,PQ64'
|
|
nprobe:
|
|
prod:
|
|
- 512
|
|
non-prod:
|
|
- 256
|
|
- 128
|
|
- 1024
|
|
- 2048
|
|
- 4096
|
|
- 8192
|
|
|
|
k: 50
|
|
index_shard_size: 50000000
|
|
query_batch_size: 50000000
|
|
evaluation_sample: 10000
|
|
training_sample: 1572864
|
|
datasets:
|
|
ssnpp_1B:
|
|
root: /checkpoint/marialomeli/ssnpp_data
|
|
size: 1000000000
|
|
files:
|
|
- dtype: uint8
|
|
format: npy
|
|
name: ssnpp_0000000000.npy
|
|
size: 50000000
|
|
- dtype: uint8
|
|
format: npy
|
|
name: ssnpp_0000000001.npy
|
|
size: 50000000
|
|
- dtype: uint8
|
|
format: npy
|
|
name: ssnpp_0000000002.npy
|
|
size: 50000000
|
|
- dtype: uint8
|
|
format: npy
|
|
name: ssnpp_0000000003.npy
|
|
size: 50000000
|
|
- dtype: uint8
|
|
format: npy
|
|
name: ssnpp_0000000004.npy
|
|
size: 50000000
|
|
- dtype: uint8
|
|
format: npy
|
|
name: ssnpp_0000000005.npy
|
|
size: 50000000
|
|
- dtype: uint8
|
|
format: npy
|
|
name: ssnpp_0000000006.npy
|
|
size: 50000000
|
|
- dtype: uint8
|
|
format: npy
|
|
name: ssnpp_0000000007.npy
|
|
size: 50000000
|
|
- dtype: uint8
|
|
format: npy
|
|
name: ssnpp_0000000008.npy
|
|
size: 50000000
|
|
- dtype: uint8
|
|
format: npy
|
|
name: ssnpp_0000000009.npy
|
|
size: 50000000
|
|
- dtype: uint8
|
|
format: npy
|
|
name: ssnpp_0000000010.npy
|
|
size: 50000000
|
|
- dtype: uint8
|
|
format: npy
|
|
name: ssnpp_0000000011.npy
|
|
size: 50000000
|
|
- dtype: uint8
|
|
format: npy
|
|
name: ssnpp_0000000012.npy
|
|
size: 50000000
|
|
- dtype: uint8
|
|
format: npy
|
|
name: ssnpp_0000000013.npy
|
|
size: 50000000
|
|
- dtype: uint8
|
|
format: npy
|
|
name: ssnpp_0000000014.npy
|
|
size: 50000000
|
|
- dtype: uint8
|
|
format: npy
|
|
name: ssnpp_0000000015.npy
|
|
size: 50000000
|
|
- dtype: uint8
|
|
format: npy
|
|
name: ssnpp_0000000016.npy
|
|
size: 50000000
|
|
- dtype: uint8
|
|
format: npy
|
|
name: ssnpp_0000000017.npy
|
|
size: 50000000
|
|
- dtype: uint8
|
|
format: npy
|
|
name: ssnpp_0000000018.npy
|
|
size: 50000000
|
|
- dtype: uint8
|
|
format: npy
|
|
name: ssnpp_0000000019.npy
|
|
size: 50000000
|