mirror of
https://github.com/facebookresearch/faiss.git
synced 2025-06-03 21:54:02 +08:00
f849680777
Summary: This diff adds an object for a few useful dataset in faiss.contrib. This includes synthetic datasets and the classic ones. It is intended to work on: - the FAIR cluster - gluster - manifold Reviewed By: wickedfoo Differential Revision: D23378763 fbshipit-source-id: 2437a7be9e712fd5ad1bccbe523cc1c936f7ab35
32 lines
611 B
Python
32 lines
611 B
Python
import numpy as np
|
|
|
|
"""
|
|
I/O functions in fvecs, bvecs, ivecs formats
|
|
"""
|
|
|
|
|
|
def ivecs_read(fname):
|
|
a = np.fromfile(fname, dtype='int32')
|
|
d = a[0]
|
|
return a.reshape(-1, d + 1)[:, 1:].copy()
|
|
|
|
|
|
def fvecs_read(fname):
|
|
return ivecs_read(fname).view('float32')
|
|
|
|
|
|
def ivecs_mmap(fname):
|
|
a = np.memmap(fname, dtype='int32', mode='r')
|
|
d = a[0]
|
|
return a.reshape(-1, d + 1)[:, 1:]
|
|
|
|
|
|
def fvecs_mmap(fname):
|
|
return ivecs_mmap(fname).view('float32')
|
|
|
|
|
|
def bvecs_mmap(fname):
|
|
x = np.memmap(fname, dtype='uint8', mode='r')
|
|
d = x[:4].view('int32')[0]
|
|
return x.reshape(-1, d + 4)[:, 4:]
|