mirrors/faiss
1
0
Fork 0
mirror of https://github.com/facebookresearch/faiss.git synced 2025-06-03 21:54:02 +08:00
Code Issues Projects Releases Wiki Activity

Revert D69984379: mem mapping and zero-copy python fixes

Browse Source 
Differential Revision:
D69984379

Original commit changeset: 9437b4ad92ef

Original Phabricator Diff: D69984379

fbshipit-source-id: 3cb921fa79b6f20b6455b17e50acc3cb96bcbe7b
This commit is contained in:
Saumya Agarwal 2025-03-11 11:43:17 -07:00 committed by Facebook GitHub Bot
parent 631b0fde4f
commit fbc7db2cce
8 changed files with 8 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
faiss/impl/mapped_io.cpp
View File

@ -1,10 +1,3 @@
/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
#include <stdio.h>
#include <string.h>
@ -18,8 +11,8 @@
#elif defined(_WIN32)
#include <Windows.h> // @manual
#include <io.h> // @manual
#include <Windows.h>
#include <io.h>
#endif
@ -285,4 +278,4 @@ int MappedFileIOReader::filedescriptor() {
return -1;
}
} // namespace faiss
} // namespace faiss

9
faiss/impl/mapped_io.h
View File

@ -1,10 +1,3 @@
/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
#pragma once
#include <cstddef>
@ -48,4 +41,4 @@ struct MappedFileIOReader : IOReader {
int filedescriptor() override;
};
} // namespace faiss
} // namespace faiss

16
faiss/impl/maybe_owned_vector.h
View File

@ -1,10 +1,3 @@
/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
#pragma once
#include <cstddef>
@ -57,13 +50,6 @@ struct MaybeOwnedVector {
c_size = owned_data.size();
}
explicit MaybeOwnedVector(const std::vector<T>& vec)
: faiss::MaybeOwnedVector<T>(vec.size()) {
if (vec.size() > 0) {
memcpy(owned_data.data(), vec.data(), sizeof(T) * vec.size());
}
}
MaybeOwnedVector(const MaybeOwnedVector& other) {
is_owned = other.is_owned;
owned_data = other.owned_data;
@ -243,4 +229,4 @@ struct is_maybe_owned_vector<MaybeOwnedVector<T>> : std::true_type {};
template <typename T>
inline constexpr bool is_maybe_owned_vector_v = is_maybe_owned_vector<T>::value;
} // namespace faiss
} // namespace faiss

12
faiss/impl/zerocopy_io.cpp
View File

@ -1,10 +1,3 @@
/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
#include <faiss/impl/zerocopy_io.h>
#include <cstring>
@ -44,9 +37,6 @@ void ZeroCopyIOReader::reset() {
}
size_t ZeroCopyIOReader::operator()(void* ptr, size_t size, size_t nitems) {
if (size * nitems == 0) {
return 0;
}
if (rp_ >= total_) {
return 0;
}
@ -63,4 +53,4 @@ int ZeroCopyIOReader::filedescriptor() {
return -1; // Indicating no file descriptor available for memory buffer
}
} // namespace faiss
} // namespace faiss

9
faiss/impl/zerocopy_io.h
View File

@ -1,10 +1,3 @@
/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
#pragma once
#include <cstdint>
@ -29,4 +22,4 @@ struct ZeroCopyIOReader : public faiss::IOReader {
int filedescriptor() override;
};
} // namespace faiss
} // namespace faiss

1
faiss/python/swigfaiss.swig
View File

@ -32,7 +32,6 @@
#pragma SWIG nowarn=341
#pragma SWIG nowarn=512
#pragma SWIG nowarn=362
#pragma SWIG nowarn=509
// we need explict control of these typedefs...
// %include <stdint.i>

3
tests/test_fast_scan_ivf.py
View File

@ -270,9 +270,8 @@ class TestEquivPQ(unittest.TestCase):
index_pq = faiss.index_factory(32, "PQ16x4np")
index_pq.pq = index.pq
index_pq.is_trained = True
codevec = faiss.downcast_InvertedLists(
index_pq.codes = faiss. downcast_InvertedLists(
index.invlists).codes.at(0)
index_pq.codes = faiss.MaybeOwnedVectorUInt8(codevec)
index_pq.ntotal = index.ntotal
Dnew, Inew = index_pq.search(xq, 4)

37
tests/test_io.py
View File

@ -481,40 +481,3 @@ class TestIVFPQRead(unittest.TestCase):
finally:
if os.path.exists(fname):
os.unlink(fname)
class TestIOFlatMMap(unittest.TestCase):
def test_mmap(self):
xt, xb, xq = get_dataset_2(32, 0, 100, 50)
index = faiss.index_factory(32, "SQfp16", faiss.METRIC_L2)
# does not need training
index.add(xb)
Dref, Iref = index.search(xq, 10)
fd, fname = tempfile.mkstemp()
os.close(fd)
try:
faiss.write_index(index, fname)
index2 = faiss.read_index(fname, faiss.IO_FLAG_MMAP_IFC)
Dnew, Inew = index2.search(xq, 10)
np.testing.assert_array_equal(Iref, Inew)
np.testing.assert_array_equal(Dref, Dnew)
finally:
if os.path.exists(fname):
os.unlink(fname)
def test_zerocopy(self):
xt, xb, xq = get_dataset_2(32, 0, 100, 50)
index = faiss.index_factory(32, "SQfp16", faiss.METRIC_L2)
# does not need training
index.add(xb)
Dref, Iref = index.search(xq, 10)
serialized_index = faiss.serialize_index(index)
reader = faiss.ZeroCopyIOReader(
faiss.swig_ptr(serialized_index), serialized_index.size)
index2 = faiss.read_index(reader)
Dnew, Inew = index2.search(xq, 10)
np.testing.assert_array_equal(Iref, Inew)
np.testing.assert_array_equal(Dref, Dnew)