mirror of
https://github.com/facebookresearch/faiss.git
synced 2025-06-03 21:54:02 +08:00
a996a4a052
Summary: Pull Request resolved: https://github.com/facebookresearch/faiss/pull/2582 A few more or less cosmetic improvements * Index::idx_t was in the Index object, which does not make much sense, this diff moves it to faiss::idx_t * replace multiprocessing.dummy with multiprocessing.pool * add Alexandr as a core contributor of Faiss in the README ;-) ``` for i in $( find . -name \*.cu -o -name \*.cuh -o -name \*.h -o -name \*.cpp ) ; do sed -i s/Index::idx_t/idx_t/ $i done ``` For the fbcode deps: ``` for i in $( fbgs Index::idx_t --exclude fbcode/faiss -l ) ; do sed -i s/Index::idx_t/idx_t/ $i done ``` Reviewed By: algoriddle Differential Revision: D41437507 fbshipit-source-id: 8300f2a3ae97cace6172f3f14a9be3a83999fb89
67 lines
2.0 KiB
C++
67 lines
2.0 KiB
C++
/**
|
|
* Copyright (c) Facebook, Inc. and its affiliates.
|
|
*
|
|
* This source code is licensed under the MIT license found in the
|
|
* LICENSE file in the root directory of this source tree.
|
|
*/
|
|
|
|
#include <faiss/IndexFlat.h>
|
|
#include <faiss/IndexIVFFlat.h>
|
|
#include <faiss/utils/random.h>
|
|
#include <faiss/utils/utils.h>
|
|
|
|
#include <gtest/gtest.h>
|
|
|
|
using namespace faiss;
|
|
|
|
TEST(MEM_LEAK, ivfflat) {
|
|
size_t num_tfidf_faiss_cells = 20;
|
|
size_t max_tfidf_features = 500;
|
|
|
|
IndexFlatIP quantizer(max_tfidf_features);
|
|
IndexIVFFlat tfidf_faiss_index(
|
|
&quantizer, max_tfidf_features, num_tfidf_faiss_cells);
|
|
|
|
std::vector<float> dense_matrix(5000 * max_tfidf_features);
|
|
float_rand(dense_matrix.data(), dense_matrix.size(), 123);
|
|
|
|
tfidf_faiss_index.train(5000, dense_matrix.data());
|
|
tfidf_faiss_index.add(5000, dense_matrix.data());
|
|
|
|
int N1 = 1000;
|
|
int N2 = 10000;
|
|
|
|
std::vector<float> ent_substr_tfidfs_list(N1 * max_tfidf_features);
|
|
float_rand(
|
|
ent_substr_tfidfs_list.data(), ent_substr_tfidfs_list.size(), 1234);
|
|
|
|
for (int bs : {1, 4, 16}) {
|
|
size_t m0 = get_mem_usage_kb();
|
|
double t0 = getmillisecs();
|
|
|
|
for (int i = 0; i < N2; i++) {
|
|
std::vector<idx_t> I(10 * bs);
|
|
std::vector<float> D(10 * bs);
|
|
|
|
tfidf_faiss_index.search(
|
|
bs,
|
|
ent_substr_tfidfs_list.data() +
|
|
(i % (N1 - bs + 1)) * max_tfidf_features,
|
|
10,
|
|
D.data(),
|
|
I.data());
|
|
if (i % 100 == 0) {
|
|
printf("[%.2f s] BS %d %d: %ld kB %.2f bytes/it\r",
|
|
(getmillisecs() - t0) / 1000,
|
|
bs,
|
|
i,
|
|
get_mem_usage_kb(),
|
|
(get_mem_usage_kb() - m0) * 1024.0 / (i + 1));
|
|
fflush(stdout);
|
|
}
|
|
}
|
|
printf("\n");
|
|
EXPECT_GE(50 * bs, (get_mem_usage_kb() - m0) * 1024.0 / N2);
|
|
}
|
|
}
|