mirrors
/
faiss
mirror of https://github.com/facebookresearch/faiss.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Cleaning up more unnecessary print (#3455) 

Summary:
Pull Request resolved: https://github.com/facebookresearch/faiss/pull/3455

Code quality control by reducing the number of prints

Reviewed By: junjieqi

Differential Revision: D57502194

fbshipit-source-id: a6cd65ed4cc49590ce73d2978d41b640b5259c17
pull/3369/head
Xiao Fu 2024-05-17 16:59:36 -07:00 committed by Facebook GitHub Bot
parent e822a8c152
commit 5e452ed52a
14 changed files with 0 additions and 105 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
tests/test_binary_hashindex.py
View File

@ -58,8 +58,6 @@ class TestRange(unittest.TestCase):
Lref, Dref, Iref = index_ref.range_search(xq, radius)
print("nb res: ", Lref[-1])
index = faiss.IndexBinaryHash(d, 10)
index.add(xb)
# index.display()
@ -80,8 +78,6 @@ class TestRange(unittest.TestCase):
self.assertTrue(snew <= set(ref))
nfound.append(Lnew[-1])
ndis.append(stats.ndis)
print('nfound=', nfound)
print('ndis=', ndis)
nfound = np.array(nfound)
self.assertTrue(nfound[-1] == Lref[-1])
self.assertTrue(np.all(nfound[1:] >= nfound[:-1]))
@ -100,8 +96,6 @@ class TestRange(unittest.TestCase):
Lref, Dref, Iref = index_ref.range_search(xq, radius)
print("nb res: ", Lref[-1])
nfound = []
ndis = []
@ -123,8 +117,6 @@ class TestRange(unittest.TestCase):
self.assertTrue(snew <= set(ref))
nfound.append(Lnew[-1])
ndis.append(stats.ndis)
print('nfound=', nfound)
print('ndis=', ndis)
nfound = np.array(nfound)
# self.assertTrue(nfound[-1] == Lref[-1])
self.assertTrue(np.all(nfound[1:] >= nfound[:-1]))
@ -163,7 +155,6 @@ class TestKnn(unittest.TestCase):
# no duplicates
self.assertTrue(len(new) == len(snew))
nf += len(set(ref) & snew)
print('nfound', nh, nbit, nf)
nfound[(nh, nbit)] = nf
self.assertGreater(nfound[(nh, 4)], nfound[(nh, 7)])
@ -175,7 +166,6 @@ class TestKnn(unittest.TestCase):
np.testing.assert_array_equal(Inew, I2)
np.testing.assert_array_equal(Dnew, D2)
print('nfound=', nfound)
self.assertGreater(3, abs(nfound[(0, 7)] - nfound[(1, 7)]))
self.assertGreater(nfound[(3, 7)], nfound[(1, 7)])
self.assertGreater(nfound[(5, 7)], nfound[(3, 7)])

15
tests/test_build_blocks.py
View File

@ -189,7 +189,6 @@ class TestNyFuncs(unittest.TestCase):
for d in 1, 2, 4, 8, 12, 16:
x = rs.rand(d).astype('float32')
for ny in 128, 129, 130:
print("d=%d ny=%d" % (d, ny))
y = rs.rand(ny, d).astype('float32')
ref = ((x - y) ** 2).sum(1)
new = np.zeros(ny, dtype='float32')
@ -204,7 +203,6 @@ class TestNyFuncs(unittest.TestCase):
for d in 1, 2, 4, 8, 12, 16:
x = rs.rand(d).astype('float32')
for ny in 128, 129, 130:
print("d=%d ny=%d" % (d, ny))
y = rs.rand(ny, d).astype('float32')
ref = (x * y).sum(1)
new = np.zeros(ny, dtype='float32')
@ -220,7 +218,6 @@ class TestMatrixStats(unittest.TestCase):
m = rs.rand(40, 20).astype('float32')
m[5:10] = 0
comments = faiss.MatrixStats(m).comments
print(comments)
assert 'has 5 copies' in comments
assert '5 null vectors' in comments
@ -229,7 +226,6 @@ class TestMatrixStats(unittest.TestCase):
m = rs.rand(40, 20).astype('float32')
m[::2] = m[1::2]
comments = faiss.MatrixStats(m).comments
print(comments)
assert '20 vectors are distinct' in comments
def test_dead_dims(self):
@ -237,7 +233,6 @@ class TestMatrixStats(unittest.TestCase):
m = rs.rand(40, 20).astype('float32')
m[:, 5:10] = 0
comments = faiss.MatrixStats(m).comments
print(comments)
assert '5 dimensions are constant' in comments
def test_rogue_means(self):
@ -245,7 +240,6 @@ class TestMatrixStats(unittest.TestCase):
m = rs.rand(40, 20).astype('float32')
m[:, 5:10] += 12345
comments = faiss.MatrixStats(m).comments
print(comments)
assert '5 dimensions are too large wrt. their variance' in comments
def test_normalized(self):
@ -253,7 +247,6 @@ class TestMatrixStats(unittest.TestCase):
m = rs.rand(40, 20).astype('float32')
faiss.normalize_L2(m)
comments = faiss.MatrixStats(m).comments
print(comments)
assert 'vectors are normalized' in comments
def test_hash(self):
@ -300,7 +293,6 @@ class TestScalarQuantizer(unittest.TestCase):
D, I = index.search(x[3:], 1)
# assert D[0, 0] == Dref[0, 0]
# print(D[0, 0], ((x[3] - x[2]) ** 2).sum())
assert D[0, 0] == ((x[3] - x[2]) ** 2).sum()
def test_6bit_equiv(self):
@ -314,8 +306,6 @@ class TestScalarQuantizer(unittest.TestCase):
d, faiss.ScalarQuantizer.QT_6bit)
index.train(trainset)
print('cs=', index.code_size)
x = rs.randint(64, size=(100, d)).astype('float32')
# verify encoder / decoder
@ -330,7 +320,6 @@ class TestScalarQuantizer(unittest.TestCase):
for i in range(20):
for j in range(10):
dis = ((y[i] - x2[I[i, j]]) ** 2).sum()
# print(dis, D[i, j])
assert abs(D[i, j] - dis) / dis < 1e-5
def test_reconstruct(self):
@ -371,7 +360,6 @@ class TestRandom(unittest.TestCase):
x = faiss.randint(20000, vmax=100)
assert np.all(x >= 0) and np.all(x < 100)
c = np.bincount(x, minlength=100)
print(c)
assert c.max() - c.min() < 50 * 2
def test_rand_vector(self):
@ -473,7 +461,6 @@ class TestSWIGWrap(unittest.TestCase):
""" tests swig_ptr and rev_swig_ptr for this type of array """
a = np.arange(12).astype(dtype)
ptr = faiss.swig_ptr(a)
print(ptr)
a2 = faiss.rev_swig_ptr(ptr, 12)
np.testing.assert_array_equal(a, a2)
@ -547,7 +534,6 @@ class TestNNDescentKNNG(unittest.TestCase):
recalls += 1
break
recall = 1.0 * recalls / (nb * K)
print('Metric: {}, knng accuracy: {}'.format(metric_names[metric], recall))
assert recall > 0.99
def test_small_nndescent(self):
@ -656,7 +642,6 @@ class TestBucketSort(unittest.TestCase):
rows, _ = np.where(tab == b)
rows.sort()
tab2[lims[b]:lims[b + 1]].sort()
# print(rows, tab2[lims[b] : lims[b + 1]])
rows = set(rows)
self.assertEqual(rows, set(tab2[lims[b]:lims[b + 1]]))

7
tests/test_graph_based.py
View File

@ -209,7 +209,6 @@ class TestNSG(unittest.TestCase):
Dnsg, Insg = index.search(self.xq, 1)
recalls = (Iref == Insg).sum()
print('metric: {}, nb equal: {}'.format(metrics[metric], recalls))
self.assertGreaterEqual(recalls, thresh)
self.subtest_connectivity(index, self.xb.shape[0])
self.subtest_io_and_clone(index, Dnsg, Insg)
@ -230,7 +229,6 @@ class TestNSG(unittest.TestCase):
Dnsg, Insg = index.search(self.xq, 1)
recalls = (Iref == Insg).sum()
print('metric: {}, nb equal: {}'.format(metrics[metric], recalls))
self.assertGreaterEqual(recalls, thresh)
self.subtest_connectivity(index, self.xb.shape[0])
@ -286,7 +284,6 @@ class TestNSG(unittest.TestCase):
index.add(self.xb)
Dnsg, Insg = index.search(self.xq, 1)
recalls = (Iref == Insg).sum()
print('metric: {}, nb equal: {}'.format(metrics[metric], recalls))
self.assertGreaterEqual(recalls, 475)
self.subtest_connectivity(index, self.xb.shape[0])
@ -294,7 +291,6 @@ class TestNSG(unittest.TestCase):
index.add(self.xb)
Dnsg, Insg = index.search(self.xq, 1)
recalls = (Iref == Insg).sum()
print('metric: {}, nb equal: {}'.format(metrics[metric], recalls))
self.assertGreaterEqual(recalls, 475)
self.subtest_connectivity(index, self.xb.shape[0])
@ -335,7 +331,6 @@ class TestNSG(unittest.TestCase):
# test accuracy
recalls = (Iref == I).sum()
print("IndexNSGPQ", recalls)
self.assertGreaterEqual(recalls, 190) # 193
# test I/O
@ -361,7 +356,6 @@ class TestNSG(unittest.TestCase):
# test accuracy
recalls = (Iref == I).sum()
print("IndexNSGSQ", recalls)
self.assertGreaterEqual(recalls, 405) # 411
# test I/O
@ -395,7 +389,6 @@ class TestNNDescent(unittest.TestCase):
# test accuracy
recalls = (Iref == I).sum()
print("IndexNNDescentFlat", recalls)
self.assertGreaterEqual(recalls, 450) # 462
# do some IO tests

5
tests/test_index.py
View File

@ -338,7 +338,6 @@ class TestScalarQuantizer(unittest.TestCase):
D, I = index.search(xq, 10)
nok[qname] = (I[:, 0] == I_ref[:, 0]).sum()
print(nok, nq)
self.assertGreaterEqual(nok['flat'], nq * 0.6)
# The tests below are a bit fragile, it happens that the
@ -373,8 +372,6 @@ class TestScalarQuantizer(unittest.TestCase):
D, I = index.search(xq, 10)
nok[qname] = (I[:, 0] == I_ref[:, 0]).sum()
print(nok, nq)
self.assertGreaterEqual(nok['QT_8bit'], nq * 0.9)
self.assertGreaterEqual(nok['QT_8bit'], nok['QT_4bit'])
self.assertGreaterEqual(nok['QT_8bit'], nok['QT_8bit_uniform'])
@ -442,7 +439,6 @@ class TestSearchAndReconstruct(unittest.TestCase):
recons_err = np.mean(norm1(R_flat - xb[I_flat]))
print('Reconstruction error = %.3f' % recons_err)
if eps is not None:
self.assertLessEqual(recons_err, eps)
@ -638,7 +634,6 @@ class TestReconsException(unittest.TestCase):
# should not raise an exception
index.reconstruct(5)
print(index.ntotal)
index.reconstruct(150)

24
tests/test_index_accuracy.py
View File

@ -56,7 +56,6 @@ class IndexAccuracy(unittest.TestCase):
Dref, Iref = ivfk.search(ev.xq, 100)
ivfk.parallel_mode = 1
Dnew, Inew = ivfk.search(ev.xq, 100)
print((Iref != Inew).sum(), Iref.size)
assert (Iref != Inew).sum() < Iref.size / 5000.0
assert np.all(Dref == Dnew)
@ -136,8 +135,6 @@ class IndexAccuracy(unittest.TestCase):
res = ev.launch("Polysemous ht=%d" % index.polysemous_ht, index)
e_polysemous = ev.evalres(res)
print(e_baseline, e_polysemous, index.polysemous_ht)
print(stats.n_hamming_pass, stats.ncode)
# The randu dataset is difficult, so we are not too picky on
# the results. Here we assert that we have < 10 % loss when
# computing full PQ on fewer than 20% of the data.
@ -248,7 +245,6 @@ class TestSQFlavors(unittest.TestCase):
index.nprobe = 4 # hopefully more robust than 1
D, I = index.search(xq, 10)
ninter = faiss.eval_intersection(I, gt_I)
print("(%d, %s): %d, " % (mt, repr(qname), ninter))
assert abs(ninter - self.ref_results[(mt, qname)]) <= 10
if qname == "6bit":
@ -264,7 +260,6 @@ class TestSQFlavors(unittest.TestCase):
radius = float(D[:, -1].max())
else:
radius = float(D[:, -1].min())
# print("radius", radius)
lims, D3, I3 = index.range_search(xq, radius)
ntot = ndiff = 0
@ -278,14 +273,11 @@ class TestSQFlavors(unittest.TestCase):
Iref = set(I2[i, mask])
ndiff += len(Inew ^ Iref)
ntot += len(Iref)
# print("ndiff %d / %d" % (ndiff, ntot))
assert ndiff < ntot * 0.01
for pm in 1, 2:
# print("parallel_mode=%d" % pm)
index.parallel_mode = pm
lims4, D4, I4 = index.range_search(xq, radius)
# print("sizes", lims4[1:] - lims4[:-1])
for qno in range(len(lims) - 1):
Iref = I3[lims[qno]: lims[qno + 1]]
Inew = I4[lims4[qno]: lims4[qno + 1]]
@ -485,7 +477,6 @@ class TestPQFlavors(unittest.TestCase):
D, I = index.search(xq, 10)
ninter = faiss.eval_intersection(I, gt_I)
print("(%d, %s): %d, " % (mt, by_residual, ninter))
assert abs(ninter - self.ref_results[mt, by_residual]) <= 3
@ -499,10 +490,6 @@ class TestPQFlavors(unittest.TestCase):
index.polysemous_ht = 20
D, I = index.search(xq, 10)
ninter = faiss.eval_intersection(I, gt_I)
print(
"(%d, %s, %d): %d, "
% (mt, by_residual, index.polysemous_ht, ninter)
)
# polysemous behaves bizarrely on ARM
assert (
@ -516,7 +503,6 @@ class TestPQFlavors(unittest.TestCase):
radius = float(D[:, -1].max())
else:
radius = float(D[:, -1].min())
print("radius", radius)
lims, D3, I3 = index.range_search(xq, radius)
ntot = ndiff = 0
@ -530,7 +516,6 @@ class TestPQFlavors(unittest.TestCase):
Iref = set(I2[i, mask])
ndiff += len(Inew ^ Iref)
ntot += len(Iref)
print("ndiff %d / %d" % (ndiff, ntot))
assert ndiff < ntot * 0.02
def test_IVFPQ_non8bit(self):
@ -555,7 +540,6 @@ class TestPQFlavors(unittest.TestCase):
D, I = index.search(xq, 10)
ninter[v] = faiss.eval_intersection(I, gt_I)
print("ninter=", ninter)
# this should be the case but we don't observe
# that... Probavly too few test points
# assert ninter['2x8'] > ninter['8x2']
@ -623,9 +607,6 @@ class OPQRelativeAccuracy(unittest.TestCase):
res = ev.launch("OPQ", index)
e_opq = ev.evalres(res)
print("e_pq=%s" % e_pq)
print("e_opq=%s" % e_opq)
# verify that OPQ better than PQ
for r in 1, 10, 100:
assert e_opq[r] > e_pq[r]
@ -656,7 +637,6 @@ class OPQRelativeAccuracy(unittest.TestCase):
# verify same on OIVFPQ
for r in 1, 10, 100:
print(e_oivfpq[r], e_ivfpq[r])
assert e_oivfpq[r] >= e_ivfpq[r]
@ -758,9 +738,6 @@ class TestSpectralHash(unittest.TestCase):
ninter = faiss.eval_intersection(I, gt_I)
key = (nbit, tt, period)
print("(%d, %s, %g): %d, " % (nbit, repr(tt), period,
ninter))
print(abs(ninter - self.ref_results[key]))
assert abs(ninter - self.ref_results[key]) <= 14
@ -799,7 +776,6 @@ class TestRefine(unittest.TestCase):
# check that with refinement, the recall@10 is the same as
# the original recall@100
recall2 = (I2 == Iref[:, :1]).sum()
# print("recalls", recall1, recall2)
self.assertEqual(recall1, recall2)
def test_IP(self):

1
tests/test_io.py
View File

@ -102,7 +102,6 @@ class TestCallbacks(unittest.TestCase):
reader = faiss.BufferedIOReader(reader, bsz)
y = np.zeros_like(x)
print('nbytes=', y.nbytes)
reader(faiss.swig_ptr(y), y.nbytes, 1)
np.testing.assert_array_equal(x, y)

1
tests/test_ivflib.py
View File

@ -125,7 +125,6 @@ class TestSearchWithParameters(unittest.TestCase):
Dpre, _ = index.search(xq, 15)
radius = float(np.median(Dpre[:, -1]))
print("Radius=", radius)
stats = faiss.cvar.indexIVF_stats
stats.reset()
Lref, Dref, Iref = index.range_search(xq, radius)

13
tests/test_lowlevel_ivf.cpp
View File

@ -364,22 +364,9 @@ void test_lowlevel_access_binary(const char* index_key) {
}
}
printf("new before reroder: [");
for (int j = 0; j < k; j++)
printf("%" PRId64 ",%d ", I[j], D[j]);
printf("]\n");
// re-order heap
heap_reorder<CMax<int32_t, idx_t>>(k, D.data(), I.data());
printf("ref: [");
for (int j = 0; j < k; j++)
printf("%" PRId64 ",%d ", I_ref[j], D_ref[j]);
printf("]\nnew: [");
for (int j = 0; j < k; j++)
printf("%" PRId64 ",%d ", I[j], D[j]);
printf("]\n");
// check that we have the same results as the reference search
for (int j = 0; j < k; j++) {
// here the order is not guaranteed to be the same

1
tests/test_merge_index.py
View File

@ -72,7 +72,6 @@ class TestMerge1(unittest.TestCase):
index.merge_from(indexes[i], index.ntotal)
_D, I = index.search(xq, k)
print(I[:5, :6])
ndiff = (I != Iref).sum()
print('%d / %d differences' % (ndiff, nq * k))

7
tests/test_meta_index.py
View File

@ -82,10 +82,8 @@ class Shards(unittest.TestCase):
k = 32
ref_index = faiss.IndexFlatL2(d)
print('ref search')
ref_index.add(xb)
_Dref, Iref = ref_index.search(xq, k)
print(Iref[:5, :6])
shard_index = faiss.IndexShards(d)
shard_index_2 = faiss.IndexShards(d, True, False)
@ -109,7 +107,6 @@ class Shards(unittest.TestCase):
for test_no in range(3):
with_threads = test_no == 1
print('shard search test_no = %d' % test_no)
if with_threads:
remember_nt = faiss.omp_get_max_threads()
faiss.omp_set_num_threads(1)
@ -122,14 +119,10 @@ class Shards(unittest.TestCase):
else:
_D, I = shard_index_2.search(xq, k)
print(I[:5, :6])
if with_threads:
faiss.omp_set_num_threads(remember_nt)
ndiff = (I != Iref).sum()
print('%d / %d differences' % (ndiff, nq * k))
assert (ndiff < nq * k / 1000.)
def test_shards_ivf(self):

1
tests/test_product_quantizer.py
View File

@ -70,7 +70,6 @@ class TestProductQuantizer(unittest.TestCase):
def test_codec(self):
for i in range(16):
print("Testing nbits=%d" % (i + 1))
self.do_test_codec(i + 1)

3
tests/test_residual_quantizer.py
View File

@ -967,8 +967,6 @@ def beam_search_encode_step_tab(codes, L, distances, codebook_cross_prods_i,
for b in range(beam_size):
dotprods[i, b, :] += cb[codes[i, b, j]]
# print("dps", dotprods[:3, :2, :4])
new_distances += 2 * dotprods
cent_distances = new_distances
@ -1174,7 +1172,6 @@ class TestProductResidualQuantizer(unittest.TestCase):
rq.train(xt)
err_rq = eval_codec(rq, xb)
# print(err_prq, err_rq)
self.assertEqual(err_prq, err_rq)

1
tests/test_search_params.py
View File

@ -465,7 +465,6 @@ class TestSortedIDSelectorRange(unittest.TestCase):
sp = faiss.swig_ptr
selr.find_sorted_ids_bounds(
len(ids), sp(ids), sp(j01[:1]), sp(j01[1:]))
print(j01)
assert j01[0] >= j01[1]

16
tests/test_sliding_ivf.cpp
View File

@ -74,8 +74,6 @@ void make_index_slices(
for (int i = 0; i < total_size; i++) {
sub_indexes.emplace_back(clone_index(trained_index));
printf("preparing sub-index # %d\n", i);
Index* index = sub_indexes.back().get();
auto xb = make_data(nb * d);
@ -122,13 +120,10 @@ int test_sliding_window(const char* index_key) {
auto xq = make_data(nq * d);
for (int i = 0; i < total_size + window_size; i++) {
printf("doing step %d / %d\n", i, total_size + window_size);
// update the index
window.step(
i < total_size ? sub_indexes[i].get() : nullptr,
i >= window_size);
printf(" current n_slice = %d\n", window.n_slice);
auto new_res = search_index(index.get(), xq.data());
@ -159,8 +154,6 @@ int test_sliding_invlists(const char* index_key) {
auto xq = make_data(nq * d);
for (int i = 0; i < total_size + window_size; i++) {
printf("doing step %d / %d\n", i, total_size + window_size);
// update the index
std::vector<const InvertedLists*> ils;
for (int j = i - window_size + 1; j <= i; j++) {
@ -178,8 +171,6 @@ int test_sliding_invlists(const char* index_key) {
// will be deleted by the index
index_ivf->replace_invlists(ci, true);
printf(" nb invlists = %zd\n", ils.size());
auto new_res = search_index(index.get(), xq.data());
std::unique_ptr<Index> merged_index(
@ -188,13 +179,6 @@ int test_sliding_invlists(const char* index_key) {
auto ref_res = search_index(merged_index.get(), xq.data());
EXPECT_EQ(ref_res.size(), new_res.size());
size_t ndiff = 0;
for (size_t j = 0; j < ref_res.size(); j++) {
if (ref_res[j] != new_res[j])
ndiff++;
}
printf(" nb differences: %zd / %zd\n", ndiff, ref_res.size());
EXPECT_EQ(ref_res, new_res);
}
return 0;