Faiss
 All Classes Namespaces Functions Variables Typedefs Enumerations Enumerator Friends
/data/users/matthijs/github_faiss/faiss/AutoTune.cpp
1 /**
2  * Copyright (c) 2015-present, Facebook, Inc.
3  * All rights reserved.
4  *
5  * This source code is licensed under the BSD+Patents license found in the
6  * LICENSE file in the root directory of this source tree.
7  */
8 
9 /* Copyright 2004-present Facebook. All Rights Reserved.
10  implementation of Hyper-parameter auto-tuning
11 */
12 
13 #include "AutoTune.h"
14 
15 #include "FaissAssert.h"
16 #include "utils.h"
17 
18 #include "IndexFlat.h"
19 #include "VectorTransform.h"
20 #include "IndexLSH.h"
21 #include "IndexPQ.h"
22 #include "IndexIVF.h"
23 #include "IndexIVFPQ.h"
24 #include "MetaIndexes.h"
25 #include "IndexScalarQuantizer.h"
26 
27 
28 namespace faiss {
29 
30 
31 AutoTuneCriterion::AutoTuneCriterion (idx_t nq, idx_t nnn):
32  nq (nq), nnn (nnn), gt_nnn (0)
33 {}
34 
35 
37  int gt_nnn, const float *gt_D_in, const idx_t *gt_I_in)
38 {
39  this->gt_nnn = gt_nnn;
40  if (gt_D_in) { // allow null for this, as it is often not used
41  gt_D.resize (nq * gt_nnn);
42  memcpy (gt_D.data(), gt_D_in, sizeof (gt_D[0]) * nq * gt_nnn);
43  }
44  gt_I.resize (nq * gt_nnn);
45  memcpy (gt_I.data(), gt_I_in, sizeof (gt_I[0]) * nq * gt_nnn);
46 }
47 
48 
49 
50 OneRecallAtRCriterion::OneRecallAtRCriterion (idx_t nq, idx_t R):
51  AutoTuneCriterion(nq, R), R(R)
52 {}
53 
54 double OneRecallAtRCriterion::evaluate(const float* /*D*/, const idx_t* I)
55  const {
56  FAISS_THROW_IF_NOT_MSG(
57  (gt_I.size() == gt_nnn * nq && gt_nnn >= 1 && nnn >= R),
58  "ground truth not initialized");
59  idx_t n_ok = 0;
60  for (idx_t q = 0; q < nq; q++) {
61  idx_t gt_nn = gt_I[q * gt_nnn];
62  const idx_t* I_line = I + q * nnn;
63  for (int i = 0; i < R; i++) {
64  if (I_line[i] == gt_nn) {
65  n_ok++;
66  break;
67  }
68  }
69  }
70  return n_ok / double(nq);
71 }
72 
73 
74 IntersectionCriterion::IntersectionCriterion (idx_t nq, idx_t R):
75  AutoTuneCriterion(nq, R), R(R)
76 {}
77 
78 double IntersectionCriterion::evaluate(const float* /*D*/, const idx_t* I)
79  const {
80  FAISS_THROW_IF_NOT_MSG(
81  (gt_I.size() == gt_nnn * nq && gt_nnn >= R && nnn >= R),
82  "ground truth not initialized");
83  long n_ok = 0;
84 #pragma omp parallel for reduction(+: n_ok)
85  for (idx_t q = 0; q < nq; q++) {
87  R, &gt_I [q * gt_nnn],
88  R, I + q * nnn);
89  }
90  return n_ok / double (nq * R);
91 }
92 
93 /***************************************************************
94  * OperatingPoints
95  ***************************************************************/
96 
97 OperatingPoints::OperatingPoints ()
98 {
99  clear();
100 }
101 
103 {
104  all_pts.clear();
105  optimal_pts.clear();
106  /// default point: doing nothing gives 0 performance and takes 0 time
107  OperatingPoint op = {0, 0, "", -1};
108  optimal_pts.push_back(op);
109 }
110 
111 /// add a performance measure
112 bool OperatingPoints::add (double perf, double t, const std::string & key,
113  size_t cno)
114 {
115  OperatingPoint op = {perf, t, key, long(cno)};
116  all_pts.push_back (op);
117  if (perf == 0) {
118  return false; // no method for 0 accuracy is faster than doing nothing
119  }
120  std::vector<OperatingPoint> & a = optimal_pts;
121  if (perf > a.back().perf) {
122  // keep unconditionally
123  a.push_back (op);
124  } else if (perf == a.back().perf) {
125  if (t < a.back ().t) {
126  a.back() = op;
127  } else {
128  return false;
129  }
130  } else {
131  int i;
132  // stricto sensu this should be a bissection
133  for (i = 0; i < a.size(); i++) {
134  if (a[i].perf >= perf) break;
135  }
136  assert (i < a.size());
137  if (t < a[i].t) {
138  if (a[i].perf == perf) {
139  a[i] = op;
140  } else {
141  a.insert (a.begin() + i, op);
142  }
143  } else {
144  return false;
145  }
146  }
147  { // remove non-optimal points from array
148  int i = a.size() - 1;
149  while (i > 0) {
150  if (a[i].t < a[i - 1].t)
151  a.erase (a.begin() + (i - 1));
152  i--;
153  }
154  }
155  return true;
156 }
157 
158 
160  const std::string & prefix)
161 {
162  int n_add = 0;
163  for (int i = 0; i < other.all_pts.size(); i++) {
164  const OperatingPoint & op = other.all_pts[i];
165  if (add (op.perf, op.t, prefix + op.key, op.cno))
166  n_add++;
167  }
168  return n_add;
169 }
170 
171 
172 
173 /// get time required to obtain a given performance measure
174 double OperatingPoints::t_for_perf (double perf) const
175 {
176  const std::vector<OperatingPoint> & a = optimal_pts;
177  if (perf > a.back().perf) return 1e50;
178  int i0 = -1, i1 = a.size() - 1;
179  while (i0 + 1 < i1) {
180  int imed = (i0 + i1 + 1) / 2;
181  if (a[imed].perf < perf) i0 = imed;
182  else i1 = imed;
183  }
184  return a[i1].t;
185 }
186 
187 
188 void OperatingPoints::all_to_gnuplot (const char *fname) const
189 {
190  FILE *f = fopen(fname, "w");
191  if (!f) {
192  fprintf (stderr, "cannot open %s", fname);
193  perror("");
194  abort();
195  }
196  for (int i = 0; i < all_pts.size(); i++) {
197  const OperatingPoint & op = all_pts[i];
198  fprintf (f, "%g %g %s\n", op.perf, op.t, op.key.c_str());
199  }
200  fclose(f);
201 }
202 
203 void OperatingPoints::optimal_to_gnuplot (const char *fname) const
204 {
205  FILE *f = fopen(fname, "w");
206  if (!f) {
207  fprintf (stderr, "cannot open %s", fname);
208  perror("");
209  abort();
210  }
211  double prev_perf = 0.0;
212  for (int i = 0; i < optimal_pts.size(); i++) {
213  const OperatingPoint & op = optimal_pts[i];
214  fprintf (f, "%g %g\n", prev_perf, op.t);
215  fprintf (f, "%g %g %s\n", op.perf, op.t, op.key.c_str());
216  prev_perf = op.perf;
217  }
218  fclose(f);
219 }
220 
221 void OperatingPoints::display (bool only_optimal) const
222 {
223  const std::vector<OperatingPoint> &pts =
224  only_optimal ? optimal_pts : all_pts;
225  printf("Tested %ld operating points, %ld ones are optimal:\n",
226  all_pts.size(), optimal_pts.size());
227 
228  for (int i = 0; i < pts.size(); i++) {
229  const OperatingPoint & op = pts[i];
230  const char *star = "";
231  if (!only_optimal) {
232  for (int j = 0; j < optimal_pts.size(); j++) {
233  if (op.cno == optimal_pts[j].cno) {
234  star = "*";
235  break;
236  }
237  }
238  }
239  printf ("cno=%ld key=%s perf=%.4f t=%.3f %s\n",
240  op.cno, op.key.c_str(), op.perf, op.t, star);
241  }
242 
243 }
244 
245 /***************************************************************
246  * ParameterSpace
247  ***************************************************************/
248 
249 ParameterSpace::ParameterSpace ():
250  verbose (1), n_experiments (500),
251  batchsize (1<<30), thread_over_batches (false)
252 {
253 }
254 
255 /* not keeping this constructor as inheritors will call the parent
256  initialize()
257  */
258 
259 #if 0
260 ParameterSpace::ParameterSpace (Index *index):
261  verbose (1), n_experiments (500),
262  batchsize (1<<30), thread_over_batches (false)
263 {
264  initialize(index);
265 }
266 #endif
267 
269 {
270  size_t n = 1;
271  for (int i = 0; i < parameter_ranges.size(); i++)
272  n *= parameter_ranges[i].values.size();
273  return n;
274 }
275 
276 /// get string representation of the combination
277 std::string ParameterSpace::combination_name (size_t cno) const {
278  char buf[1000], *wp = buf;
279  *wp = 0;
280  for (int i = 0; i < parameter_ranges.size(); i++) {
281  const ParameterRange & pr = parameter_ranges[i];
282  size_t j = cno % pr.values.size();
283  cno /= pr.values.size();
284  wp += snprintf (
285  wp, buf + 1000 - wp, "%s%s=%g", i == 0 ? "" : ",",
286  pr.name.c_str(), pr.values[j]);
287  }
288  return std::string (buf);
289 }
290 
291 
292 bool ParameterSpace::combination_ge (size_t c1, size_t c2) const
293 {
294  for (int i = 0; i < parameter_ranges.size(); i++) {
295  int nval = parameter_ranges[i].values.size();
296  size_t j1 = c1 % nval;
297  size_t j2 = c2 % nval;
298  if (!(j1 >= j2)) return false;
299  c1 /= nval;
300  c2 /= nval;
301  }
302  return true;
303 }
304 
305 
306 
307 #define DC(classname) \
308  const classname *ix = dynamic_cast<const classname *>(index)
309 
310 static void init_pq_ParameterRange (const ProductQuantizer & pq,
311  ParameterRange & pr)
312 {
313  if (pq.code_size % 4 == 0) {
314  // Polysemous not supported for code sizes that are not a
315  // multiple of 4
316  for (int i = 2; i <= pq.code_size * 8 / 2; i+= 2)
317  pr.values.push_back(i);
318  }
319  pr.values.push_back (pq.code_size * 8);
320 }
321 
323 {
324  parameter_ranges.push_back (ParameterRange ());
325  parameter_ranges.back ().name = name;
326  return parameter_ranges.back ();
327 }
328 
329 
330 /// initialize with reasonable parameters for the index
331 void ParameterSpace::initialize (const Index * index)
332 {
333  if (DC (IndexPreTransform)) {
334  index = ix->index;
335  }
336  if (DC (IndexRefineFlat)) {
337  ParameterRange & pr = add_range("k_factor_rf");
338  for (int i = 0; i <= 6; i++) {
339  pr.values.push_back (1 << i);
340  }
341  index = ix->base_index;
342  }
343  if (DC (IndexPreTransform)) {
344  index = ix->index;
345  }
346 
347  if (DC (IndexIVF)) {
348  {
349  ParameterRange & pr = add_range("nprobe");
350  for (int i = 0; i < 13; i++) {
351  size_t nprobe = 1 << i;
352  if (nprobe >= ix->nlist) break;
353  pr.values.push_back (nprobe);
354  }
355  }
356  }
357  if (DC (IndexPQ)) {
358  ParameterRange & pr = add_range("ht");
359  init_pq_ParameterRange (ix->pq, pr);
360  }
361  if (DC (IndexIVFPQ)) {
362  ParameterRange & pr = add_range("ht");
363  init_pq_ParameterRange (ix->pq, pr);
364 
365  const MultiIndexQuantizer *miq =
366  dynamic_cast<const MultiIndexQuantizer *> (ix->quantizer);
367  if (miq) {
368  ParameterRange & pr_max_codes = add_range("max_codes");
369  for (int i = 8; i < 20; i++) {
370  pr_max_codes.values.push_back (1 << i);
371  }
372  pr_max_codes.values.push_back (1.0 / 0.0);
373  }
374  }
375  if (DC (IndexIVFPQR)) {
376  ParameterRange & pr = add_range("k_factor");
377  for (int i = 0; i <= 6; i++) {
378  pr.values.push_back (1 << i);
379  }
380  }
381 }
382 
383 #undef DC
384 
385 // non-const version
386 #define DC(classname) classname *ix = dynamic_cast<classname *>(index)
387 
388 
389 /// set a combination of parameters on an index
390 void ParameterSpace::set_index_parameters (Index *index, size_t cno) const
391 {
392 
393  for (int i = 0; i < parameter_ranges.size(); i++) {
394  const ParameterRange & pr = parameter_ranges[i];
395  size_t j = cno % pr.values.size();
396  cno /= pr.values.size();
397  double val = pr.values [j];
398  set_index_parameter (index, pr.name, val);
399  }
400 }
401 
402 /// set a combination of parameters on an index
404  Index *index, const char *description_in) const
405 {
406  char description[strlen(description_in) + 1];
407  char *ptr;
408  memcpy (description, description_in, strlen(description_in) + 1);
409 
410  for (char *tok = strtok_r (description, " ,", &ptr);
411  tok;
412  tok = strtok_r (nullptr, " ,", &ptr)) {
413  char name[100];
414  double val;
415  int ret = sscanf (tok, "%100[^=]=%lf", name, &val);
416  FAISS_THROW_IF_NOT_FMT (
417  ret == 2, "could not interpret parameters %s", tok);
418  set_index_parameter (index, name, val);
419  }
420 
421 }
422 
424  Index * index, const std::string & name, double val) const
425 {
426  if (verbose > 1)
427  printf(" set %s=%g\n", name.c_str(), val);
428 
429  if (name == "verbose") {
430  index->verbose = int(val);
431  // and fall through to also enable it on sub-indexes
432  }
433  if (DC (IndexPreTransform)) {
434  index = ix->index;
435  }
436  if (DC (IndexShards)) {
437  // call on all sub-indexes
438  for (auto & shard_index : ix->shard_indexes) {
439  set_index_parameter (shard_index, name, val);
440  }
441  return;
442  }
443  if (name == "verbose") {
444  index->verbose = int(val);
445  // in case it was an IndexPreTransform
446  }
447  if (DC (IndexRefineFlat)) {
448  if (name == "k_factor_rf") {
449  ix->k_factor = int(val);
450  return;
451  }
452  index = ix->base_index;
453  }
454  if (DC (IndexPreTransform)) {
455  index = ix->index;
456  }
457  if (name == "verbose") {
458  index->verbose = int(val);
459  return; // last verbose that we could find
460  }
461  if (name == "nprobe") {
462  if ( DC(IndexIVF)) {
463  ix->nprobe = int(val);
464  return;
465  }
466  }
467  if (name == "ht") {
468  if (DC (IndexPQ)) {
469  if (val >= ix->pq.code_size * 8) {
470  ix->search_type = IndexPQ::ST_PQ;
471  } else {
472  ix->search_type = IndexPQ::ST_polysemous;
473  ix->polysemous_ht = int(val);
474  }
475  return;
476  } else if (DC (IndexIVFPQ)) {
477  if (val >= ix->pq.code_size * 8) {
478  ix->polysemous_ht = 0;
479  } else {
480  ix->polysemous_ht = int(val);
481  }
482  return;
483  }
484  }
485 
486  if (name == "k_factor") {
487  if (DC (IndexIVFPQR)) {
488  ix->k_factor = val;
489  return;
490  }
491  }
492  if (name == "max_codes") {
493  if (DC (IndexIVFPQ)) {
494  ix->max_codes = finite(val) ? size_t(val) : 0;
495  return;
496  }
497  }
498  FAISS_THROW_FMT ("ParameterSpace::set_index_parameter:"
499  "could not set parameter %s",
500  name.c_str());
501 }
502 
504 {
505  printf ("ParameterSpace, %ld parameters, %ld combinations:\n",
506  parameter_ranges.size (), n_combinations ());
507  for (int i = 0; i < parameter_ranges.size(); i++) {
508  const ParameterRange & pr = parameter_ranges[i];
509  printf (" %s: ", pr.name.c_str ());
510  char sep = '[';
511  for (int j = 0; j < pr.values.size(); j++) {
512  printf ("%c %g", sep, pr.values [j]);
513  sep = ',';
514  }
515  printf ("]\n");
516  }
517 }
518 
519 
520 
521 void ParameterSpace::update_bounds (size_t cno, const OperatingPoint & op,
522  double *upper_bound_perf,
523  double *lower_bound_t) const
524 {
525  if (combination_ge (cno, op.cno)) {
526  if (op.t > *lower_bound_t) *lower_bound_t = op.t;
527  }
528  if (combination_ge (op.cno, cno)) {
529  if (op.perf < *upper_bound_perf) *upper_bound_perf = op.perf;
530  }
531 }
532 
533 
534 
536  size_t nq, const float *xq,
537  const AutoTuneCriterion & crit,
538  OperatingPoints * ops) const
539 {
540  FAISS_THROW_IF_NOT_MSG (nq == crit.nq,
541  "criterion does not have the same nb of queries");
542 
543  size_t n_comb = n_combinations ();
544 
545  if (n_experiments == 0) {
546 
547  for (size_t cno = 0; cno < n_comb; cno++) {
548  set_index_parameters (index, cno);
549  std::vector<Index::idx_t> I(nq * crit.nnn);
550  std::vector<float> D(nq * crit.nnn);
551 
552  double t0 = getmillisecs ();
553  index->search (nq, xq, crit.nnn, D.data(), I.data());
554  double t_search = (getmillisecs() - t0) / 1e3;
555 
556  double perf = crit.evaluate (D.data(), I.data());
557 
558  bool keep = ops->add (perf, t_search, combination_name (cno), cno);
559 
560  if (verbose)
561  printf(" %ld/%ld: %s perf=%.3f t=%.3f s %s\n", cno, n_comb,
562  combination_name (cno).c_str(), perf, t_search,
563  keep ? "*" : "");
564  }
565  return;
566  }
567 
568  int n_exp = n_experiments;
569 
570  if (n_exp > n_comb) n_exp = n_comb;
571  FAISS_THROW_IF_NOT (n_comb == 1 || n_exp > 2);
572  std::vector<int> perm (n_comb);
573  // make sure the slowest and fastest experiment are run
574  perm[0] = 0;
575  if (n_comb > 1) {
576  perm[1] = n_comb - 1;
577  rand_perm (&perm[2], n_comb - 2, 1234);
578  for (int i = 2; i < perm.size(); i++) perm[i] ++;
579  }
580 
581  for (size_t xp = 0; xp < n_exp; xp++) {
582  size_t cno = perm[xp];
583 
584  if (verbose)
585  printf(" %ld/%d: cno=%ld %s ", xp, n_exp, cno,
586  combination_name (cno).c_str());
587 
588  {
589  double lower_bound_t = 0.0;
590  double upper_bound_perf = 1.0;
591  for (int i = 0; i < ops->all_pts.size(); i++) {
592  update_bounds (cno, ops->all_pts[i],
593  &upper_bound_perf, &lower_bound_t);
594  }
595  double best_t = ops->t_for_perf (upper_bound_perf);
596  if (verbose)
597  printf ("bounds [perf<=%.3f t>=%.3f] %s",
598  upper_bound_perf, lower_bound_t,
599  best_t <= lower_bound_t ? "skip\n" : "");
600  if (best_t <= lower_bound_t) continue;
601  }
602 
603  set_index_parameters (index, cno);
604  std::vector<Index::idx_t> I(nq * crit.nnn);
605  std::vector<float> D(nq * crit.nnn);
606 
607  double t0 = getmillisecs ();
608 
609  if (thread_over_batches) {
610 #pragma omp parallel for
611  for (size_t q0 = 0; q0 < nq; q0 += batchsize) {
612  size_t q1 = q0 + batchsize;
613  if (q1 > nq) q1 = nq;
614  index->search (q1 - q0, xq + q0 * index->d,
615  crit.nnn,
616  D.data() + q0 * crit.nnn,
617  I.data() + q0 * crit.nnn);
618  }
619  } else {
620  for (size_t q0 = 0; q0 < nq; q0 += batchsize) {
621  size_t q1 = q0 + batchsize;
622  if (q1 > nq) q1 = nq;
623  index->search (q1 - q0, xq + q0 * index->d,
624  crit.nnn,
625  D.data() + q0 * crit.nnn,
626  I.data() + q0 * crit.nnn);
627  }
628  }
629 
630  double t_search = (getmillisecs() - t0) / 1e3;
631 
632  double perf = crit.evaluate (D.data(), I.data());
633 
634  bool keep = ops->add (perf, t_search, combination_name (cno), cno);
635 
636  if (verbose)
637  printf(" perf %.3f t %.3f %s\n", perf, t_search,
638  keep ? "*" : "");
639  }
640 }
641 
642 /***************************************************************
643  * index_factory
644  ***************************************************************/
645 
646 namespace {
647 
648 struct VTChain {
649  std::vector<VectorTransform *> chain;
650  ~VTChain () {
651  for (int i = 0; i < chain.size(); i++) {
652  delete chain[i];
653  }
654  }
655 };
656 
657 
658 /// what kind of training does this coarse quantizer require?
659 char get_trains_alone(const Index *coarse_quantizer) {
660  return
661  dynamic_cast<const MultiIndexQuantizer*>(coarse_quantizer) ? 1 :
662  0;
663 }
664 
665 
666 }
667 
668 Index *index_factory (int d, const char *description_in, MetricType metric)
669 {
670  VTChain vts;
671  Index *coarse_quantizer = nullptr;
672  Index *index = nullptr;
673  bool add_idmap = false;
674  bool make_IndexRefineFlat = false;
675 
676  ScopeDeleter1<Index> del_coarse_quantizer, del_index;
677 
678  char description[strlen(description_in) + 1];
679  char *ptr;
680  memcpy (description, description_in, strlen(description_in) + 1);
681 
682  int ncentroids = -1;
683 
684  for (char *tok = strtok_r (description, " ,", &ptr);
685  tok;
686  tok = strtok_r (nullptr, " ,", &ptr)) {
687  int d_out, opq_M, nbit, M, M2;
688  char option[100];
689  std::string stok(tok);
690 
691  // to avoid mem leaks with exceptions:
692  // do all tests before any instanciation
693 
694  VectorTransform *vt_1 = nullptr;
695  Index *coarse_quantizer_1 = nullptr;
696  Index *index_1 = nullptr;
697 
698  // VectorTransforms
699  if (sscanf (tok, "PCA%d", &d_out) == 1) {
700  vt_1 = new PCAMatrix (d, d_out);
701  d = d_out;
702  } else if (sscanf (tok, "PCAR%d", &d_out) == 1) {
703  vt_1 = new PCAMatrix (d, d_out, 0, true);
704  d = d_out;
705  } else if (sscanf (tok, "PCAW%d", &d_out) == 1) {
706  vt_1 = new PCAMatrix (d, d_out, -0.5, false);
707  d = d_out;
708  } else if (sscanf (tok, "PCAWR%d", &d_out) == 1) {
709  vt_1 = new PCAMatrix (d, d_out, -0.5, true);
710  d = d_out;
711  } else if (sscanf (tok, "OPQ%d_%d", &opq_M, &d_out) == 2) {
712  vt_1 = new OPQMatrix (d, opq_M, d_out);
713  d = d_out;
714  } else if (sscanf (tok, "OPQ%d", &opq_M) == 1) {
715  vt_1 = new OPQMatrix (d, opq_M);
716  } else if (stok == "L2norm") {
717  vt_1 = new NormalizationTransform (d, 2.0);
718 
719 
720  } else if (!coarse_quantizer &&
721  sscanf (tok, "IVF%d", &ncentroids) == 1) {
722  if (metric == METRIC_L2) {
723  coarse_quantizer_1 = new IndexFlatL2 (d);
724  } else { // if (metric == METRIC_IP)
725  coarse_quantizer_1 = new IndexFlatIP (d);
726  }
727  } else if (!coarse_quantizer && sscanf (tok, "IMI2x%d", &nbit) == 1) {
728  FAISS_THROW_IF_NOT_MSG (metric == METRIC_L2,
729  "MultiIndex not implemented for inner prod search");
730  coarse_quantizer_1 = new MultiIndexQuantizer (d, 2, nbit);
731  ncentroids = 1 << (2 * nbit);
732  } else if (stok == "IDMap") {
733  add_idmap = true;
734 
735  // IVFs
736  } else if (!index && stok == "Flat") {
737  if (coarse_quantizer) {
738  // if there was an IVF in front, then it is an IVFFlat
739  IndexIVF *index_ivf = new IndexIVFFlat (
740  coarse_quantizer, d, ncentroids, metric);
741  index_ivf->quantizer_trains_alone =
742  get_trains_alone (coarse_quantizer);
743  index_ivf->cp.spherical = metric == METRIC_INNER_PRODUCT;
744  del_coarse_quantizer.release ();
745  index_ivf->own_fields = true;
746  index_1 = index_ivf;
747  } else {
748  index_1 = new IndexFlat (d, metric);
749  }
750  } else if (!index && (stok == "SQ8" || stok == "SQ4")) {
752  stok == "SQ8" ? ScalarQuantizer::QT_8bit :
753  stok == "SQ4" ? ScalarQuantizer::QT_4bit :
755  if (coarse_quantizer) {
756  IndexIVFScalarQuantizer *index_ivf =
758  coarse_quantizer, d, ncentroids, qt, metric);
759  index_ivf->quantizer_trains_alone =
760  get_trains_alone (coarse_quantizer);
761  del_coarse_quantizer.release ();
762  index_ivf->own_fields = true;
763  index_1 = index_ivf;
764  } else {
765  index_1 = new IndexScalarQuantizer (d, qt, metric);
766  }
767  } else if (!index && sscanf (tok, "PQ%d+%d", &M, &M2) == 2) {
768  FAISS_THROW_IF_NOT_MSG(coarse_quantizer,
769  "PQ with + works only with an IVF");
770  FAISS_THROW_IF_NOT_MSG(metric == METRIC_L2,
771  "IVFPQR not implemented for inner product search");
772  IndexIVFPQR *index_ivf = new IndexIVFPQR (
773  coarse_quantizer, d, ncentroids, M, 8, M2, 8);
774  index_ivf->quantizer_trains_alone =
775  get_trains_alone (coarse_quantizer);
776  del_coarse_quantizer.release ();
777  index_ivf->own_fields = true;
778  index_1 = index_ivf;
779  } else if (!index && sscanf (tok, "PQ%d%10s", &M, option) == 2) {
780  std::string soption = option;
781  // np to disable polysemous trainign
782  FAISS_THROW_IF_NOT(soption == "" || soption == "np");
783  if (coarse_quantizer) {
784  IndexIVFPQ *index_ivf = new IndexIVFPQ (
785  coarse_quantizer, d, ncentroids, M, 8);
786  index_ivf->quantizer_trains_alone =
787  get_trains_alone (coarse_quantizer);
788  index_ivf->metric_type = metric;
789  index_ivf->cp.spherical = metric == METRIC_INNER_PRODUCT;
790  del_coarse_quantizer.release ();
791  index_ivf->own_fields = true;
792  index_ivf->do_polysemous_training = soption != "np";
793  index_1 = index_ivf;
794  } else {
795  IndexPQ *index_pq = new IndexPQ (d, M, 8, metric);
796  index_pq->do_polysemous_training = soption != "np";
797  index_1 = index_pq;
798  }
799 
800  } else if (stok == "RFlat") {
801  make_IndexRefineFlat = true;
802  } else {
803  FAISS_THROW_FMT( "could not parse token \"%s\" in %s\n",
804  tok, description_in);
805  }
806 
807  if (index_1 && add_idmap) {
808  IndexIDMap *idmap = new IndexIDMap(index_1);
809  del_index.set (idmap);
810  idmap->own_fields = true;
811  index_1 = idmap;
812  add_idmap = false;
813  }
814 
815  if (vt_1) {
816  vts.chain.push_back (vt_1);
817  }
818 
819  if (coarse_quantizer_1) {
820  coarse_quantizer = coarse_quantizer_1;
821  del_coarse_quantizer.set (coarse_quantizer);
822  }
823 
824  if (index_1) {
825  index = index_1;
826  del_index.set (index);
827  }
828  }
829 
830  FAISS_THROW_IF_NOT_FMT(index, "descrption %s did not generate an index",
831  description_in);
832 
833  // nothing can go wrong now
834  del_index.release ();
835  del_coarse_quantizer.release ();
836 
837  if (add_idmap) {
838  fprintf(stderr, "index_factory: WARNING: "
839  "IDMap option not used\n");
840  }
841 
842  if (vts.chain.size() > 0) {
843  IndexPreTransform *index_pt = new IndexPreTransform (index);
844  index_pt->own_fields = true;
845  // add from back
846  while (vts.chain.size() > 0) {
847  index_pt->prepend_transform (vts.chain.back());
848  vts.chain.pop_back ();
849  }
850  index = index_pt;
851  }
852 
853  if (make_IndexRefineFlat) {
854  IndexRefineFlat *index_rf = new IndexRefineFlat (index);
855  index_rf->own_fields = true;
856  index = index_rf;
857  }
858 
859  return index;
860 }
861 
862 
863 
864 
865 }; // namespace faiss
void explore(Index *index, size_t nq, const float *xq, const AutoTuneCriterion &crit, OperatingPoints *ops) const
Definition: AutoTune.cpp:535
std::vector< ParameterRange > parameter_ranges
all tunable parameters
Definition: AutoTune.h:134
std::string key
key that identifies this op pt
Definition: AutoTune.h:89
long cno
integer identifer
Definition: AutoTune.h:90
bool do_polysemous_training
false = standard PQ
Definition: IndexPQ.h:69
char quantizer_trains_alone
Definition: IndexIVF.h:56
void display(bool only_optimal=true) const
easy-to-read output
Definition: AutoTune.cpp:221
double perf
performance measure (output of a Criterion)
Definition: AutoTune.h:87
double t_for_perf(double perf) const
get time required to obtain a given performance measure
Definition: AutoTune.cpp:174
idx_t nnn
nb of NNs that the query should request
Definition: AutoTune.h:29
bool add(double perf, double t, const std::string &key, size_t cno=0)
add a performance measure. Return whether it is an optimal point
Definition: AutoTune.cpp:112
bool do_polysemous_training
reorder PQ centroids after training?
Definition: IndexIVFPQ.h:34
size_t batchsize
maximum number of queries to submit at a time.
Definition: AutoTune.h:145
virtual double evaluate(const float *D, const idx_t *I) const =0
idx_t nq
nb of queries this criterion is evaluated on
Definition: AutoTune.h:28
std::vector< OperatingPoint > optimal_pts
optimal operating points, sorted by perf
Definition: AutoTune.h:98
void set_groundtruth(int gt_nnn, const float *gt_D_in, const idx_t *gt_I_in)
Definition: AutoTune.cpp:36
ParameterRange & add_range(const char *name)
add a new parameter
Definition: AutoTune.cpp:322
idx_t gt_nnn
nb of GT NNs required to evaluate crterion
Definition: AutoTune.h:30
void all_to_gnuplot(const char *fname) const
output to a format easy to digest by gnuplot
Definition: AutoTune.cpp:188
bool own_fields
should the base index be deallocated?
Definition: IndexFlat.h:110
int d
vector dimension
Definition: Index.h:64
size_t code_size
byte per indexed vector
ClusteringParameters cp
to override default clustering params
Definition: IndexIVF.h:59
bool own_fields
whether object owns the quantizer
Definition: IndexIVF.h:57
std::vector< OperatingPoint > all_pts
all operating points
Definition: AutoTune.h:95
size_t ranklist_intersection_size(size_t k1, const long *v1, size_t k2, const long *v2_in)
Definition: utils.cpp:1574
bool verbose
verbosity level
Definition: Index.h:66
double getmillisecs()
ms elapsed since some arbitrary epoch
Definition: utils.cpp:74
std::vector< float > gt_D
Ground-truth distances (size nq * gt_nnn)
Definition: AutoTune.h:32
std::string combination_name(size_t cno) const
get string representation of the combination
Definition: AutoTune.cpp:277
virtual void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const =0
void update_bounds(size_t cno, const OperatingPoint &op, double *upper_bound_perf, double *lower_bound_t) const
Definition: AutoTune.cpp:521
bool own_fields
! the sub-index
virtual void initialize(const Index *index)
initialize with reasonable parameters for the index
Definition: AutoTune.cpp:331
int verbose
verbosity during exploration
Definition: AutoTune.h:139
int merge_with(const OperatingPoints &other, const std::string &prefix="")
add operating points from other to this, with a prefix to the keys
Definition: AutoTune.cpp:159
virtual void set_index_parameter(Index *index, const std::string &name, double val) const
set one of the parameters
Definition: AutoTune.cpp:423
size_t n_combinations() const
nb of combinations, = product of values sizes
Definition: AutoTune.cpp:268
MetricType metric_type
type of metric this index uses for search
Definition: Index.h:72
void set_index_parameters(Index *index, size_t cno) const
set a combination of parameters on an index
Definition: AutoTune.cpp:390
asymmetric product quantizer (default)
Definition: IndexPQ.h:76
void display() const
print a description on stdout
Definition: AutoTune.cpp:503
HE filter (using ht) + PQ combination.
Definition: IndexPQ.h:80
bool combination_ge(size_t c1, size_t c2) const
returns whether combinations c1 &gt;= c2 in the tuple sense
Definition: AutoTune.cpp:292
bool spherical
do we want normalized centroids?
Definition: Clustering.h:29
possible values of a parameter, sorted from least to most expensive/accurate
Definition: AutoTune.h:125
Index * index_factory(int d, const char *description_in, MetricType metric)
Definition: AutoTune.cpp:668
int n_experiments
nb of experiments during optimization (0 = try all combinations)
Definition: AutoTune.h:142
std::vector< idx_t > gt_I
Ground-truth indexes (size nq * gt_nnn)
Definition: AutoTune.h:33
double t
corresponding execution time (ms)
Definition: AutoTune.h:88
MetricType
Some algorithms support both an inner product version and a L2 search version.
Definition: Index.h:43
bool own_fields
! the sub-index
Definition: MetaIndexes.h:28