mirror of
https://github.com/facebookresearch/faiss.git
synced 2025-06-03 21:54:02 +08:00
0df3f646fc
various bugfixes from github issues kmean with some frozen centroids GPU better tiling for large flat datasets default AVX for vector ops
887 lines
129 KiB
HTML
887 lines
129 KiB
HTML
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
|
|
<html xmlns="http://www.w3.org/1999/xhtml">
|
|
<head>
|
|
<meta http-equiv="Content-Type" content="text/xhtml;charset=UTF-8"/>
|
|
<meta http-equiv="X-UA-Compatible" content="IE=9"/>
|
|
<meta name="generator" content="Doxygen 1.8.5"/>
|
|
<title>Faiss: /data/users/matthijs/github_faiss/faiss/MetaIndexes.cpp Source File</title>
|
|
<link href="tabs.css" rel="stylesheet" type="text/css"/>
|
|
<script type="text/javascript" src="jquery.js"></script>
|
|
<script type="text/javascript" src="dynsections.js"></script>
|
|
<link href="search/search.css" rel="stylesheet" type="text/css"/>
|
|
<script type="text/javascript" src="search/search.js"></script>
|
|
<script type="text/javascript">
|
|
$(document).ready(function() { searchBox.OnSelectItem(0); });
|
|
</script>
|
|
<link href="doxygen.css" rel="stylesheet" type="text/css" />
|
|
</head>
|
|
<body>
|
|
<div id="top"><!-- do not remove this div, it is closed by doxygen! -->
|
|
<div id="titlearea">
|
|
<table cellspacing="0" cellpadding="0">
|
|
<tbody>
|
|
<tr style="height: 56px;">
|
|
<td style="padding-left: 0.5em;">
|
|
<div id="projectname">Faiss
|
|
</div>
|
|
</td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
</div>
|
|
<!-- end header part -->
|
|
<!-- Generated by Doxygen 1.8.5 -->
|
|
<script type="text/javascript">
|
|
var searchBox = new SearchBox("searchBox", "search",false,'Search');
|
|
</script>
|
|
<div id="navrow1" class="tabs">
|
|
<ul class="tablist">
|
|
<li><a href="index.html"><span>Main Page</span></a></li>
|
|
<li><a href="namespaces.html"><span>Namespaces</span></a></li>
|
|
<li><a href="annotated.html"><span>Classes</span></a></li>
|
|
<li class="current"><a href="files.html"><span>Files</span></a></li>
|
|
<li>
|
|
<div id="MSearchBox" class="MSearchBoxInactive">
|
|
<span class="left">
|
|
<img id="MSearchSelect" src="search/mag_sel.png"
|
|
onmouseover="return searchBox.OnSearchSelectShow()"
|
|
onmouseout="return searchBox.OnSearchSelectHide()"
|
|
alt=""/>
|
|
<input type="text" id="MSearchField" value="Search" accesskey="S"
|
|
onfocus="searchBox.OnSearchFieldFocus(true)"
|
|
onblur="searchBox.OnSearchFieldFocus(false)"
|
|
onkeyup="searchBox.OnSearchFieldChange(event)"/>
|
|
</span><span class="right">
|
|
<a id="MSearchClose" href="javascript:searchBox.CloseResultsWindow()"><img id="MSearchCloseImg" border="0" src="search/close.png" alt=""/></a>
|
|
</span>
|
|
</div>
|
|
</li>
|
|
</ul>
|
|
</div>
|
|
<div id="navrow2" class="tabs2">
|
|
<ul class="tablist">
|
|
<li><a href="files.html"><span>File List</span></a></li>
|
|
</ul>
|
|
</div>
|
|
</div><!-- top -->
|
|
<!-- window showing the filter options -->
|
|
<div id="MSearchSelectWindow"
|
|
onmouseover="return searchBox.OnSearchSelectShow()"
|
|
onmouseout="return searchBox.OnSearchSelectHide()"
|
|
onkeydown="return searchBox.OnSearchSelectKey(event)">
|
|
<a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(0)"><span class="SelectionMark"> </span>All</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(1)"><span class="SelectionMark"> </span>Classes</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(2)"><span class="SelectionMark"> </span>Namespaces</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(3)"><span class="SelectionMark"> </span>Functions</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(4)"><span class="SelectionMark"> </span>Variables</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(5)"><span class="SelectionMark"> </span>Typedefs</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(6)"><span class="SelectionMark"> </span>Enumerations</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(7)"><span class="SelectionMark"> </span>Enumerator</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(8)"><span class="SelectionMark"> </span>Friends</a></div>
|
|
|
|
<!-- iframe showing the search results (closed by default) -->
|
|
<div id="MSearchResultsWindow">
|
|
<iframe src="javascript:void(0)" frameborder="0"
|
|
name="MSearchResults" id="MSearchResults">
|
|
</iframe>
|
|
</div>
|
|
|
|
<div class="header">
|
|
<div class="headertitle">
|
|
<div class="title">/data/users/matthijs/github_faiss/faiss/MetaIndexes.cpp</div> </div>
|
|
</div><!--header-->
|
|
<div class="contents">
|
|
<div class="fragment"><div class="line"><a name="l00001"></a><span class="lineno"> 1</span> <span class="comment">/**</span></div>
|
|
<div class="line"><a name="l00002"></a><span class="lineno"> 2</span> <span class="comment"> * Copyright (c) 2015-present, Facebook, Inc.</span></div>
|
|
<div class="line"><a name="l00003"></a><span class="lineno"> 3</span> <span class="comment"> * All rights reserved.</span></div>
|
|
<div class="line"><a name="l00004"></a><span class="lineno"> 4</span> <span class="comment"> *</span></div>
|
|
<div class="line"><a name="l00005"></a><span class="lineno"> 5</span> <span class="comment"> * This source code is licensed under the BSD+Patents license found in the</span></div>
|
|
<div class="line"><a name="l00006"></a><span class="lineno"> 6</span> <span class="comment"> * LICENSE file in the root directory of this source tree.</span></div>
|
|
<div class="line"><a name="l00007"></a><span class="lineno"> 7</span> <span class="comment"> */</span></div>
|
|
<div class="line"><a name="l00008"></a><span class="lineno"> 8</span> </div>
|
|
<div class="line"><a name="l00009"></a><span class="lineno"> 9</span> <span class="comment">// Copyright 2004-present Facebook. All Rights Reserved</span></div>
|
|
<div class="line"><a name="l00010"></a><span class="lineno"> 10</span> <span class="comment">// -*- c++ -*-</span></div>
|
|
<div class="line"><a name="l00011"></a><span class="lineno"> 11</span> </div>
|
|
<div class="line"><a name="l00012"></a><span class="lineno"> 12</span> <span class="preprocessor">#include "MetaIndexes.h"</span></div>
|
|
<div class="line"><a name="l00013"></a><span class="lineno"> 13</span> </div>
|
|
<div class="line"><a name="l00014"></a><span class="lineno"> 14</span> <span class="preprocessor">#include <pthread.h></span></div>
|
|
<div class="line"><a name="l00015"></a><span class="lineno"> 15</span> </div>
|
|
<div class="line"><a name="l00016"></a><span class="lineno"> 16</span> <span class="preprocessor">#include <cstdio></span></div>
|
|
<div class="line"><a name="l00017"></a><span class="lineno"> 17</span> </div>
|
|
<div class="line"><a name="l00018"></a><span class="lineno"> 18</span> <span class="preprocessor">#include "FaissAssert.h"</span></div>
|
|
<div class="line"><a name="l00019"></a><span class="lineno"> 19</span> <span class="preprocessor">#include "Heap.h"</span></div>
|
|
<div class="line"><a name="l00020"></a><span class="lineno"> 20</span> <span class="preprocessor">#include "AuxIndexStructures.h"</span></div>
|
|
<div class="line"><a name="l00021"></a><span class="lineno"> 21</span> </div>
|
|
<div class="line"><a name="l00022"></a><span class="lineno"> 22</span> </div>
|
|
<div class="line"><a name="l00023"></a><span class="lineno"> 23</span> <span class="keyword">namespace </span>faiss {</div>
|
|
<div class="line"><a name="l00024"></a><span class="lineno"> 24</span> </div>
|
|
<div class="line"><a name="l00025"></a><span class="lineno"> 25</span> <span class="comment">/*****************************************************</span></div>
|
|
<div class="line"><a name="l00026"></a><span class="lineno"> 26</span> <span class="comment"> * IndexIDMap implementation</span></div>
|
|
<div class="line"><a name="l00027"></a><span class="lineno"> 27</span> <span class="comment"> *******************************************************/</span></div>
|
|
<div class="line"><a name="l00028"></a><span class="lineno"> 28</span> </div>
|
|
<div class="line"><a name="l00029"></a><span class="lineno"> 29</span> IndexIDMap::IndexIDMap (Index *index):</div>
|
|
<div class="line"><a name="l00030"></a><span class="lineno"> 30</span>  index (index),</div>
|
|
<div class="line"><a name="l00031"></a><span class="lineno"> 31</span>  own_fields (false)</div>
|
|
<div class="line"><a name="l00032"></a><span class="lineno"> 32</span> {</div>
|
|
<div class="line"><a name="l00033"></a><span class="lineno"> 33</span>  FAISS_THROW_IF_NOT_MSG (index->ntotal == 0, <span class="stringliteral">"index must be empty on input"</span>);</div>
|
|
<div class="line"><a name="l00034"></a><span class="lineno"> 34</span>  <a class="code" href="structfaiss_1_1Index.html#a6e92732617c4dbe364e7678dd8773a7f">is_trained</a> = index->is_trained;</div>
|
|
<div class="line"><a name="l00035"></a><span class="lineno"> 35</span>  <a class="code" href="structfaiss_1_1Index.html#a8e18f641854b2bde83ecff0a2f9a6f4e">metric_type</a> = index->metric_type;</div>
|
|
<div class="line"><a name="l00036"></a><span class="lineno"> 36</span>  <a class="code" href="structfaiss_1_1Index.html#a5590d847c5c2b958affd2a05e58a6f23">verbose</a> = index->verbose;</div>
|
|
<div class="line"><a name="l00037"></a><span class="lineno"> 37</span>  <a class="code" href="structfaiss_1_1Index.html#a2a002388d2c081c2dbab8508dcefe73d">d</a> = index->d;</div>
|
|
<div class="line"><a name="l00038"></a><span class="lineno"> 38</span> }</div>
|
|
<div class="line"><a name="l00039"></a><span class="lineno"> 39</span> </div>
|
|
<div class="line"><a name="l00040"></a><span class="lineno"><a class="line" href="structfaiss_1_1IndexIDMap.html#a16cce6c1e83d0d5f069c9b0399896450"> 40</a></span> <span class="keywordtype">void</span> <a class="code" href="structfaiss_1_1IndexIDMap.html#a16cce6c1e83d0d5f069c9b0399896450">IndexIDMap::add</a> (<a class="code" href="structfaiss_1_1Index.html#a040c6aed1f224f3ea7bf58eebc0c31a4">idx_t</a>, <span class="keyword">const</span> <span class="keywordtype">float</span> *)</div>
|
|
<div class="line"><a name="l00041"></a><span class="lineno"> 41</span> {</div>
|
|
<div class="line"><a name="l00042"></a><span class="lineno"> 42</span>  FAISS_THROW_MSG (<span class="stringliteral">"add does not make sense with IndexIDMap, "</span></div>
|
|
<div class="line"><a name="l00043"></a><span class="lineno"> 43</span>  <span class="stringliteral">"use add_with_ids"</span>);</div>
|
|
<div class="line"><a name="l00044"></a><span class="lineno"> 44</span> }</div>
|
|
<div class="line"><a name="l00045"></a><span class="lineno"> 45</span> </div>
|
|
<div class="line"><a name="l00046"></a><span class="lineno"> 46</span> </div>
|
|
<div class="line"><a name="l00047"></a><span class="lineno"><a class="line" href="structfaiss_1_1IndexIDMap.html#ac873cb03aa323a06dca5d6f3af5fe559"> 47</a></span> <span class="keywordtype">void</span> <a class="code" href="structfaiss_1_1IndexIDMap.html#ac873cb03aa323a06dca5d6f3af5fe559">IndexIDMap::train</a> (<a class="code" href="structfaiss_1_1Index.html#a040c6aed1f224f3ea7bf58eebc0c31a4">idx_t</a> n, <span class="keyword">const</span> <span class="keywordtype">float</span> *x)</div>
|
|
<div class="line"><a name="l00048"></a><span class="lineno"> 48</span> {</div>
|
|
<div class="line"><a name="l00049"></a><span class="lineno"> 49</span>  index-><a class="code" href="structfaiss_1_1Index.html#a1ffe916c958605c38b0b1bfad42485e4">train</a> (n, x);</div>
|
|
<div class="line"><a name="l00050"></a><span class="lineno"> 50</span>  <a class="code" href="structfaiss_1_1Index.html#a6e92732617c4dbe364e7678dd8773a7f">is_trained</a> = index-><a class="code" href="structfaiss_1_1Index.html#a6e92732617c4dbe364e7678dd8773a7f">is_trained</a>;</div>
|
|
<div class="line"><a name="l00051"></a><span class="lineno"> 51</span> }</div>
|
|
<div class="line"><a name="l00052"></a><span class="lineno"> 52</span> </div>
|
|
<div class="line"><a name="l00053"></a><span class="lineno"><a class="line" href="structfaiss_1_1IndexIDMap.html#a70aca5b0f665c5a109d5dadf1e13b448"> 53</a></span> <span class="keywordtype">void</span> <a class="code" href="structfaiss_1_1IndexIDMap.html#a70aca5b0f665c5a109d5dadf1e13b448">IndexIDMap::reset</a> ()</div>
|
|
<div class="line"><a name="l00054"></a><span class="lineno"> 54</span> {</div>
|
|
<div class="line"><a name="l00055"></a><span class="lineno"> 55</span>  index-><a class="code" href="structfaiss_1_1Index.html#a849361f5f0ab0aba8d419c86f2594191">reset</a> ();</div>
|
|
<div class="line"><a name="l00056"></a><span class="lineno"> 56</span>  <a class="code" href="structfaiss_1_1Index.html#a6970683faa021b7a6f1a0865c0d4eccd">ntotal</a> = 0;</div>
|
|
<div class="line"><a name="l00057"></a><span class="lineno"> 57</span> }</div>
|
|
<div class="line"><a name="l00058"></a><span class="lineno"> 58</span> </div>
|
|
<div class="line"><a name="l00059"></a><span class="lineno"> 59</span> </div>
|
|
<div class="line"><a name="l00060"></a><span class="lineno"><a class="line" href="structfaiss_1_1IndexIDMap.html#aeace75e53648c052d05c357b744e090b"> 60</a></span> <span class="keywordtype">void</span> <a class="code" href="structfaiss_1_1IndexIDMap.html#aeace75e53648c052d05c357b744e090b">IndexIDMap::add_with_ids</a> (<a class="code" href="structfaiss_1_1Index.html#a040c6aed1f224f3ea7bf58eebc0c31a4">idx_t</a> n, <span class="keyword">const</span> <span class="keywordtype">float</span> * x, <span class="keyword">const</span> <span class="keywordtype">long</span> *xids)</div>
|
|
<div class="line"><a name="l00061"></a><span class="lineno"> 61</span> {</div>
|
|
<div class="line"><a name="l00062"></a><span class="lineno"> 62</span>  index-><a class="code" href="structfaiss_1_1Index.html#a1b5e9ac70adbce0897dd6c8276ad96f2">add</a> (n, x);</div>
|
|
<div class="line"><a name="l00063"></a><span class="lineno"> 63</span>  <span class="keywordflow">for</span> (<a class="code" href="structfaiss_1_1Index.html#a040c6aed1f224f3ea7bf58eebc0c31a4">idx_t</a> i = 0; i < n; i++)</div>
|
|
<div class="line"><a name="l00064"></a><span class="lineno"> 64</span>  <a class="code" href="structfaiss_1_1IndexIDMap.html#ad1cc5658f1879f6a2b00d76cca58cf3b">id_map</a>.push_back (xids[i]);</div>
|
|
<div class="line"><a name="l00065"></a><span class="lineno"> 65</span>  <a class="code" href="structfaiss_1_1Index.html#a6970683faa021b7a6f1a0865c0d4eccd">ntotal</a> = index-><a class="code" href="structfaiss_1_1Index.html#a6970683faa021b7a6f1a0865c0d4eccd">ntotal</a>;</div>
|
|
<div class="line"><a name="l00066"></a><span class="lineno"> 66</span> }</div>
|
|
<div class="line"><a name="l00067"></a><span class="lineno"> 67</span> </div>
|
|
<div class="line"><a name="l00068"></a><span class="lineno"> 68</span> </div>
|
|
<div class="line"><a name="l00069"></a><span class="lineno"><a class="line" href="structfaiss_1_1IndexIDMap.html#a2f36fa9150a465b48952afbba856e2a1"> 69</a></span> <span class="keywordtype">void</span> <a class="code" href="structfaiss_1_1IndexIDMap.html#a2f36fa9150a465b48952afbba856e2a1">IndexIDMap::search</a> (<a class="code" href="structfaiss_1_1Index.html#a040c6aed1f224f3ea7bf58eebc0c31a4">idx_t</a> n, <span class="keyword">const</span> <span class="keywordtype">float</span> *x, <a class="code" href="structfaiss_1_1Index.html#a040c6aed1f224f3ea7bf58eebc0c31a4">idx_t</a> k,</div>
|
|
<div class="line"><a name="l00070"></a><span class="lineno"> 70</span>  <span class="keywordtype">float</span> *distances, <a class="code" href="structfaiss_1_1Index.html#a040c6aed1f224f3ea7bf58eebc0c31a4">idx_t</a> *labels)<span class="keyword"> const</span></div>
|
|
<div class="line"><a name="l00071"></a><span class="lineno"> 71</span> <span class="keyword"></span>{</div>
|
|
<div class="line"><a name="l00072"></a><span class="lineno"> 72</span>  index-><a class="code" href="structfaiss_1_1Index.html#aced51b1ebc33c47ab3ae15ea906559a7">search</a> (n, x, k, distances, labels);</div>
|
|
<div class="line"><a name="l00073"></a><span class="lineno"> 73</span>  <a class="code" href="structfaiss_1_1Index.html#a040c6aed1f224f3ea7bf58eebc0c31a4">idx_t</a> *li = labels;</div>
|
|
<div class="line"><a name="l00074"></a><span class="lineno"> 74</span>  <span class="keywordflow">for</span> (<a class="code" href="structfaiss_1_1Index.html#a040c6aed1f224f3ea7bf58eebc0c31a4">idx_t</a> i = 0; i < n * k; i++) {</div>
|
|
<div class="line"><a name="l00075"></a><span class="lineno"> 75</span>  li[i] = li[i] < 0 ? li[i] : <a class="code" href="structfaiss_1_1IndexIDMap.html#ad1cc5658f1879f6a2b00d76cca58cf3b">id_map</a>[li[i]];</div>
|
|
<div class="line"><a name="l00076"></a><span class="lineno"> 76</span>  }</div>
|
|
<div class="line"><a name="l00077"></a><span class="lineno"> 77</span> }</div>
|
|
<div class="line"><a name="l00078"></a><span class="lineno"> 78</span> </div>
|
|
<div class="line"><a name="l00079"></a><span class="lineno"> 79</span> </div>
|
|
<div class="line"><a name="l00080"></a><span class="lineno"><a class="line" href="structfaiss_1_1IndexIDMap.html#a7dbeaf9d44d6783d371f12405e8b7f40"> 80</a></span> <span class="keywordtype">void</span> <a class="code" href="structfaiss_1_1IndexIDMap.html#a7dbeaf9d44d6783d371f12405e8b7f40">IndexIDMap::range_search</a> (<a class="code" href="structfaiss_1_1Index.html#a040c6aed1f224f3ea7bf58eebc0c31a4">idx_t</a> n, <span class="keyword">const</span> <span class="keywordtype">float</span> *x, <span class="keywordtype">float</span> radius,</div>
|
|
<div class="line"><a name="l00081"></a><span class="lineno"> 81</span>  <a class="code" href="structfaiss_1_1RangeSearchResult.html">RangeSearchResult</a> *result)<span class="keyword"> const</span></div>
|
|
<div class="line"><a name="l00082"></a><span class="lineno"> 82</span> <span class="keyword"></span>{</div>
|
|
<div class="line"><a name="l00083"></a><span class="lineno"> 83</span>  index-><a class="code" href="structfaiss_1_1Index.html#a236897832a58dbb205e2d662c1929fb1">range_search</a>(n, x, radius, result);</div>
|
|
<div class="line"><a name="l00084"></a><span class="lineno"> 84</span>  <span class="keywordflow">for</span> (<a class="code" href="structfaiss_1_1Index.html#a040c6aed1f224f3ea7bf58eebc0c31a4">idx_t</a> i = 0; i < result-><a class="code" href="structfaiss_1_1RangeSearchResult.html#aef57f9db99ca470c01bcdce0366cb6ff">lims</a>[result-><a class="code" href="structfaiss_1_1RangeSearchResult.html#a29269b98c33af874d379ca4f62408b90">nq</a>]; i++) {</div>
|
|
<div class="line"><a name="l00085"></a><span class="lineno"> 85</span>  result-><a class="code" href="structfaiss_1_1RangeSearchResult.html#aa8ff848474a4cc16c5464cc4ec187824">labels</a>[i] = result-><a class="code" href="structfaiss_1_1RangeSearchResult.html#aa8ff848474a4cc16c5464cc4ec187824">labels</a>[i] < 0 ?</div>
|
|
<div class="line"><a name="l00086"></a><span class="lineno"> 86</span>  result-><a class="code" href="structfaiss_1_1RangeSearchResult.html#aa8ff848474a4cc16c5464cc4ec187824">labels</a>[i] : <a class="code" href="structfaiss_1_1IndexIDMap.html#ad1cc5658f1879f6a2b00d76cca58cf3b">id_map</a>[result-><a class="code" href="structfaiss_1_1RangeSearchResult.html#aa8ff848474a4cc16c5464cc4ec187824">labels</a>[i]];</div>
|
|
<div class="line"><a name="l00087"></a><span class="lineno"> 87</span>  }</div>
|
|
<div class="line"><a name="l00088"></a><span class="lineno"> 88</span> }</div>
|
|
<div class="line"><a name="l00089"></a><span class="lineno"> 89</span> </div>
|
|
<div class="line"><a name="l00090"></a><span class="lineno"> 90</span> <span class="keyword">namespace </span>{</div>
|
|
<div class="line"><a name="l00091"></a><span class="lineno"> 91</span> </div>
|
|
<div class="line"><a name="l00092"></a><span class="lineno"> 92</span> <span class="keyword">struct </span>IDTranslatedSelector: <a class="code" href="structfaiss_1_1IDSelector.html">IDSelector</a> {</div>
|
|
<div class="line"><a name="l00093"></a><span class="lineno"> 93</span>  <span class="keyword">const</span> std::vector <long> & id_map;</div>
|
|
<div class="line"><a name="l00094"></a><span class="lineno"> 94</span>  <span class="keyword">const</span> <a class="code" href="structfaiss_1_1IDSelector.html">IDSelector</a> & sel;</div>
|
|
<div class="line"><a name="l00095"></a><span class="lineno"> 95</span>  IDTranslatedSelector (<span class="keyword">const</span> std::vector <long> & id_map,</div>
|
|
<div class="line"><a name="l00096"></a><span class="lineno"> 96</span>  <span class="keyword">const</span> <a class="code" href="structfaiss_1_1IDSelector.html">IDSelector</a> & sel):</div>
|
|
<div class="line"><a name="l00097"></a><span class="lineno"> 97</span>  id_map (id_map), sel (sel)</div>
|
|
<div class="line"><a name="l00098"></a><span class="lineno"> 98</span>  {}</div>
|
|
<div class="line"><a name="l00099"></a><span class="lineno"> 99</span>  <span class="keywordtype">bool</span> is_member(idx_t <span class="keywordtype">id</span>)<span class="keyword"> const override </span>{</div>
|
|
<div class="line"><a name="l00100"></a><span class="lineno"> 100</span>  <span class="keywordflow">return</span> sel.is_member(id_map[<span class="keywordtype">id</span>]);</div>
|
|
<div class="line"><a name="l00101"></a><span class="lineno"> 101</span>  }</div>
|
|
<div class="line"><a name="l00102"></a><span class="lineno"> 102</span> };</div>
|
|
<div class="line"><a name="l00103"></a><span class="lineno"> 103</span> </div>
|
|
<div class="line"><a name="l00104"></a><span class="lineno"> 104</span> }</div>
|
|
<div class="line"><a name="l00105"></a><span class="lineno"> 105</span> </div>
|
|
<div class="line"><a name="l00106"></a><span class="lineno"><a class="line" href="structfaiss_1_1IndexIDMap.html#a699ca4891bf3a437b2a182c04d33d9b9"> 106</a></span> <span class="keywordtype">long</span> <a class="code" href="structfaiss_1_1IndexIDMap.html#a699ca4891bf3a437b2a182c04d33d9b9">IndexIDMap::remove_ids</a> (<span class="keyword">const</span> <a class="code" href="structfaiss_1_1IDSelector.html">IDSelector</a> & sel)</div>
|
|
<div class="line"><a name="l00107"></a><span class="lineno"> 107</span> {</div>
|
|
<div class="line"><a name="l00108"></a><span class="lineno"> 108</span>  <span class="comment">// remove in sub-index first</span></div>
|
|
<div class="line"><a name="l00109"></a><span class="lineno"> 109</span>  IDTranslatedSelector sel2 (<a class="code" href="structfaiss_1_1IndexIDMap.html#ad1cc5658f1879f6a2b00d76cca58cf3b">id_map</a>, sel);</div>
|
|
<div class="line"><a name="l00110"></a><span class="lineno"> 110</span>  <span class="keywordtype">long</span> nremove = index-><a class="code" href="structfaiss_1_1Index.html#af8bf7bd97aeed8ad5fc48b242379a68a">remove_ids</a> (sel2);</div>
|
|
<div class="line"><a name="l00111"></a><span class="lineno"> 111</span> </div>
|
|
<div class="line"><a name="l00112"></a><span class="lineno"> 112</span>  <span class="keywordtype">long</span> j = 0;</div>
|
|
<div class="line"><a name="l00113"></a><span class="lineno"> 113</span>  <span class="keywordflow">for</span> (<a class="code" href="structfaiss_1_1Index.html#a040c6aed1f224f3ea7bf58eebc0c31a4">idx_t</a> i = 0; i < <a class="code" href="structfaiss_1_1Index.html#a6970683faa021b7a6f1a0865c0d4eccd">ntotal</a>; i++) {</div>
|
|
<div class="line"><a name="l00114"></a><span class="lineno"> 114</span>  <span class="keywordflow">if</span> (sel.is_member (<a class="code" href="structfaiss_1_1IndexIDMap.html#ad1cc5658f1879f6a2b00d76cca58cf3b">id_map</a>[i])) {</div>
|
|
<div class="line"><a name="l00115"></a><span class="lineno"> 115</span>  <span class="comment">// remove</span></div>
|
|
<div class="line"><a name="l00116"></a><span class="lineno"> 116</span>  } <span class="keywordflow">else</span> {</div>
|
|
<div class="line"><a name="l00117"></a><span class="lineno"> 117</span>  <a class="code" href="structfaiss_1_1IndexIDMap.html#ad1cc5658f1879f6a2b00d76cca58cf3b">id_map</a>[j] = <a class="code" href="structfaiss_1_1IndexIDMap.html#ad1cc5658f1879f6a2b00d76cca58cf3b">id_map</a>[i];</div>
|
|
<div class="line"><a name="l00118"></a><span class="lineno"> 118</span>  j++;</div>
|
|
<div class="line"><a name="l00119"></a><span class="lineno"> 119</span>  }</div>
|
|
<div class="line"><a name="l00120"></a><span class="lineno"> 120</span>  }</div>
|
|
<div class="line"><a name="l00121"></a><span class="lineno"> 121</span>  FAISS_ASSERT (j == index-><a class="code" href="structfaiss_1_1Index.html#a6970683faa021b7a6f1a0865c0d4eccd">ntotal</a>);</div>
|
|
<div class="line"><a name="l00122"></a><span class="lineno"> 122</span>  ntotal = j;</div>
|
|
<div class="line"><a name="l00123"></a><span class="lineno"> 123</span>  <a class="code" href="structfaiss_1_1IndexIDMap.html#ad1cc5658f1879f6a2b00d76cca58cf3b">id_map</a>.resize(ntotal);</div>
|
|
<div class="line"><a name="l00124"></a><span class="lineno"> 124</span>  <span class="keywordflow">return</span> nremove;</div>
|
|
<div class="line"><a name="l00125"></a><span class="lineno"> 125</span> }</div>
|
|
<div class="line"><a name="l00126"></a><span class="lineno"> 126</span> </div>
|
|
<div class="line"><a name="l00127"></a><span class="lineno"> 127</span> </div>
|
|
<div class="line"><a name="l00128"></a><span class="lineno"> 128</span> </div>
|
|
<div class="line"><a name="l00129"></a><span class="lineno"> 129</span> </div>
|
|
<div class="line"><a name="l00130"></a><span class="lineno"> 130</span> IndexIDMap::~IndexIDMap ()</div>
|
|
<div class="line"><a name="l00131"></a><span class="lineno"> 131</span> {</div>
|
|
<div class="line"><a name="l00132"></a><span class="lineno"> 132</span>  <span class="keywordflow">if</span> (<a class="code" href="structfaiss_1_1IndexIDMap.html#affa8a764304410c96489c2b2e178bcdb">own_fields</a>) <span class="keyword">delete</span> index;</div>
|
|
<div class="line"><a name="l00133"></a><span class="lineno"> 133</span> }</div>
|
|
<div class="line"><a name="l00134"></a><span class="lineno"> 134</span> </div>
|
|
<div class="line"><a name="l00135"></a><span class="lineno"> 135</span> <span class="comment">/*****************************************************</span></div>
|
|
<div class="line"><a name="l00136"></a><span class="lineno"> 136</span> <span class="comment"> * IndexIDMap2 implementation</span></div>
|
|
<div class="line"><a name="l00137"></a><span class="lineno"> 137</span> <span class="comment"> *******************************************************/</span></div>
|
|
<div class="line"><a name="l00138"></a><span class="lineno"> 138</span> </div>
|
|
<div class="line"><a name="l00139"></a><span class="lineno"> 139</span> IndexIDMap2::IndexIDMap2 (Index *index): IndexIDMap (index)</div>
|
|
<div class="line"><a name="l00140"></a><span class="lineno"> 140</span> {}</div>
|
|
<div class="line"><a name="l00141"></a><span class="lineno"> 141</span> </div>
|
|
<div class="line"><a name="l00142"></a><span class="lineno"><a class="line" href="structfaiss_1_1IndexIDMap2.html#a33ff76975891695c2fa45ed26fc8eae9"> 142</a></span> <span class="keywordtype">void</span> <a class="code" href="structfaiss_1_1IndexIDMap2.html#a33ff76975891695c2fa45ed26fc8eae9">IndexIDMap2::add_with_ids</a>(idx_t n, <span class="keyword">const</span> <span class="keywordtype">float</span>* x, <span class="keyword">const</span> <span class="keywordtype">long</span>* xids)</div>
|
|
<div class="line"><a name="l00143"></a><span class="lineno"> 143</span> {</div>
|
|
<div class="line"><a name="l00144"></a><span class="lineno"> 144</span>  <span class="keywordtype">size_t</span> prev_ntotal = <a class="code" href="structfaiss_1_1Index.html#a6970683faa021b7a6f1a0865c0d4eccd">ntotal</a>;</div>
|
|
<div class="line"><a name="l00145"></a><span class="lineno"> 145</span>  <a class="code" href="structfaiss_1_1IndexIDMap.html#aeace75e53648c052d05c357b744e090b">IndexIDMap::add_with_ids</a> (n, x, xids);</div>
|
|
<div class="line"><a name="l00146"></a><span class="lineno"> 146</span>  <span class="keywordflow">for</span> (<span class="keywordtype">size_t</span> i = prev_ntotal; i < <a class="code" href="structfaiss_1_1Index.html#a6970683faa021b7a6f1a0865c0d4eccd">ntotal</a>; i++) {</div>
|
|
<div class="line"><a name="l00147"></a><span class="lineno"> 147</span>  rev_map [<a class="code" href="structfaiss_1_1IndexIDMap.html#ad1cc5658f1879f6a2b00d76cca58cf3b">id_map</a> [i]] = i;</div>
|
|
<div class="line"><a name="l00148"></a><span class="lineno"> 148</span>  }</div>
|
|
<div class="line"><a name="l00149"></a><span class="lineno"> 149</span> }</div>
|
|
<div class="line"><a name="l00150"></a><span class="lineno"> 150</span> </div>
|
|
<div class="line"><a name="l00151"></a><span class="lineno"><a class="line" href="structfaiss_1_1IndexIDMap2.html#a646302669189e370d9d864fc18f096fe"> 151</a></span> <span class="keywordtype">void</span> <a class="code" href="structfaiss_1_1IndexIDMap2.html#a646302669189e370d9d864fc18f096fe">IndexIDMap2::construct_rev_map</a> ()</div>
|
|
<div class="line"><a name="l00152"></a><span class="lineno"> 152</span> {</div>
|
|
<div class="line"><a name="l00153"></a><span class="lineno"> 153</span>  rev_map.clear ();</div>
|
|
<div class="line"><a name="l00154"></a><span class="lineno"> 154</span>  <span class="keywordflow">for</span> (<span class="keywordtype">size_t</span> i = 0; i < <a class="code" href="structfaiss_1_1Index.html#a6970683faa021b7a6f1a0865c0d4eccd">ntotal</a>; i++) {</div>
|
|
<div class="line"><a name="l00155"></a><span class="lineno"> 155</span>  rev_map [<a class="code" href="structfaiss_1_1IndexIDMap.html#ad1cc5658f1879f6a2b00d76cca58cf3b">id_map</a> [i]] = i;</div>
|
|
<div class="line"><a name="l00156"></a><span class="lineno"> 156</span>  }</div>
|
|
<div class="line"><a name="l00157"></a><span class="lineno"> 157</span> }</div>
|
|
<div class="line"><a name="l00158"></a><span class="lineno"> 158</span> </div>
|
|
<div class="line"><a name="l00159"></a><span class="lineno"> 159</span> </div>
|
|
<div class="line"><a name="l00160"></a><span class="lineno"><a class="line" href="structfaiss_1_1IndexIDMap2.html#a87e313ebc8dd62fb734a61a6d95a98f0"> 160</a></span> <span class="keywordtype">long</span> <a class="code" href="structfaiss_1_1IndexIDMap2.html#a87e313ebc8dd62fb734a61a6d95a98f0">IndexIDMap2::remove_ids</a>(<span class="keyword">const</span> <a class="code" href="structfaiss_1_1IDSelector.html">IDSelector</a>& sel)</div>
|
|
<div class="line"><a name="l00161"></a><span class="lineno"> 161</span> {</div>
|
|
<div class="line"><a name="l00162"></a><span class="lineno"> 162</span>  <span class="comment">// This is quite inefficient</span></div>
|
|
<div class="line"><a name="l00163"></a><span class="lineno"> 163</span>  <span class="keywordtype">long</span> nremove = <a class="code" href="structfaiss_1_1IndexIDMap.html#a699ca4891bf3a437b2a182c04d33d9b9">IndexIDMap::remove_ids</a> (sel);</div>
|
|
<div class="line"><a name="l00164"></a><span class="lineno"> 164</span>  <a class="code" href="structfaiss_1_1IndexIDMap2.html#a646302669189e370d9d864fc18f096fe">construct_rev_map</a> ();</div>
|
|
<div class="line"><a name="l00165"></a><span class="lineno"> 165</span>  <span class="keywordflow">return</span> nremove;</div>
|
|
<div class="line"><a name="l00166"></a><span class="lineno"> 166</span> }</div>
|
|
<div class="line"><a name="l00167"></a><span class="lineno"> 167</span> </div>
|
|
<div class="line"><a name="l00168"></a><span class="lineno"><a class="line" href="structfaiss_1_1IndexIDMap2.html#a3660adb35a3299bc328988f5ff2199fe"> 168</a></span> <span class="keywordtype">void</span> <a class="code" href="structfaiss_1_1IndexIDMap2.html#a3660adb35a3299bc328988f5ff2199fe">IndexIDMap2::reconstruct</a> (idx_t key, <span class="keywordtype">float</span> * recons)<span class="keyword"> const</span></div>
|
|
<div class="line"><a name="l00169"></a><span class="lineno"> 169</span> <span class="keyword"></span>{</div>
|
|
<div class="line"><a name="l00170"></a><span class="lineno"> 170</span>  <span class="keywordflow">try</span> {</div>
|
|
<div class="line"><a name="l00171"></a><span class="lineno"> 171</span>  index-><a class="code" href="structfaiss_1_1Index.html#aaef6f4132b1f390121ebb11058baf190">reconstruct</a> (rev_map.at (key), recons);</div>
|
|
<div class="line"><a name="l00172"></a><span class="lineno"> 172</span>  } <span class="keywordflow">catch</span> (<span class="keyword">const</span> std::out_of_range& e) {</div>
|
|
<div class="line"><a name="l00173"></a><span class="lineno"> 173</span>  FAISS_THROW_FMT (<span class="stringliteral">"key %ld not found"</span>, key);</div>
|
|
<div class="line"><a name="l00174"></a><span class="lineno"> 174</span>  }</div>
|
|
<div class="line"><a name="l00175"></a><span class="lineno"> 175</span> }</div>
|
|
<div class="line"><a name="l00176"></a><span class="lineno"> 176</span> </div>
|
|
<div class="line"><a name="l00177"></a><span class="lineno"> 177</span> </div>
|
|
<div class="line"><a name="l00178"></a><span class="lineno"> 178</span> </div>
|
|
<div class="line"><a name="l00179"></a><span class="lineno"> 179</span> <span class="comment">/*****************************************************</span></div>
|
|
<div class="line"><a name="l00180"></a><span class="lineno"> 180</span> <span class="comment"> * IndexShards implementation</span></div>
|
|
<div class="line"><a name="l00181"></a><span class="lineno"> 181</span> <span class="comment"> *******************************************************/</span></div>
|
|
<div class="line"><a name="l00182"></a><span class="lineno"> 182</span> </div>
|
|
<div class="line"><a name="l00183"></a><span class="lineno"> 183</span> <span class="comment">// subroutines</span></div>
|
|
<div class="line"><a name="l00184"></a><span class="lineno"> 184</span> <span class="keyword">namespace </span>{</div>
|
|
<div class="line"><a name="l00185"></a><span class="lineno"> 185</span> </div>
|
|
<div class="line"><a name="l00186"></a><span class="lineno"> 186</span> </div>
|
|
<div class="line"><a name="l00187"></a><span class="lineno"> 187</span> <span class="keyword">typedef</span> <a class="code" href="structfaiss_1_1Index.html#a040c6aed1f224f3ea7bf58eebc0c31a4">Index::idx_t</a> idx_t;</div>
|
|
<div class="line"><a name="l00188"></a><span class="lineno"> 188</span> </div>
|
|
<div class="line"><a name="l00189"></a><span class="lineno"> 189</span> </div>
|
|
<div class="line"><a name="l00190"></a><span class="lineno"> 190</span> <span class="keyword">template</span><<span class="keyword">class</span> Job></div>
|
|
<div class="line"><a name="l00191"></a><span class="lineno"> 191</span> <span class="keyword">struct </span>Thread {</div>
|
|
<div class="line"><a name="l00192"></a><span class="lineno"> 192</span>  Job job;</div>
|
|
<div class="line"><a name="l00193"></a><span class="lineno"> 193</span>  pthread_t thread;</div>
|
|
<div class="line"><a name="l00194"></a><span class="lineno"> 194</span> </div>
|
|
<div class="line"><a name="l00195"></a><span class="lineno"> 195</span>  Thread () {}</div>
|
|
<div class="line"><a name="l00196"></a><span class="lineno"> 196</span> </div>
|
|
<div class="line"><a name="l00197"></a><span class="lineno"> 197</span>  <span class="keyword">explicit</span> Thread (<span class="keyword">const</span> Job & job): job(job) {}</div>
|
|
<div class="line"><a name="l00198"></a><span class="lineno"> 198</span> </div>
|
|
<div class="line"><a name="l00199"></a><span class="lineno"> 199</span>  <span class="keywordtype">void</span> start () {</div>
|
|
<div class="line"><a name="l00200"></a><span class="lineno"> 200</span>  pthread_create (&thread, <span class="keyword">nullptr</span>, run, <span class="keyword">this</span>);</div>
|
|
<div class="line"><a name="l00201"></a><span class="lineno"> 201</span>  }</div>
|
|
<div class="line"><a name="l00202"></a><span class="lineno"> 202</span> </div>
|
|
<div class="line"><a name="l00203"></a><span class="lineno"> 203</span>  <span class="keywordtype">void</span> wait () {</div>
|
|
<div class="line"><a name="l00204"></a><span class="lineno"> 204</span>  pthread_join (thread, <span class="keyword">nullptr</span>);</div>
|
|
<div class="line"><a name="l00205"></a><span class="lineno"> 205</span>  }</div>
|
|
<div class="line"><a name="l00206"></a><span class="lineno"> 206</span> </div>
|
|
<div class="line"><a name="l00207"></a><span class="lineno"> 207</span>  <span class="keyword">static</span> <span class="keywordtype">void</span> * run (<span class="keywordtype">void</span> *arg) {</div>
|
|
<div class="line"><a name="l00208"></a><span class="lineno"> 208</span>  <span class="keyword">static_cast<</span>Thread*<span class="keyword">></span> (arg)->job.run();</div>
|
|
<div class="line"><a name="l00209"></a><span class="lineno"> 209</span>  <span class="keywordflow">return</span> <span class="keyword">nullptr</span>;</div>
|
|
<div class="line"><a name="l00210"></a><span class="lineno"> 210</span>  }</div>
|
|
<div class="line"><a name="l00211"></a><span class="lineno"> 211</span> </div>
|
|
<div class="line"><a name="l00212"></a><span class="lineno"> 212</span> };</div>
|
|
<div class="line"><a name="l00213"></a><span class="lineno"> 213</span> </div>
|
|
<div class="line"><a name="l00214"></a><span class="lineno"> 214</span> <span class="comment"></span></div>
|
|
<div class="line"><a name="l00215"></a><span class="lineno"> 215</span> <span class="comment">/// callback + thread management to train 1 shard</span></div>
|
|
<div class="line"><a name="l00216"></a><span class="lineno"> 216</span> <span class="comment"></span><span class="keyword">struct </span>TrainJob {</div>
|
|
<div class="line"><a name="l00217"></a><span class="lineno"> 217</span>  IndexShards *index; <span class="comment">// the relevant index</span></div>
|
|
<div class="line"><a name="l00218"></a><span class="lineno"> 218</span>  <span class="keywordtype">int</span> no; <span class="comment">// shard number</span></div>
|
|
<div class="line"><a name="l00219"></a><span class="lineno"> 219</span>  idx_t n; <span class="comment">// train points</span></div>
|
|
<div class="line"><a name="l00220"></a><span class="lineno"> 220</span>  <span class="keyword">const</span> <span class="keywordtype">float</span> *x;</div>
|
|
<div class="line"><a name="l00221"></a><span class="lineno"> 221</span> </div>
|
|
<div class="line"><a name="l00222"></a><span class="lineno"> 222</span>  <span class="keywordtype">void</span> run ()</div>
|
|
<div class="line"><a name="l00223"></a><span class="lineno"> 223</span>  {</div>
|
|
<div class="line"><a name="l00224"></a><span class="lineno"> 224</span>  <span class="keywordflow">if</span> (index->verbose)</div>
|
|
<div class="line"><a name="l00225"></a><span class="lineno"> 225</span>  printf (<span class="stringliteral">"begin train shard %d on %ld points\n"</span>, no, n);</div>
|
|
<div class="line"><a name="l00226"></a><span class="lineno"> 226</span>  index->shard_indexes [no]->train(n, x);</div>
|
|
<div class="line"><a name="l00227"></a><span class="lineno"> 227</span>  <span class="keywordflow">if</span> (index->verbose)</div>
|
|
<div class="line"><a name="l00228"></a><span class="lineno"> 228</span>  printf (<span class="stringliteral">"end train shard %d\n"</span>, no);</div>
|
|
<div class="line"><a name="l00229"></a><span class="lineno"> 229</span>  }</div>
|
|
<div class="line"><a name="l00230"></a><span class="lineno"> 230</span> </div>
|
|
<div class="line"><a name="l00231"></a><span class="lineno"> 231</span> };</div>
|
|
<div class="line"><a name="l00232"></a><span class="lineno"> 232</span> </div>
|
|
<div class="line"><a name="l00233"></a><span class="lineno"> 233</span> <span class="keyword">struct </span>AddJob {</div>
|
|
<div class="line"><a name="l00234"></a><span class="lineno"> 234</span>  IndexShards *index; <span class="comment">// the relevant index</span></div>
|
|
<div class="line"><a name="l00235"></a><span class="lineno"> 235</span>  <span class="keywordtype">int</span> no; <span class="comment">// shard number</span></div>
|
|
<div class="line"><a name="l00236"></a><span class="lineno"> 236</span> </div>
|
|
<div class="line"><a name="l00237"></a><span class="lineno"> 237</span>  idx_t n;</div>
|
|
<div class="line"><a name="l00238"></a><span class="lineno"> 238</span>  <span class="keyword">const</span> <span class="keywordtype">float</span> *x;</div>
|
|
<div class="line"><a name="l00239"></a><span class="lineno"> 239</span>  <span class="keyword">const</span> idx_t *ids;</div>
|
|
<div class="line"><a name="l00240"></a><span class="lineno"> 240</span> </div>
|
|
<div class="line"><a name="l00241"></a><span class="lineno"> 241</span>  <span class="keywordtype">void</span> run ()</div>
|
|
<div class="line"><a name="l00242"></a><span class="lineno"> 242</span>  {</div>
|
|
<div class="line"><a name="l00243"></a><span class="lineno"> 243</span>  <span class="keywordflow">if</span> (index->verbose)</div>
|
|
<div class="line"><a name="l00244"></a><span class="lineno"> 244</span>  printf (<span class="stringliteral">"begin add shard %d on %ld points\n"</span>, no, n);</div>
|
|
<div class="line"><a name="l00245"></a><span class="lineno"> 245</span>  <span class="keywordflow">if</span> (ids)</div>
|
|
<div class="line"><a name="l00246"></a><span class="lineno"> 246</span>  index->shard_indexes[no]->add_with_ids (n, x, ids);</div>
|
|
<div class="line"><a name="l00247"></a><span class="lineno"> 247</span>  <span class="keywordflow">else</span></div>
|
|
<div class="line"><a name="l00248"></a><span class="lineno"> 248</span>  index->shard_indexes[no]->add (n, x);</div>
|
|
<div class="line"><a name="l00249"></a><span class="lineno"> 249</span>  <span class="keywordflow">if</span> (index->verbose)</div>
|
|
<div class="line"><a name="l00250"></a><span class="lineno"> 250</span>  printf (<span class="stringliteral">"end add shard %d on %ld points\n"</span>, no, n);</div>
|
|
<div class="line"><a name="l00251"></a><span class="lineno"> 251</span>  }</div>
|
|
<div class="line"><a name="l00252"></a><span class="lineno"> 252</span> };</div>
|
|
<div class="line"><a name="l00253"></a><span class="lineno"> 253</span> </div>
|
|
<div class="line"><a name="l00254"></a><span class="lineno"> 254</span> </div>
|
|
<div class="line"><a name="l00255"></a><span class="lineno"> 255</span> <span class="comment"></span></div>
|
|
<div class="line"><a name="l00256"></a><span class="lineno"> 256</span> <span class="comment">/// callback + thread management to query in 1 shard</span></div>
|
|
<div class="line"><a name="l00257"></a><span class="lineno"> 257</span> <span class="comment"></span><span class="keyword">struct </span>QueryJob {</div>
|
|
<div class="line"><a name="l00258"></a><span class="lineno"> 258</span>  <span class="keyword">const</span> IndexShards *index; <span class="comment">// the relevant index</span></div>
|
|
<div class="line"><a name="l00259"></a><span class="lineno"> 259</span>  <span class="keywordtype">int</span> no; <span class="comment">// shard number</span></div>
|
|
<div class="line"><a name="l00260"></a><span class="lineno"> 260</span> </div>
|
|
<div class="line"><a name="l00261"></a><span class="lineno"> 261</span>  <span class="comment">// query params</span></div>
|
|
<div class="line"><a name="l00262"></a><span class="lineno"> 262</span>  idx_t n;</div>
|
|
<div class="line"><a name="l00263"></a><span class="lineno"> 263</span>  <span class="keyword">const</span> <span class="keywordtype">float</span> *x;</div>
|
|
<div class="line"><a name="l00264"></a><span class="lineno"> 264</span>  idx_t k;</div>
|
|
<div class="line"><a name="l00265"></a><span class="lineno"> 265</span>  <span class="keywordtype">float</span> *distances;</div>
|
|
<div class="line"><a name="l00266"></a><span class="lineno"> 266</span>  idx_t *labels;</div>
|
|
<div class="line"><a name="l00267"></a><span class="lineno"> 267</span> </div>
|
|
<div class="line"><a name="l00268"></a><span class="lineno"> 268</span> </div>
|
|
<div class="line"><a name="l00269"></a><span class="lineno"> 269</span>  <span class="keywordtype">void</span> run ()</div>
|
|
<div class="line"><a name="l00270"></a><span class="lineno"> 270</span>  {</div>
|
|
<div class="line"><a name="l00271"></a><span class="lineno"> 271</span>  <span class="keywordflow">if</span> (index->verbose)</div>
|
|
<div class="line"><a name="l00272"></a><span class="lineno"> 272</span>  printf (<span class="stringliteral">"begin query shard %d on %ld points\n"</span>, no, n);</div>
|
|
<div class="line"><a name="l00273"></a><span class="lineno"> 273</span>  index->shard_indexes [no]->search (n, x, k,</div>
|
|
<div class="line"><a name="l00274"></a><span class="lineno"> 274</span>  distances, labels);</div>
|
|
<div class="line"><a name="l00275"></a><span class="lineno"> 275</span>  <span class="keywordflow">if</span> (index->verbose)</div>
|
|
<div class="line"><a name="l00276"></a><span class="lineno"> 276</span>  printf (<span class="stringliteral">"end query shard %d\n"</span>, no);</div>
|
|
<div class="line"><a name="l00277"></a><span class="lineno"> 277</span>  }</div>
|
|
<div class="line"><a name="l00278"></a><span class="lineno"> 278</span> </div>
|
|
<div class="line"><a name="l00279"></a><span class="lineno"> 279</span> </div>
|
|
<div class="line"><a name="l00280"></a><span class="lineno"> 280</span> };</div>
|
|
<div class="line"><a name="l00281"></a><span class="lineno"> 281</span> </div>
|
|
<div class="line"><a name="l00282"></a><span class="lineno"> 282</span> </div>
|
|
<div class="line"><a name="l00283"></a><span class="lineno"> 283</span> </div>
|
|
<div class="line"><a name="l00284"></a><span class="lineno"> 284</span> </div>
|
|
<div class="line"><a name="l00285"></a><span class="lineno"> 285</span> <span class="comment">// add translation to all valid labels</span></div>
|
|
<div class="line"><a name="l00286"></a><span class="lineno"> 286</span> <span class="keywordtype">void</span> translate_labels (<span class="keywordtype">long</span> n, idx_t *labels, <span class="keywordtype">long</span> translation)</div>
|
|
<div class="line"><a name="l00287"></a><span class="lineno"> 287</span> {</div>
|
|
<div class="line"><a name="l00288"></a><span class="lineno"> 288</span>  <span class="keywordflow">if</span> (translation == 0) <span class="keywordflow">return</span>;</div>
|
|
<div class="line"><a name="l00289"></a><span class="lineno"> 289</span>  <span class="keywordflow">for</span> (<span class="keywordtype">long</span> i = 0; i < n; i++) {</div>
|
|
<div class="line"><a name="l00290"></a><span class="lineno"> 290</span>  <span class="keywordflow">if</span>(labels[i] < 0) <span class="keywordflow">return</span>;</div>
|
|
<div class="line"><a name="l00291"></a><span class="lineno"> 291</span>  labels[i] += translation;</div>
|
|
<div class="line"><a name="l00292"></a><span class="lineno"> 292</span>  }</div>
|
|
<div class="line"><a name="l00293"></a><span class="lineno"> 293</span> }</div>
|
|
<div class="line"><a name="l00294"></a><span class="lineno"> 294</span> </div>
|
|
<div class="line"><a name="l00295"></a><span class="lineno"> 295</span> <span class="comment"></span></div>
|
|
<div class="line"><a name="l00296"></a><span class="lineno"> 296</span> <span class="comment">/** merge result tables from several shards.</span></div>
|
|
<div class="line"><a name="l00297"></a><span class="lineno"> 297</span> <span class="comment"> * @param all_distances size nshard * n * k</span></div>
|
|
<div class="line"><a name="l00298"></a><span class="lineno"> 298</span> <span class="comment"> * @param all_labels idem</span></div>
|
|
<div class="line"><a name="l00299"></a><span class="lineno"> 299</span> <span class="comment"> * @param translartions label translations to apply, size nshard</span></div>
|
|
<div class="line"><a name="l00300"></a><span class="lineno"> 300</span> <span class="comment"> */</span></div>
|
|
<div class="line"><a name="l00301"></a><span class="lineno"> 301</span> </div>
|
|
<div class="line"><a name="l00302"></a><span class="lineno"> 302</span> <span class="keyword">template</span> <<span class="keyword">class</span> C></div>
|
|
<div class="line"><a name="l00303"></a><span class="lineno"> 303</span> <span class="keywordtype">void</span> merge_tables (<span class="keywordtype">long</span> n, <span class="keywordtype">long</span> k, <span class="keywordtype">long</span> nshard,</div>
|
|
<div class="line"><a name="l00304"></a><span class="lineno"> 304</span>  <span class="keywordtype">float</span> *distances, idx_t *labels,</div>
|
|
<div class="line"><a name="l00305"></a><span class="lineno"> 305</span>  <span class="keyword">const</span> <span class="keywordtype">float</span> *all_distances,</div>
|
|
<div class="line"><a name="l00306"></a><span class="lineno"> 306</span>  idx_t *all_labels,</div>
|
|
<div class="line"><a name="l00307"></a><span class="lineno"> 307</span>  <span class="keyword">const</span> <span class="keywordtype">long</span> *translations)</div>
|
|
<div class="line"><a name="l00308"></a><span class="lineno"> 308</span> {</div>
|
|
<div class="line"><a name="l00309"></a><span class="lineno"> 309</span>  <span class="keywordflow">if</span>(k == 0) {</div>
|
|
<div class="line"><a name="l00310"></a><span class="lineno"> 310</span>  <span class="keywordflow">return</span>;</div>
|
|
<div class="line"><a name="l00311"></a><span class="lineno"> 311</span>  }</div>
|
|
<div class="line"><a name="l00312"></a><span class="lineno"> 312</span> </div>
|
|
<div class="line"><a name="l00313"></a><span class="lineno"> 313</span>  <span class="keywordtype">long</span> stride = n * k;</div>
|
|
<div class="line"><a name="l00314"></a><span class="lineno"> 314</span> <span class="preprocessor">#pragma omp parallel</span></div>
|
|
<div class="line"><a name="l00315"></a><span class="lineno"> 315</span> <span class="preprocessor"></span> {</div>
|
|
<div class="line"><a name="l00316"></a><span class="lineno"> 316</span>  std::vector<int> buf (2 * nshard);</div>
|
|
<div class="line"><a name="l00317"></a><span class="lineno"> 317</span>  <span class="keywordtype">int</span> * pointer = buf.data();</div>
|
|
<div class="line"><a name="l00318"></a><span class="lineno"> 318</span>  <span class="keywordtype">int</span> * shard_ids = pointer + nshard;</div>
|
|
<div class="line"><a name="l00319"></a><span class="lineno"> 319</span>  std::vector<float> buf2 (nshard);</div>
|
|
<div class="line"><a name="l00320"></a><span class="lineno"> 320</span>  <span class="keywordtype">float</span> * heap_vals = buf2.data();</div>
|
|
<div class="line"><a name="l00321"></a><span class="lineno"> 321</span> <span class="preprocessor">#pragma omp for</span></div>
|
|
<div class="line"><a name="l00322"></a><span class="lineno"> 322</span> <span class="preprocessor"></span> <span class="keywordflow">for</span> (<span class="keywordtype">long</span> i = 0; i < n; i++) {</div>
|
|
<div class="line"><a name="l00323"></a><span class="lineno"> 323</span>  <span class="comment">// the heap maps values to the shard where they are</span></div>
|
|
<div class="line"><a name="l00324"></a><span class="lineno"> 324</span>  <span class="comment">// produced.</span></div>
|
|
<div class="line"><a name="l00325"></a><span class="lineno"> 325</span>  <span class="keyword">const</span> <span class="keywordtype">float</span> *D_in = all_distances + i * k;</div>
|
|
<div class="line"><a name="l00326"></a><span class="lineno"> 326</span>  <span class="keyword">const</span> idx_t *I_in = all_labels + i * k;</div>
|
|
<div class="line"><a name="l00327"></a><span class="lineno"> 327</span>  <span class="keywordtype">int</span> heap_size = 0;</div>
|
|
<div class="line"><a name="l00328"></a><span class="lineno"> 328</span> </div>
|
|
<div class="line"><a name="l00329"></a><span class="lineno"> 329</span>  <span class="keywordflow">for</span> (<span class="keywordtype">long</span> s = 0; s < nshard; s++) {</div>
|
|
<div class="line"><a name="l00330"></a><span class="lineno"> 330</span>  pointer[s] = 0;</div>
|
|
<div class="line"><a name="l00331"></a><span class="lineno"> 331</span>  <span class="keywordflow">if</span> (I_in[stride * s] >= 0)</div>
|
|
<div class="line"><a name="l00332"></a><span class="lineno"> 332</span>  heap_push<C> (++heap_size, heap_vals, shard_ids,</div>
|
|
<div class="line"><a name="l00333"></a><span class="lineno"> 333</span>  D_in[stride * s], s);</div>
|
|
<div class="line"><a name="l00334"></a><span class="lineno"> 334</span>  }</div>
|
|
<div class="line"><a name="l00335"></a><span class="lineno"> 335</span> </div>
|
|
<div class="line"><a name="l00336"></a><span class="lineno"> 336</span>  <span class="keywordtype">float</span> *D = distances + i * k;</div>
|
|
<div class="line"><a name="l00337"></a><span class="lineno"> 337</span>  idx_t *I = labels + i * k;</div>
|
|
<div class="line"><a name="l00338"></a><span class="lineno"> 338</span> </div>
|
|
<div class="line"><a name="l00339"></a><span class="lineno"> 339</span>  <span class="keywordflow">for</span> (<span class="keywordtype">int</span> j = 0; j < k; j++) {</div>
|
|
<div class="line"><a name="l00340"></a><span class="lineno"> 340</span>  <span class="keywordflow">if</span> (heap_size == 0) {</div>
|
|
<div class="line"><a name="l00341"></a><span class="lineno"> 341</span>  I[j] = -1;</div>
|
|
<div class="line"><a name="l00342"></a><span class="lineno"> 342</span>  D[j] = C::neutral();</div>
|
|
<div class="line"><a name="l00343"></a><span class="lineno"> 343</span>  } <span class="keywordflow">else</span> {</div>
|
|
<div class="line"><a name="l00344"></a><span class="lineno"> 344</span>  <span class="comment">// pop best element</span></div>
|
|
<div class="line"><a name="l00345"></a><span class="lineno"> 345</span>  <span class="keywordtype">int</span> s = shard_ids[0];</div>
|
|
<div class="line"><a name="l00346"></a><span class="lineno"> 346</span>  <span class="keywordtype">int</span> & p = pointer[s];</div>
|
|
<div class="line"><a name="l00347"></a><span class="lineno"> 347</span>  D[j] = heap_vals[0];</div>
|
|
<div class="line"><a name="l00348"></a><span class="lineno"> 348</span>  I[j] = I_in[stride * s + p] + translations[s];</div>
|
|
<div class="line"><a name="l00349"></a><span class="lineno"> 349</span> </div>
|
|
<div class="line"><a name="l00350"></a><span class="lineno"> 350</span>  heap_pop<C> (heap_size--, heap_vals, shard_ids);</div>
|
|
<div class="line"><a name="l00351"></a><span class="lineno"> 351</span>  p++;</div>
|
|
<div class="line"><a name="l00352"></a><span class="lineno"> 352</span>  <span class="keywordflow">if</span> (p < k && I_in[stride * s + p] >= 0)</div>
|
|
<div class="line"><a name="l00353"></a><span class="lineno"> 353</span>  heap_push<C> (++heap_size, heap_vals, shard_ids,</div>
|
|
<div class="line"><a name="l00354"></a><span class="lineno"> 354</span>  D_in[stride * s + p], s);</div>
|
|
<div class="line"><a name="l00355"></a><span class="lineno"> 355</span>  }</div>
|
|
<div class="line"><a name="l00356"></a><span class="lineno"> 356</span>  }</div>
|
|
<div class="line"><a name="l00357"></a><span class="lineno"> 357</span>  }</div>
|
|
<div class="line"><a name="l00358"></a><span class="lineno"> 358</span>  }</div>
|
|
<div class="line"><a name="l00359"></a><span class="lineno"> 359</span> }</div>
|
|
<div class="line"><a name="l00360"></a><span class="lineno"> 360</span> </div>
|
|
<div class="line"><a name="l00361"></a><span class="lineno"> 361</span> </div>
|
|
<div class="line"><a name="l00362"></a><span class="lineno"> 362</span> };</div>
|
|
<div class="line"><a name="l00363"></a><span class="lineno"> 363</span> </div>
|
|
<div class="line"><a name="l00364"></a><span class="lineno"> 364</span> </div>
|
|
<div class="line"><a name="l00365"></a><span class="lineno"> 365</span> </div>
|
|
<div class="line"><a name="l00366"></a><span class="lineno"> 366</span> </div>
|
|
<div class="line"><a name="l00367"></a><span class="lineno"><a class="line" href="structfaiss_1_1IndexShards.html#ad987a18a1786919e32cc02ca36e5ab46"> 367</a></span> <a class="code" href="structfaiss_1_1IndexShards.html#ad987a18a1786919e32cc02ca36e5ab46">IndexShards::IndexShards</a> (idx_t d, <span class="keywordtype">bool</span> threaded, <span class="keywordtype">bool</span> successive_ids):</div>
|
|
<div class="line"><a name="l00368"></a><span class="lineno"> 368</span>  <a class="code" href="structfaiss_1_1Index.html">Index</a> (d), own_fields (false),</div>
|
|
<div class="line"><a name="l00369"></a><span class="lineno"> 369</span>  threaded (threaded), successive_ids (successive_ids)</div>
|
|
<div class="line"><a name="l00370"></a><span class="lineno"> 370</span> {</div>
|
|
<div class="line"><a name="l00371"></a><span class="lineno"> 371</span> </div>
|
|
<div class="line"><a name="l00372"></a><span class="lineno"> 372</span> }</div>
|
|
<div class="line"><a name="l00373"></a><span class="lineno"> 373</span> </div>
|
|
<div class="line"><a name="l00374"></a><span class="lineno"> 374</span> <span class="keywordtype">void</span> IndexShards::add_shard (<a class="code" href="structfaiss_1_1Index.html">Index</a> *idx)</div>
|
|
<div class="line"><a name="l00375"></a><span class="lineno"> 375</span> {</div>
|
|
<div class="line"><a name="l00376"></a><span class="lineno"> 376</span>  shard_indexes.push_back (idx);</div>
|
|
<div class="line"><a name="l00377"></a><span class="lineno"> 377</span>  sync_with_shard_indexes ();</div>
|
|
<div class="line"><a name="l00378"></a><span class="lineno"> 378</span> }</div>
|
|
<div class="line"><a name="l00379"></a><span class="lineno"> 379</span> </div>
|
|
<div class="line"><a name="l00380"></a><span class="lineno"> 380</span> <span class="keywordtype">void</span> IndexShards::sync_with_shard_indexes ()</div>
|
|
<div class="line"><a name="l00381"></a><span class="lineno"> 381</span> {</div>
|
|
<div class="line"><a name="l00382"></a><span class="lineno"> 382</span>  <span class="keywordflow">if</span> (shard_indexes.empty()) <span class="keywordflow">return</span>;</div>
|
|
<div class="line"><a name="l00383"></a><span class="lineno"> 383</span>  Index * index0 = shard_indexes[0];</div>
|
|
<div class="line"><a name="l00384"></a><span class="lineno"> 384</span>  <a class="code" href="structfaiss_1_1Index.html#a2a002388d2c081c2dbab8508dcefe73d">d</a> = index0->d;</div>
|
|
<div class="line"><a name="l00385"></a><span class="lineno"> 385</span>  <a class="code" href="structfaiss_1_1Index.html#a8e18f641854b2bde83ecff0a2f9a6f4e">metric_type</a> = index0->metric_type;</div>
|
|
<div class="line"><a name="l00386"></a><span class="lineno"> 386</span>  <a class="code" href="structfaiss_1_1Index.html#a6e92732617c4dbe364e7678dd8773a7f">is_trained</a> = index0->is_trained;</div>
|
|
<div class="line"><a name="l00387"></a><span class="lineno"> 387</span>  <a class="code" href="structfaiss_1_1Index.html#a6970683faa021b7a6f1a0865c0d4eccd">ntotal</a> = index0->ntotal;</div>
|
|
<div class="line"><a name="l00388"></a><span class="lineno"> 388</span>  <span class="keywordflow">for</span> (<span class="keywordtype">int</span> i = 1; i < shard_indexes.size(); i++) {</div>
|
|
<div class="line"><a name="l00389"></a><span class="lineno"> 389</span>  Index * index = shard_indexes[i];</div>
|
|
<div class="line"><a name="l00390"></a><span class="lineno"> 390</span>  FAISS_THROW_IF_NOT (<a class="code" href="structfaiss_1_1Index.html#a8e18f641854b2bde83ecff0a2f9a6f4e">metric_type</a> == index->metric_type);</div>
|
|
<div class="line"><a name="l00391"></a><span class="lineno"> 391</span>  FAISS_THROW_IF_NOT (<a class="code" href="structfaiss_1_1Index.html#a2a002388d2c081c2dbab8508dcefe73d">d</a> == index->d);</div>
|
|
<div class="line"><a name="l00392"></a><span class="lineno"> 392</span>  <a class="code" href="structfaiss_1_1Index.html#a6970683faa021b7a6f1a0865c0d4eccd">ntotal</a> += index->ntotal;</div>
|
|
<div class="line"><a name="l00393"></a><span class="lineno"> 393</span>  }</div>
|
|
<div class="line"><a name="l00394"></a><span class="lineno"> 394</span> }</div>
|
|
<div class="line"><a name="l00395"></a><span class="lineno"> 395</span> </div>
|
|
<div class="line"><a name="l00396"></a><span class="lineno"> 396</span> </div>
|
|
<div class="line"><a name="l00397"></a><span class="lineno"><a class="line" href="structfaiss_1_1IndexShards.html#a36fbcc85eed9bf2ea15587f04be9ae0b"> 397</a></span> <span class="keywordtype">void</span> <a class="code" href="structfaiss_1_1IndexShards.html#a36fbcc85eed9bf2ea15587f04be9ae0b">IndexShards::train</a> (idx_t n, <span class="keyword">const</span> <span class="keywordtype">float</span> *x)</div>
|
|
<div class="line"><a name="l00398"></a><span class="lineno"> 398</span> {</div>
|
|
<div class="line"><a name="l00399"></a><span class="lineno"> 399</span> </div>
|
|
<div class="line"><a name="l00400"></a><span class="lineno"> 400</span>  <span class="comment">// pre-alloc because we don't want reallocs</span></div>
|
|
<div class="line"><a name="l00401"></a><span class="lineno"> 401</span>  std::vector<Thread<TrainJob > > tss (shard_indexes.size());</div>
|
|
<div class="line"><a name="l00402"></a><span class="lineno"> 402</span>  <span class="keywordtype">int</span> nt = 0;</div>
|
|
<div class="line"><a name="l00403"></a><span class="lineno"> 403</span>  <span class="keywordflow">for</span> (<span class="keywordtype">int</span> i = 0; i < shard_indexes.size(); i++) {</div>
|
|
<div class="line"><a name="l00404"></a><span class="lineno"> 404</span>  <span class="keywordflow">if</span>(!shard_indexes[i]-><a class="code" href="structfaiss_1_1Index.html#a6e92732617c4dbe364e7678dd8773a7f">is_trained</a>) {</div>
|
|
<div class="line"><a name="l00405"></a><span class="lineno"> 405</span>  TrainJob ts = {<span class="keyword">this</span>, i, n, x};</div>
|
|
<div class="line"><a name="l00406"></a><span class="lineno"> 406</span>  <span class="keywordflow">if</span> (<a class="code" href="structfaiss_1_1IndexShards.html#a493da171720f2c3f35058f793fb9a6ce">threaded</a>) {</div>
|
|
<div class="line"><a name="l00407"></a><span class="lineno"> 407</span>  tss[nt] = Thread<TrainJob> (ts);</div>
|
|
<div class="line"><a name="l00408"></a><span class="lineno"> 408</span>  tss[nt++].start();</div>
|
|
<div class="line"><a name="l00409"></a><span class="lineno"> 409</span>  } <span class="keywordflow">else</span> {</div>
|
|
<div class="line"><a name="l00410"></a><span class="lineno"> 410</span>  ts.run();</div>
|
|
<div class="line"><a name="l00411"></a><span class="lineno"> 411</span>  }</div>
|
|
<div class="line"><a name="l00412"></a><span class="lineno"> 412</span>  }</div>
|
|
<div class="line"><a name="l00413"></a><span class="lineno"> 413</span>  }</div>
|
|
<div class="line"><a name="l00414"></a><span class="lineno"> 414</span>  <span class="keywordflow">for</span> (<span class="keywordtype">int</span> i = 0; i < nt; i++) {</div>
|
|
<div class="line"><a name="l00415"></a><span class="lineno"> 415</span>  tss[i].wait();</div>
|
|
<div class="line"><a name="l00416"></a><span class="lineno"> 416</span>  }</div>
|
|
<div class="line"><a name="l00417"></a><span class="lineno"> 417</span>  sync_with_shard_indexes ();</div>
|
|
<div class="line"><a name="l00418"></a><span class="lineno"> 418</span> }</div>
|
|
<div class="line"><a name="l00419"></a><span class="lineno"> 419</span> </div>
|
|
<div class="line"><a name="l00420"></a><span class="lineno"><a class="line" href="structfaiss_1_1IndexShards.html#a2e06889521c566525011eb46cc97cda6"> 420</a></span> <span class="keywordtype">void</span> <a class="code" href="structfaiss_1_1IndexShards.html#a2e06889521c566525011eb46cc97cda6">IndexShards::add</a> (idx_t n, <span class="keyword">const</span> <span class="keywordtype">float</span> *x)</div>
|
|
<div class="line"><a name="l00421"></a><span class="lineno"> 421</span> {</div>
|
|
<div class="line"><a name="l00422"></a><span class="lineno"> 422</span>  <a class="code" href="structfaiss_1_1IndexShards.html#a1f771d50133494f682f05c9ab9e39ad9">add_with_ids</a> (n, x, <span class="keyword">nullptr</span>);</div>
|
|
<div class="line"><a name="l00423"></a><span class="lineno"> 423</span> }</div>
|
|
<div class="line"><a name="l00424"></a><span class="lineno"> 424</span> <span class="comment"></span></div>
|
|
<div class="line"><a name="l00425"></a><span class="lineno"> 425</span> <span class="comment"> /**</span></div>
|
|
<div class="line"><a name="l00426"></a><span class="lineno"> 426</span> <span class="comment"> * Cases (successive_ids, xids):</span></div>
|
|
<div class="line"><a name="l00427"></a><span class="lineno"> 427</span> <span class="comment"> * - true, non-NULL ERROR: it makes no sense to pass in ids and</span></div>
|
|
<div class="line"><a name="l00428"></a><span class="lineno"> 428</span> <span class="comment"> * request them to be shifted</span></div>
|
|
<div class="line"><a name="l00429"></a><span class="lineno"> 429</span> <span class="comment"> * - true, NULL OK, but should be called only once (calls add()</span></div>
|
|
<div class="line"><a name="l00430"></a><span class="lineno"> 430</span> <span class="comment"> * on sub-indexes).</span></div>
|
|
<div class="line"><a name="l00431"></a><span class="lineno"> 431</span> <span class="comment"> * - false, non-NULL OK: will call add_with_ids with passed in xids</span></div>
|
|
<div class="line"><a name="l00432"></a><span class="lineno"> 432</span> <span class="comment"> * distributed evenly over shards</span></div>
|
|
<div class="line"><a name="l00433"></a><span class="lineno"> 433</span> <span class="comment"> * - false, NULL OK: will call add_with_ids on each sub-index,</span></div>
|
|
<div class="line"><a name="l00434"></a><span class="lineno"> 434</span> <span class="comment"> * starting at ntotal</span></div>
|
|
<div class="line"><a name="l00435"></a><span class="lineno"> 435</span> <span class="comment"> */</span></div>
|
|
<div class="line"><a name="l00436"></a><span class="lineno"> 436</span> </div>
|
|
<div class="line"><a name="l00437"></a><span class="lineno"><a class="line" href="structfaiss_1_1IndexShards.html#a1f771d50133494f682f05c9ab9e39ad9"> 437</a></span> <span class="keywordtype">void</span> <a class="code" href="structfaiss_1_1IndexShards.html#a1f771d50133494f682f05c9ab9e39ad9">IndexShards::add_with_ids</a> (idx_t n, <span class="keyword">const</span> <span class="keywordtype">float</span> * x, <span class="keyword">const</span> <span class="keywordtype">long</span> *xids)</div>
|
|
<div class="line"><a name="l00438"></a><span class="lineno"> 438</span> {</div>
|
|
<div class="line"><a name="l00439"></a><span class="lineno"> 439</span> </div>
|
|
<div class="line"><a name="l00440"></a><span class="lineno"> 440</span>  FAISS_THROW_IF_NOT_MSG(!(successive_ids && xids),</div>
|
|
<div class="line"><a name="l00441"></a><span class="lineno"> 441</span>  <span class="stringliteral">"It makes no sense to pass in ids and "</span></div>
|
|
<div class="line"><a name="l00442"></a><span class="lineno"> 442</span>  <span class="stringliteral">"request them to be shifted"</span>);</div>
|
|
<div class="line"><a name="l00443"></a><span class="lineno"> 443</span> </div>
|
|
<div class="line"><a name="l00444"></a><span class="lineno"> 444</span>  <span class="keywordflow">if</span> (successive_ids) {</div>
|
|
<div class="line"><a name="l00445"></a><span class="lineno"> 445</span>  FAISS_THROW_IF_NOT_MSG(!xids,</div>
|
|
<div class="line"><a name="l00446"></a><span class="lineno"> 446</span>  <span class="stringliteral">"It makes no sense to pass in ids and "</span></div>
|
|
<div class="line"><a name="l00447"></a><span class="lineno"> 447</span>  <span class="stringliteral">"request them to be shifted"</span>);</div>
|
|
<div class="line"><a name="l00448"></a><span class="lineno"> 448</span>  FAISS_THROW_IF_NOT_MSG(<a class="code" href="structfaiss_1_1Index.html#a6970683faa021b7a6f1a0865c0d4eccd">ntotal</a> == 0,</div>
|
|
<div class="line"><a name="l00449"></a><span class="lineno"> 449</span>  <span class="stringliteral">"when adding to IndexShards with sucessive_ids, "</span></div>
|
|
<div class="line"><a name="l00450"></a><span class="lineno"> 450</span>  <span class="stringliteral">"only add() in a single pass is supported"</span>);</div>
|
|
<div class="line"><a name="l00451"></a><span class="lineno"> 451</span>  }</div>
|
|
<div class="line"><a name="l00452"></a><span class="lineno"> 452</span> </div>
|
|
<div class="line"><a name="l00453"></a><span class="lineno"> 453</span>  <span class="keywordtype">long</span> nshard = shard_indexes.size();</div>
|
|
<div class="line"><a name="l00454"></a><span class="lineno"> 454</span>  <span class="keyword">const</span> <span class="keywordtype">long</span> *ids = xids;</div>
|
|
<div class="line"><a name="l00455"></a><span class="lineno"> 455</span>  <a class="code" href="structfaiss_1_1ScopeDeleter.html">ScopeDeleter<long></a> del;</div>
|
|
<div class="line"><a name="l00456"></a><span class="lineno"> 456</span>  <span class="keywordflow">if</span> (!ids && !successive_ids) {</div>
|
|
<div class="line"><a name="l00457"></a><span class="lineno"> 457</span>  <span class="keywordtype">long</span> *aids = <span class="keyword">new</span> <span class="keywordtype">long</span>[n];</div>
|
|
<div class="line"><a name="l00458"></a><span class="lineno"> 458</span>  <span class="keywordflow">for</span> (<span class="keywordtype">long</span> i = 0; i < n; i++)</div>
|
|
<div class="line"><a name="l00459"></a><span class="lineno"> 459</span>  aids[i] = <a class="code" href="structfaiss_1_1Index.html#a6970683faa021b7a6f1a0865c0d4eccd">ntotal</a> + i;</div>
|
|
<div class="line"><a name="l00460"></a><span class="lineno"> 460</span>  ids = aids;</div>
|
|
<div class="line"><a name="l00461"></a><span class="lineno"> 461</span>  del.set (ids);</div>
|
|
<div class="line"><a name="l00462"></a><span class="lineno"> 462</span>  }</div>
|
|
<div class="line"><a name="l00463"></a><span class="lineno"> 463</span> </div>
|
|
<div class="line"><a name="l00464"></a><span class="lineno"> 464</span>  std::vector<Thread<AddJob > > asa (shard_indexes.size());</div>
|
|
<div class="line"><a name="l00465"></a><span class="lineno"> 465</span>  <span class="keywordtype">int</span> nt = 0;</div>
|
|
<div class="line"><a name="l00466"></a><span class="lineno"> 466</span>  <span class="keywordflow">for</span> (<span class="keywordtype">int</span> i = 0; i < nshard; i++) {</div>
|
|
<div class="line"><a name="l00467"></a><span class="lineno"> 467</span>  <span class="keywordtype">long</span> i0 = i * n / nshard;</div>
|
|
<div class="line"><a name="l00468"></a><span class="lineno"> 468</span>  <span class="keywordtype">long</span> i1 = (i + 1) * n / nshard;</div>
|
|
<div class="line"><a name="l00469"></a><span class="lineno"> 469</span> </div>
|
|
<div class="line"><a name="l00470"></a><span class="lineno"> 470</span>  AddJob as = {<span class="keyword">this</span>, i,</div>
|
|
<div class="line"><a name="l00471"></a><span class="lineno"> 471</span>  i1 - i0, x + i0 * <a class="code" href="structfaiss_1_1Index.html#a2a002388d2c081c2dbab8508dcefe73d">d</a>,</div>
|
|
<div class="line"><a name="l00472"></a><span class="lineno"> 472</span>  ids ? ids + i0 : <span class="keyword">nullptr</span>};</div>
|
|
<div class="line"><a name="l00473"></a><span class="lineno"> 473</span>  <span class="keywordflow">if</span> (<a class="code" href="structfaiss_1_1IndexShards.html#a493da171720f2c3f35058f793fb9a6ce">threaded</a>) {</div>
|
|
<div class="line"><a name="l00474"></a><span class="lineno"> 474</span>  asa[nt] = Thread<AddJob>(as);</div>
|
|
<div class="line"><a name="l00475"></a><span class="lineno"> 475</span>  asa[nt++].start();</div>
|
|
<div class="line"><a name="l00476"></a><span class="lineno"> 476</span>  } <span class="keywordflow">else</span> {</div>
|
|
<div class="line"><a name="l00477"></a><span class="lineno"> 477</span>  as.run();</div>
|
|
<div class="line"><a name="l00478"></a><span class="lineno"> 478</span>  }</div>
|
|
<div class="line"><a name="l00479"></a><span class="lineno"> 479</span>  }</div>
|
|
<div class="line"><a name="l00480"></a><span class="lineno"> 480</span>  <span class="keywordflow">for</span> (<span class="keywordtype">int</span> i = 0; i < nt; i++) {</div>
|
|
<div class="line"><a name="l00481"></a><span class="lineno"> 481</span>  asa[i].wait();</div>
|
|
<div class="line"><a name="l00482"></a><span class="lineno"> 482</span>  }</div>
|
|
<div class="line"><a name="l00483"></a><span class="lineno"> 483</span>  <a class="code" href="structfaiss_1_1Index.html#a6970683faa021b7a6f1a0865c0d4eccd">ntotal</a> += n;</div>
|
|
<div class="line"><a name="l00484"></a><span class="lineno"> 484</span> }</div>
|
|
<div class="line"><a name="l00485"></a><span class="lineno"> 485</span> </div>
|
|
<div class="line"><a name="l00486"></a><span class="lineno"> 486</span> </div>
|
|
<div class="line"><a name="l00487"></a><span class="lineno"> 487</span> </div>
|
|
<div class="line"><a name="l00488"></a><span class="lineno"> 488</span> </div>
|
|
<div class="line"><a name="l00489"></a><span class="lineno"> 489</span> </div>
|
|
<div class="line"><a name="l00490"></a><span class="lineno"><a class="line" href="structfaiss_1_1IndexShards.html#aa7dc5d020de4a0552058cbf2844cb00f"> 490</a></span> <span class="keywordtype">void</span> <a class="code" href="structfaiss_1_1IndexShards.html#aa7dc5d020de4a0552058cbf2844cb00f">IndexShards::reset</a> ()</div>
|
|
<div class="line"><a name="l00491"></a><span class="lineno"> 491</span> {</div>
|
|
<div class="line"><a name="l00492"></a><span class="lineno"> 492</span>  <span class="keywordflow">for</span> (<span class="keywordtype">int</span> i = 0; i < shard_indexes.size(); i++) {</div>
|
|
<div class="line"><a name="l00493"></a><span class="lineno"> 493</span>  shard_indexes[i]->reset ();</div>
|
|
<div class="line"><a name="l00494"></a><span class="lineno"> 494</span>  }</div>
|
|
<div class="line"><a name="l00495"></a><span class="lineno"> 495</span>  sync_with_shard_indexes ();</div>
|
|
<div class="line"><a name="l00496"></a><span class="lineno"> 496</span> }</div>
|
|
<div class="line"><a name="l00497"></a><span class="lineno"> 497</span> </div>
|
|
<div class="line"><a name="l00498"></a><span class="lineno"><a class="line" href="structfaiss_1_1IndexShards.html#a83391295650370e8987358a898bb2cca"> 498</a></span> <span class="keywordtype">void</span> <a class="code" href="structfaiss_1_1IndexShards.html#a83391295650370e8987358a898bb2cca">IndexShards::search</a> (</div>
|
|
<div class="line"><a name="l00499"></a><span class="lineno"> 499</span>  idx_t n, <span class="keyword">const</span> <span class="keywordtype">float</span> *x, idx_t k,</div>
|
|
<div class="line"><a name="l00500"></a><span class="lineno"> 500</span>  <span class="keywordtype">float</span> *distances, idx_t *labels)<span class="keyword"> const</span></div>
|
|
<div class="line"><a name="l00501"></a><span class="lineno"> 501</span> <span class="keyword"></span>{</div>
|
|
<div class="line"><a name="l00502"></a><span class="lineno"> 502</span>  <span class="keywordtype">long</span> nshard = shard_indexes.size();</div>
|
|
<div class="line"><a name="l00503"></a><span class="lineno"> 503</span>  <span class="keywordtype">float</span> *all_distances = <span class="keyword">new</span> <span class="keywordtype">float</span> [nshard * k * n];</div>
|
|
<div class="line"><a name="l00504"></a><span class="lineno"> 504</span>  idx_t *all_labels = <span class="keyword">new</span> idx_t [nshard * k * n];</div>
|
|
<div class="line"><a name="l00505"></a><span class="lineno"> 505</span>  <a class="code" href="structfaiss_1_1ScopeDeleter.html">ScopeDeleter<float></a> del (all_distances);</div>
|
|
<div class="line"><a name="l00506"></a><span class="lineno"> 506</span>  <a class="code" href="structfaiss_1_1ScopeDeleter.html">ScopeDeleter<idx_t></a> del2 (all_labels);</div>
|
|
<div class="line"><a name="l00507"></a><span class="lineno"> 507</span> </div>
|
|
<div class="line"><a name="l00508"></a><span class="lineno"> 508</span> <span class="preprocessor">#if 1</span></div>
|
|
<div class="line"><a name="l00509"></a><span class="lineno"> 509</span> <span class="preprocessor"></span></div>
|
|
<div class="line"><a name="l00510"></a><span class="lineno"> 510</span>  <span class="comment">// pre-alloc because we don't want reallocs</span></div>
|
|
<div class="line"><a name="l00511"></a><span class="lineno"> 511</span>  std::vector<Thread<QueryJob> > qss (nshard);</div>
|
|
<div class="line"><a name="l00512"></a><span class="lineno"> 512</span>  <span class="keywordflow">for</span> (<span class="keywordtype">int</span> i = 0; i < nshard; i++) {</div>
|
|
<div class="line"><a name="l00513"></a><span class="lineno"> 513</span>  QueryJob qs = {</div>
|
|
<div class="line"><a name="l00514"></a><span class="lineno"> 514</span>  <span class="keyword">this</span>, i, n, x, k,</div>
|
|
<div class="line"><a name="l00515"></a><span class="lineno"> 515</span>  all_distances + i * k * n,</div>
|
|
<div class="line"><a name="l00516"></a><span class="lineno"> 516</span>  all_labels + i * k * n</div>
|
|
<div class="line"><a name="l00517"></a><span class="lineno"> 517</span>  };</div>
|
|
<div class="line"><a name="l00518"></a><span class="lineno"> 518</span>  <span class="keywordflow">if</span> (<a class="code" href="structfaiss_1_1IndexShards.html#a493da171720f2c3f35058f793fb9a6ce">threaded</a>) {</div>
|
|
<div class="line"><a name="l00519"></a><span class="lineno"> 519</span>  qss[i] = Thread<QueryJob> (qs);</div>
|
|
<div class="line"><a name="l00520"></a><span class="lineno"> 520</span>  qss[i].start();</div>
|
|
<div class="line"><a name="l00521"></a><span class="lineno"> 521</span>  } <span class="keywordflow">else</span> {</div>
|
|
<div class="line"><a name="l00522"></a><span class="lineno"> 522</span>  qs.run();</div>
|
|
<div class="line"><a name="l00523"></a><span class="lineno"> 523</span>  }</div>
|
|
<div class="line"><a name="l00524"></a><span class="lineno"> 524</span>  }</div>
|
|
<div class="line"><a name="l00525"></a><span class="lineno"> 525</span> </div>
|
|
<div class="line"><a name="l00526"></a><span class="lineno"> 526</span>  <span class="keywordflow">if</span> (<a class="code" href="structfaiss_1_1IndexShards.html#a493da171720f2c3f35058f793fb9a6ce">threaded</a>) {</div>
|
|
<div class="line"><a name="l00527"></a><span class="lineno"> 527</span>  <span class="keywordflow">for</span> (<span class="keywordtype">int</span> i = 0; i < qss.size(); i++) {</div>
|
|
<div class="line"><a name="l00528"></a><span class="lineno"> 528</span>  qss[i].wait();</div>
|
|
<div class="line"><a name="l00529"></a><span class="lineno"> 529</span>  }</div>
|
|
<div class="line"><a name="l00530"></a><span class="lineno"> 530</span>  }</div>
|
|
<div class="line"><a name="l00531"></a><span class="lineno"> 531</span> <span class="preprocessor">#else</span></div>
|
|
<div class="line"><a name="l00532"></a><span class="lineno"> 532</span> <span class="preprocessor"></span></div>
|
|
<div class="line"><a name="l00533"></a><span class="lineno"> 533</span>  <span class="comment">// pre-alloc because we don't want reallocs</span></div>
|
|
<div class="line"><a name="l00534"></a><span class="lineno"> 534</span>  std::vector<QueryJob> qss (nshard);</div>
|
|
<div class="line"><a name="l00535"></a><span class="lineno"> 535</span>  <span class="keywordflow">for</span> (<span class="keywordtype">int</span> i = 0; i < nshard; i++) {</div>
|
|
<div class="line"><a name="l00536"></a><span class="lineno"> 536</span>  QueryJob qs = {</div>
|
|
<div class="line"><a name="l00537"></a><span class="lineno"> 537</span>  <span class="keyword">this</span>, i, n, x, k,</div>
|
|
<div class="line"><a name="l00538"></a><span class="lineno"> 538</span>  all_distances + i * k * n,</div>
|
|
<div class="line"><a name="l00539"></a><span class="lineno"> 539</span>  all_labels + i * k * n</div>
|
|
<div class="line"><a name="l00540"></a><span class="lineno"> 540</span>  };</div>
|
|
<div class="line"><a name="l00541"></a><span class="lineno"> 541</span>  <span class="keywordflow">if</span> (<a class="code" href="structfaiss_1_1IndexShards.html#a493da171720f2c3f35058f793fb9a6ce">threaded</a>) {</div>
|
|
<div class="line"><a name="l00542"></a><span class="lineno"> 542</span>  qss[i] = qs;</div>
|
|
<div class="line"><a name="l00543"></a><span class="lineno"> 543</span>  } <span class="keywordflow">else</span> {</div>
|
|
<div class="line"><a name="l00544"></a><span class="lineno"> 544</span>  qs.run();</div>
|
|
<div class="line"><a name="l00545"></a><span class="lineno"> 545</span>  }</div>
|
|
<div class="line"><a name="l00546"></a><span class="lineno"> 546</span>  }</div>
|
|
<div class="line"><a name="l00547"></a><span class="lineno"> 547</span> </div>
|
|
<div class="line"><a name="l00548"></a><span class="lineno"> 548</span>  <span class="keywordflow">if</span> (<a class="code" href="structfaiss_1_1IndexShards.html#a493da171720f2c3f35058f793fb9a6ce">threaded</a>) {</div>
|
|
<div class="line"><a name="l00549"></a><span class="lineno"> 549</span> <span class="preprocessor">#pragma omp parallel for</span></div>
|
|
<div class="line"><a name="l00550"></a><span class="lineno"> 550</span> <span class="preprocessor"></span> <span class="keywordflow">for</span> (<span class="keywordtype">int</span> i = 0; i < qss.size(); i++) {</div>
|
|
<div class="line"><a name="l00551"></a><span class="lineno"> 551</span>  qss[i].run();</div>
|
|
<div class="line"><a name="l00552"></a><span class="lineno"> 552</span>  }</div>
|
|
<div class="line"><a name="l00553"></a><span class="lineno"> 553</span>  }</div>
|
|
<div class="line"><a name="l00554"></a><span class="lineno"> 554</span> </div>
|
|
<div class="line"><a name="l00555"></a><span class="lineno"> 555</span> <span class="preprocessor">#endif</span></div>
|
|
<div class="line"><a name="l00556"></a><span class="lineno"> 556</span> <span class="preprocessor"></span> std::vector<long> translations (nshard, 0);</div>
|
|
<div class="line"><a name="l00557"></a><span class="lineno"> 557</span>  <span class="keywordflow">if</span> (successive_ids) {</div>
|
|
<div class="line"><a name="l00558"></a><span class="lineno"> 558</span>  translations[0] = 0;</div>
|
|
<div class="line"><a name="l00559"></a><span class="lineno"> 559</span>  <span class="keywordflow">for</span> (<span class="keywordtype">int</span> s = 0; s + 1 < nshard; s++)</div>
|
|
<div class="line"><a name="l00560"></a><span class="lineno"> 560</span>  translations [s + 1] = translations [s] +</div>
|
|
<div class="line"><a name="l00561"></a><span class="lineno"> 561</span>  shard_indexes [s]-><a class="code" href="structfaiss_1_1Index.html#a6970683faa021b7a6f1a0865c0d4eccd">ntotal</a>;</div>
|
|
<div class="line"><a name="l00562"></a><span class="lineno"> 562</span>  }</div>
|
|
<div class="line"><a name="l00563"></a><span class="lineno"> 563</span> </div>
|
|
<div class="line"><a name="l00564"></a><span class="lineno"> 564</span>  <span class="keywordflow">if</span> (<a class="code" href="structfaiss_1_1Index.html#a8e18f641854b2bde83ecff0a2f9a6f4e">metric_type</a> == METRIC_L2) {</div>
|
|
<div class="line"><a name="l00565"></a><span class="lineno"> 565</span>  merge_tables< CMin<float, int> > (</div>
|
|
<div class="line"><a name="l00566"></a><span class="lineno"> 566</span>  n, k, nshard, distances, labels,</div>
|
|
<div class="line"><a name="l00567"></a><span class="lineno"> 567</span>  all_distances, all_labels, translations.data ());</div>
|
|
<div class="line"><a name="l00568"></a><span class="lineno"> 568</span>  } <span class="keywordflow">else</span> {</div>
|
|
<div class="line"><a name="l00569"></a><span class="lineno"> 569</span>  merge_tables< CMax<float, int> > (</div>
|
|
<div class="line"><a name="l00570"></a><span class="lineno"> 570</span>  n, k, nshard, distances, labels,</div>
|
|
<div class="line"><a name="l00571"></a><span class="lineno"> 571</span>  all_distances, all_labels, translations.data ());</div>
|
|
<div class="line"><a name="l00572"></a><span class="lineno"> 572</span>  }</div>
|
|
<div class="line"><a name="l00573"></a><span class="lineno"> 573</span> </div>
|
|
<div class="line"><a name="l00574"></a><span class="lineno"> 574</span> }</div>
|
|
<div class="line"><a name="l00575"></a><span class="lineno"> 575</span> </div>
|
|
<div class="line"><a name="l00576"></a><span class="lineno"> 576</span> </div>
|
|
<div class="line"><a name="l00577"></a><span class="lineno"> 577</span> </div>
|
|
<div class="line"><a name="l00578"></a><span class="lineno"> 578</span> IndexShards::~IndexShards ()</div>
|
|
<div class="line"><a name="l00579"></a><span class="lineno"> 579</span> {</div>
|
|
<div class="line"><a name="l00580"></a><span class="lineno"> 580</span>  <span class="keywordflow">if</span> (own_fields) {</div>
|
|
<div class="line"><a name="l00581"></a><span class="lineno"> 581</span>  <span class="keywordflow">for</span> (<span class="keywordtype">int</span> s = 0; s < shard_indexes.size(); s++)</div>
|
|
<div class="line"><a name="l00582"></a><span class="lineno"> 582</span>  <span class="keyword">delete</span> shard_indexes [s];</div>
|
|
<div class="line"><a name="l00583"></a><span class="lineno"> 583</span>  }</div>
|
|
<div class="line"><a name="l00584"></a><span class="lineno"> 584</span> }</div>
|
|
<div class="line"><a name="l00585"></a><span class="lineno"> 585</span> </div>
|
|
<div class="line"><a name="l00586"></a><span class="lineno"> 586</span> </div>
|
|
<div class="line"><a name="l00587"></a><span class="lineno"> 587</span> <span class="comment">/*****************************************************</span></div>
|
|
<div class="line"><a name="l00588"></a><span class="lineno"> 588</span> <span class="comment"> * IndexSplitVectors implementation</span></div>
|
|
<div class="line"><a name="l00589"></a><span class="lineno"> 589</span> <span class="comment"> *******************************************************/</span></div>
|
|
<div class="line"><a name="l00590"></a><span class="lineno"> 590</span> </div>
|
|
<div class="line"><a name="l00591"></a><span class="lineno"> 591</span> </div>
|
|
<div class="line"><a name="l00592"></a><span class="lineno"><a class="line" href="structfaiss_1_1IndexSplitVectors.html#a70ba3367a08722fe4466dc49c21a4d2b"> 592</a></span> <a class="code" href="structfaiss_1_1IndexSplitVectors.html#a70ba3367a08722fe4466dc49c21a4d2b">IndexSplitVectors::IndexSplitVectors</a> (idx_t d, <span class="keywordtype">bool</span> threaded):</div>
|
|
<div class="line"><a name="l00593"></a><span class="lineno"> 593</span>  <a class="code" href="structfaiss_1_1Index.html">Index</a> (d), own_fields (false),</div>
|
|
<div class="line"><a name="l00594"></a><span class="lineno"> 594</span>  threaded (threaded), sum_d (0)</div>
|
|
<div class="line"><a name="l00595"></a><span class="lineno"> 595</span> {</div>
|
|
<div class="line"><a name="l00596"></a><span class="lineno"> 596</span> </div>
|
|
<div class="line"><a name="l00597"></a><span class="lineno"> 597</span> }</div>
|
|
<div class="line"><a name="l00598"></a><span class="lineno"> 598</span> </div>
|
|
<div class="line"><a name="l00599"></a><span class="lineno"> 599</span> <span class="keywordtype">void</span> IndexSplitVectors::add_sub_index (<a class="code" href="structfaiss_1_1Index.html">Index</a> *index)</div>
|
|
<div class="line"><a name="l00600"></a><span class="lineno"> 600</span> {</div>
|
|
<div class="line"><a name="l00601"></a><span class="lineno"> 601</span>  sub_indexes.push_back (index);</div>
|
|
<div class="line"><a name="l00602"></a><span class="lineno"> 602</span>  sync_with_sub_indexes ();</div>
|
|
<div class="line"><a name="l00603"></a><span class="lineno"> 603</span> }</div>
|
|
<div class="line"><a name="l00604"></a><span class="lineno"> 604</span> </div>
|
|
<div class="line"><a name="l00605"></a><span class="lineno"> 605</span> <span class="keywordtype">void</span> IndexSplitVectors::sync_with_sub_indexes ()</div>
|
|
<div class="line"><a name="l00606"></a><span class="lineno"> 606</span> {</div>
|
|
<div class="line"><a name="l00607"></a><span class="lineno"> 607</span>  <span class="keywordflow">if</span> (sub_indexes.empty()) <span class="keywordflow">return</span>;</div>
|
|
<div class="line"><a name="l00608"></a><span class="lineno"> 608</span>  Index * index0 = sub_indexes[0];</div>
|
|
<div class="line"><a name="l00609"></a><span class="lineno"> 609</span>  sum_d = index0->d;</div>
|
|
<div class="line"><a name="l00610"></a><span class="lineno"> 610</span>  <a class="code" href="structfaiss_1_1Index.html#a8e18f641854b2bde83ecff0a2f9a6f4e">metric_type</a> = index0->metric_type;</div>
|
|
<div class="line"><a name="l00611"></a><span class="lineno"> 611</span>  <a class="code" href="structfaiss_1_1Index.html#a6e92732617c4dbe364e7678dd8773a7f">is_trained</a> = index0->is_trained;</div>
|
|
<div class="line"><a name="l00612"></a><span class="lineno"> 612</span>  <a class="code" href="structfaiss_1_1Index.html#a6970683faa021b7a6f1a0865c0d4eccd">ntotal</a> = index0->ntotal;</div>
|
|
<div class="line"><a name="l00613"></a><span class="lineno"> 613</span>  <span class="keywordflow">for</span> (<span class="keywordtype">int</span> i = 1; i < sub_indexes.size(); i++) {</div>
|
|
<div class="line"><a name="l00614"></a><span class="lineno"> 614</span>  Index * index = sub_indexes[i];</div>
|
|
<div class="line"><a name="l00615"></a><span class="lineno"> 615</span>  FAISS_THROW_IF_NOT (<a class="code" href="structfaiss_1_1Index.html#a8e18f641854b2bde83ecff0a2f9a6f4e">metric_type</a> == index->metric_type);</div>
|
|
<div class="line"><a name="l00616"></a><span class="lineno"> 616</span>  FAISS_THROW_IF_NOT (<a class="code" href="structfaiss_1_1Index.html#a6970683faa021b7a6f1a0865c0d4eccd">ntotal</a> == index->ntotal);</div>
|
|
<div class="line"><a name="l00617"></a><span class="lineno"> 617</span>  sum_d += index->d;</div>
|
|
<div class="line"><a name="l00618"></a><span class="lineno"> 618</span>  }</div>
|
|
<div class="line"><a name="l00619"></a><span class="lineno"> 619</span> </div>
|
|
<div class="line"><a name="l00620"></a><span class="lineno"> 620</span> }</div>
|
|
<div class="line"><a name="l00621"></a><span class="lineno"> 621</span> </div>
|
|
<div class="line"><a name="l00622"></a><span class="lineno"><a class="line" href="structfaiss_1_1IndexSplitVectors.html#af2ef88dc72b4a94f1190ecc8cb25881b"> 622</a></span> <span class="keywordtype">void</span> <a class="code" href="structfaiss_1_1IndexSplitVectors.html#af2ef88dc72b4a94f1190ecc8cb25881b">IndexSplitVectors::add</a>(idx_t <span class="comment">/*n*/</span>, <span class="keyword">const</span> <span class="keywordtype">float</span>* <span class="comment">/*x*/</span>) {</div>
|
|
<div class="line"><a name="l00623"></a><span class="lineno"> 623</span>  FAISS_THROW_MSG(<span class="stringliteral">"not implemented"</span>);</div>
|
|
<div class="line"><a name="l00624"></a><span class="lineno"> 624</span> }</div>
|
|
<div class="line"><a name="l00625"></a><span class="lineno"> 625</span> </div>
|
|
<div class="line"><a name="l00626"></a><span class="lineno"> 626</span> <span class="keyword">namespace </span>{</div>
|
|
<div class="line"><a name="l00627"></a><span class="lineno"> 627</span> <span class="comment"></span></div>
|
|
<div class="line"><a name="l00628"></a><span class="lineno"> 628</span> <span class="comment">/// callback + thread management to query in 1 shard</span></div>
|
|
<div class="line"><a name="l00629"></a><span class="lineno"> 629</span> <span class="comment"></span><span class="keyword">struct </span>SplitQueryJob {</div>
|
|
<div class="line"><a name="l00630"></a><span class="lineno"> 630</span>  <span class="keyword">const</span> <a class="code" href="structfaiss_1_1IndexSplitVectors.html">IndexSplitVectors</a> *index; <span class="comment">// the relevant index</span></div>
|
|
<div class="line"><a name="l00631"></a><span class="lineno"> 631</span>  <span class="keywordtype">int</span> no; <span class="comment">// shard number</span></div>
|
|
<div class="line"><a name="l00632"></a><span class="lineno"> 632</span> </div>
|
|
<div class="line"><a name="l00633"></a><span class="lineno"> 633</span>  <span class="comment">// query params</span></div>
|
|
<div class="line"><a name="l00634"></a><span class="lineno"> 634</span>  idx_t n;</div>
|
|
<div class="line"><a name="l00635"></a><span class="lineno"> 635</span>  <span class="keyword">const</span> <span class="keywordtype">float</span> *x;</div>
|
|
<div class="line"><a name="l00636"></a><span class="lineno"> 636</span>  idx_t k;</div>
|
|
<div class="line"><a name="l00637"></a><span class="lineno"> 637</span>  <span class="keywordtype">float</span> *distances;</div>
|
|
<div class="line"><a name="l00638"></a><span class="lineno"> 638</span>  idx_t *labels;</div>
|
|
<div class="line"><a name="l00639"></a><span class="lineno"> 639</span> </div>
|
|
<div class="line"><a name="l00640"></a><span class="lineno"> 640</span> </div>
|
|
<div class="line"><a name="l00641"></a><span class="lineno"> 641</span>  <span class="keywordtype">void</span> run ()</div>
|
|
<div class="line"><a name="l00642"></a><span class="lineno"> 642</span>  {</div>
|
|
<div class="line"><a name="l00643"></a><span class="lineno"> 643</span>  <span class="keywordflow">if</span> (index-><a class="code" href="structfaiss_1_1Index.html#a5590d847c5c2b958affd2a05e58a6f23">verbose</a>)</div>
|
|
<div class="line"><a name="l00644"></a><span class="lineno"> 644</span>  printf (<span class="stringliteral">"begin query shard %d on %ld points\n"</span>, no, n);</div>
|
|
<div class="line"><a name="l00645"></a><span class="lineno"> 645</span>  <span class="keyword">const</span> <a class="code" href="structfaiss_1_1Index.html">Index</a> * sub_index = index->sub_indexes[no];</div>
|
|
<div class="line"><a name="l00646"></a><span class="lineno"> 646</span>  <span class="keywordtype">long</span> sub_d = sub_index-><a class="code" href="structfaiss_1_1Index.html#a2a002388d2c081c2dbab8508dcefe73d">d</a>, d = index-><a class="code" href="structfaiss_1_1Index.html#a2a002388d2c081c2dbab8508dcefe73d">d</a>;</div>
|
|
<div class="line"><a name="l00647"></a><span class="lineno"> 647</span>  idx_t ofs = 0;</div>
|
|
<div class="line"><a name="l00648"></a><span class="lineno"> 648</span>  <span class="keywordflow">for</span> (<span class="keywordtype">int</span> i = 0; i < no; i++) ofs += index->sub_indexes[i]-><a class="code" href="structfaiss_1_1Index.html#a2a002388d2c081c2dbab8508dcefe73d">d</a>;</div>
|
|
<div class="line"><a name="l00649"></a><span class="lineno"> 649</span>  <span class="keywordtype">float</span> *sub_x = <span class="keyword">new</span> <span class="keywordtype">float</span> [sub_d * n];</div>
|
|
<div class="line"><a name="l00650"></a><span class="lineno"> 650</span>  <a class="code" href="structfaiss_1_1ScopeDeleter.html">ScopeDeleter<float></a> del (sub_x);</div>
|
|
<div class="line"><a name="l00651"></a><span class="lineno"> 651</span>  <span class="keywordflow">for</span> (idx_t i = 0; i < n; i++)</div>
|
|
<div class="line"><a name="l00652"></a><span class="lineno"> 652</span>  memcpy (sub_x + i * sub_d, x + ofs + i * d, sub_d * <span class="keyword">sizeof</span> (sub_x));</div>
|
|
<div class="line"><a name="l00653"></a><span class="lineno"> 653</span>  sub_index-><a class="code" href="structfaiss_1_1Index.html#aced51b1ebc33c47ab3ae15ea906559a7">search</a> (n, sub_x, k, distances, labels);</div>
|
|
<div class="line"><a name="l00654"></a><span class="lineno"> 654</span>  <span class="keywordflow">if</span> (index-><a class="code" href="structfaiss_1_1Index.html#a5590d847c5c2b958affd2a05e58a6f23">verbose</a>)</div>
|
|
<div class="line"><a name="l00655"></a><span class="lineno"> 655</span>  printf (<span class="stringliteral">"end query shard %d\n"</span>, no);</div>
|
|
<div class="line"><a name="l00656"></a><span class="lineno"> 656</span>  }</div>
|
|
<div class="line"><a name="l00657"></a><span class="lineno"> 657</span> </div>
|
|
<div class="line"><a name="l00658"></a><span class="lineno"> 658</span> };</div>
|
|
<div class="line"><a name="l00659"></a><span class="lineno"> 659</span> </div>
|
|
<div class="line"><a name="l00660"></a><span class="lineno"> 660</span> </div>
|
|
<div class="line"><a name="l00661"></a><span class="lineno"> 661</span> </div>
|
|
<div class="line"><a name="l00662"></a><span class="lineno"> 662</span> }</div>
|
|
<div class="line"><a name="l00663"></a><span class="lineno"> 663</span> </div>
|
|
<div class="line"><a name="l00664"></a><span class="lineno"> 664</span> </div>
|
|
<div class="line"><a name="l00665"></a><span class="lineno"> 665</span> </div>
|
|
<div class="line"><a name="l00666"></a><span class="lineno"><a class="line" href="structfaiss_1_1IndexSplitVectors.html#a4a010daf51e44d9717e94400cc23c159"> 666</a></span> <span class="keywordtype">void</span> <a class="code" href="structfaiss_1_1IndexSplitVectors.html#a4a010daf51e44d9717e94400cc23c159">IndexSplitVectors::search</a> (</div>
|
|
<div class="line"><a name="l00667"></a><span class="lineno"> 667</span>  idx_t n, <span class="keyword">const</span> <span class="keywordtype">float</span> *x, idx_t k,</div>
|
|
<div class="line"><a name="l00668"></a><span class="lineno"> 668</span>  <span class="keywordtype">float</span> *distances, idx_t *labels)<span class="keyword"> const</span></div>
|
|
<div class="line"><a name="l00669"></a><span class="lineno"> 669</span> <span class="keyword"></span>{</div>
|
|
<div class="line"><a name="l00670"></a><span class="lineno"> 670</span>  FAISS_THROW_IF_NOT_MSG (k == 1,</div>
|
|
<div class="line"><a name="l00671"></a><span class="lineno"> 671</span>  <span class="stringliteral">"search implemented only for k=1"</span>);</div>
|
|
<div class="line"><a name="l00672"></a><span class="lineno"> 672</span>  FAISS_THROW_IF_NOT_MSG (sum_d == d,</div>
|
|
<div class="line"><a name="l00673"></a><span class="lineno"> 673</span>  <span class="stringliteral">"not enough indexes compared to # dimensions"</span>);</div>
|
|
<div class="line"><a name="l00674"></a><span class="lineno"> 674</span> </div>
|
|
<div class="line"><a name="l00675"></a><span class="lineno"> 675</span>  <span class="keywordtype">long</span> nshard = sub_indexes.size();</div>
|
|
<div class="line"><a name="l00676"></a><span class="lineno"> 676</span>  <span class="keywordtype">float</span> *all_distances = <span class="keyword">new</span> <span class="keywordtype">float</span> [nshard * k * n];</div>
|
|
<div class="line"><a name="l00677"></a><span class="lineno"> 677</span>  idx_t *all_labels = <span class="keyword">new</span> idx_t [nshard * k * n];</div>
|
|
<div class="line"><a name="l00678"></a><span class="lineno"> 678</span>  <a class="code" href="structfaiss_1_1ScopeDeleter.html">ScopeDeleter<float></a> del (all_distances);</div>
|
|
<div class="line"><a name="l00679"></a><span class="lineno"> 679</span>  <a class="code" href="structfaiss_1_1ScopeDeleter.html">ScopeDeleter<idx_t></a> del2 (all_labels);</div>
|
|
<div class="line"><a name="l00680"></a><span class="lineno"> 680</span> </div>
|
|
<div class="line"><a name="l00681"></a><span class="lineno"> 681</span>  <span class="comment">// pre-alloc because we don't want reallocs</span></div>
|
|
<div class="line"><a name="l00682"></a><span class="lineno"> 682</span>  std::vector<Thread<SplitQueryJob> > qss (nshard);</div>
|
|
<div class="line"><a name="l00683"></a><span class="lineno"> 683</span>  <span class="keywordflow">for</span> (<span class="keywordtype">int</span> i = 0; i < nshard; i++) {</div>
|
|
<div class="line"><a name="l00684"></a><span class="lineno"> 684</span>  SplitQueryJob qs = {</div>
|
|
<div class="line"><a name="l00685"></a><span class="lineno"> 685</span>  <span class="keyword">this</span>, i, n, x, k,</div>
|
|
<div class="line"><a name="l00686"></a><span class="lineno"> 686</span>  i == 0 ? distances : all_distances + i * k * n,</div>
|
|
<div class="line"><a name="l00687"></a><span class="lineno"> 687</span>  i == 0 ? labels : all_labels + i * k * n</div>
|
|
<div class="line"><a name="l00688"></a><span class="lineno"> 688</span>  };</div>
|
|
<div class="line"><a name="l00689"></a><span class="lineno"> 689</span>  <span class="keywordflow">if</span> (threaded) {</div>
|
|
<div class="line"><a name="l00690"></a><span class="lineno"> 690</span>  qss[i] = Thread<SplitQueryJob> (qs);</div>
|
|
<div class="line"><a name="l00691"></a><span class="lineno"> 691</span>  qss[i].start();</div>
|
|
<div class="line"><a name="l00692"></a><span class="lineno"> 692</span>  } <span class="keywordflow">else</span> {</div>
|
|
<div class="line"><a name="l00693"></a><span class="lineno"> 693</span>  qs.run();</div>
|
|
<div class="line"><a name="l00694"></a><span class="lineno"> 694</span>  }</div>
|
|
<div class="line"><a name="l00695"></a><span class="lineno"> 695</span>  }</div>
|
|
<div class="line"><a name="l00696"></a><span class="lineno"> 696</span> </div>
|
|
<div class="line"><a name="l00697"></a><span class="lineno"> 697</span>  <span class="keywordflow">if</span> (threaded) {</div>
|
|
<div class="line"><a name="l00698"></a><span class="lineno"> 698</span>  <span class="keywordflow">for</span> (<span class="keywordtype">int</span> i = 0; i < qss.size(); i++) {</div>
|
|
<div class="line"><a name="l00699"></a><span class="lineno"> 699</span>  qss[i].wait();</div>
|
|
<div class="line"><a name="l00700"></a><span class="lineno"> 700</span>  }</div>
|
|
<div class="line"><a name="l00701"></a><span class="lineno"> 701</span>  }</div>
|
|
<div class="line"><a name="l00702"></a><span class="lineno"> 702</span> </div>
|
|
<div class="line"><a name="l00703"></a><span class="lineno"> 703</span>  <span class="keywordtype">long</span> factor = 1;</div>
|
|
<div class="line"><a name="l00704"></a><span class="lineno"> 704</span>  <span class="keywordflow">for</span> (<span class="keywordtype">int</span> i = 0; i < nshard; i++) {</div>
|
|
<div class="line"><a name="l00705"></a><span class="lineno"> 705</span>  <span class="keywordflow">if</span> (i > 0) { <span class="comment">// results of 0 are already in the table</span></div>
|
|
<div class="line"><a name="l00706"></a><span class="lineno"> 706</span>  <span class="keyword">const</span> <span class="keywordtype">float</span> *distances_i = all_distances + i * k * n;</div>
|
|
<div class="line"><a name="l00707"></a><span class="lineno"> 707</span>  <span class="keyword">const</span> idx_t *labels_i = all_labels + i * k * n;</div>
|
|
<div class="line"><a name="l00708"></a><span class="lineno"> 708</span>  <span class="keywordflow">for</span> (<span class="keywordtype">long</span> j = 0; j < n; j++) {</div>
|
|
<div class="line"><a name="l00709"></a><span class="lineno"> 709</span>  <span class="keywordflow">if</span> (labels[j] >= 0 && labels_i[j] >= 0) {</div>
|
|
<div class="line"><a name="l00710"></a><span class="lineno"> 710</span>  labels[j] += labels_i[j] * factor;</div>
|
|
<div class="line"><a name="l00711"></a><span class="lineno"> 711</span>  distances[j] += distances_i[j];</div>
|
|
<div class="line"><a name="l00712"></a><span class="lineno"> 712</span>  } <span class="keywordflow">else</span> {</div>
|
|
<div class="line"><a name="l00713"></a><span class="lineno"> 713</span>  labels[j] = -1;</div>
|
|
<div class="line"><a name="l00714"></a><span class="lineno"> 714</span>  distances[j] = 0.0 / 0.0;</div>
|
|
<div class="line"><a name="l00715"></a><span class="lineno"> 715</span>  }</div>
|
|
<div class="line"><a name="l00716"></a><span class="lineno"> 716</span>  }</div>
|
|
<div class="line"><a name="l00717"></a><span class="lineno"> 717</span>  }</div>
|
|
<div class="line"><a name="l00718"></a><span class="lineno"> 718</span>  factor *= sub_indexes[i]->ntotal;</div>
|
|
<div class="line"><a name="l00719"></a><span class="lineno"> 719</span>  }</div>
|
|
<div class="line"><a name="l00720"></a><span class="lineno"> 720</span> </div>
|
|
<div class="line"><a name="l00721"></a><span class="lineno"> 721</span> }</div>
|
|
<div class="line"><a name="l00722"></a><span class="lineno"> 722</span> </div>
|
|
<div class="line"><a name="l00723"></a><span class="lineno"><a class="line" href="structfaiss_1_1IndexSplitVectors.html#aafa4c780e325855fe247052453967845"> 723</a></span> <span class="keywordtype">void</span> <a class="code" href="structfaiss_1_1IndexSplitVectors.html#aafa4c780e325855fe247052453967845">IndexSplitVectors::train</a>(idx_t <span class="comment">/*n*/</span>, <span class="keyword">const</span> <span class="keywordtype">float</span>* <span class="comment">/*x*/</span>) {</div>
|
|
<div class="line"><a name="l00724"></a><span class="lineno"> 724</span>  FAISS_THROW_MSG(<span class="stringliteral">"not implemented"</span>);</div>
|
|
<div class="line"><a name="l00725"></a><span class="lineno"> 725</span> }</div>
|
|
<div class="line"><a name="l00726"></a><span class="lineno"> 726</span> </div>
|
|
<div class="line"><a name="l00727"></a><span class="lineno"><a class="line" href="structfaiss_1_1IndexSplitVectors.html#a8047981cb8d10c54107664c4955df402"> 727</a></span> <span class="keywordtype">void</span> <a class="code" href="structfaiss_1_1IndexSplitVectors.html#a8047981cb8d10c54107664c4955df402">IndexSplitVectors::reset</a> ()</div>
|
|
<div class="line"><a name="l00728"></a><span class="lineno"> 728</span> {</div>
|
|
<div class="line"><a name="l00729"></a><span class="lineno"> 729</span>  FAISS_THROW_MSG (<span class="stringliteral">"not implemented"</span>);</div>
|
|
<div class="line"><a name="l00730"></a><span class="lineno"> 730</span> }</div>
|
|
<div class="line"><a name="l00731"></a><span class="lineno"> 731</span> </div>
|
|
<div class="line"><a name="l00732"></a><span class="lineno"> 732</span> </div>
|
|
<div class="line"><a name="l00733"></a><span class="lineno"> 733</span> IndexSplitVectors::~IndexSplitVectors ()</div>
|
|
<div class="line"><a name="l00734"></a><span class="lineno"> 734</span> {</div>
|
|
<div class="line"><a name="l00735"></a><span class="lineno"> 735</span>  <span class="keywordflow">if</span> (own_fields) {</div>
|
|
<div class="line"><a name="l00736"></a><span class="lineno"> 736</span>  <span class="keywordflow">for</span> (<span class="keywordtype">int</span> s = 0; s < sub_indexes.size(); s++)</div>
|
|
<div class="line"><a name="l00737"></a><span class="lineno"> 737</span>  <span class="keyword">delete</span> sub_indexes [s];</div>
|
|
<div class="line"><a name="l00738"></a><span class="lineno"> 738</span>  }</div>
|
|
<div class="line"><a name="l00739"></a><span class="lineno"> 739</span> }</div>
|
|
<div class="line"><a name="l00740"></a><span class="lineno"> 740</span> </div>
|
|
<div class="line"><a name="l00741"></a><span class="lineno"> 741</span> </div>
|
|
<div class="line"><a name="l00742"></a><span class="lineno"> 742</span> </div>
|
|
<div class="line"><a name="l00743"></a><span class="lineno"> 743</span> </div>
|
|
<div class="line"><a name="l00744"></a><span class="lineno"> 744</span> </div>
|
|
<div class="line"><a name="l00745"></a><span class="lineno"> 745</span> </div>
|
|
<div class="line"><a name="l00746"></a><span class="lineno"> 746</span> }; <span class="comment">// namespace faiss</span></div>
|
|
<div class="ttc" id="structfaiss_1_1IndexShards_html_a36fbcc85eed9bf2ea15587f04be9ae0b"><div class="ttname"><a href="structfaiss_1_1IndexShards.html#a36fbcc85eed9bf2ea15587f04be9ae0b">faiss::IndexShards::train</a></div><div class="ttdeci">void train(idx_t n, const float *x) override</div><div class="ttdef"><b>Definition:</b> <a href="MetaIndexes_8cpp_source.html#l00397">MetaIndexes.cpp:397</a></div></div>
|
|
<div class="ttc" id="structfaiss_1_1RangeSearchResult_html_a29269b98c33af874d379ca4f62408b90"><div class="ttname"><a href="structfaiss_1_1RangeSearchResult.html#a29269b98c33af874d379ca4f62408b90">faiss::RangeSearchResult::nq</a></div><div class="ttdeci">size_t nq</div><div class="ttdoc">nb of queries </div><div class="ttdef"><b>Definition:</b> <a href="AuxIndexStructures_8h_source.html#l00030">AuxIndexStructures.h:30</a></div></div>
|
|
<div class="ttc" id="structfaiss_1_1IndexShards_html_ad987a18a1786919e32cc02ca36e5ab46"><div class="ttname"><a href="structfaiss_1_1IndexShards.html#ad987a18a1786919e32cc02ca36e5ab46">faiss::IndexShards::IndexShards</a></div><div class="ttdeci">IndexShards(idx_t d, bool threaded=false, bool successive_ids=true)</div><div class="ttdef"><b>Definition:</b> <a href="MetaIndexes_8cpp_source.html#l00367">MetaIndexes.cpp:367</a></div></div>
|
|
<div class="ttc" id="structfaiss_1_1Index_html_a849361f5f0ab0aba8d419c86f2594191"><div class="ttname"><a href="structfaiss_1_1Index.html#a849361f5f0ab0aba8d419c86f2594191">faiss::Index::reset</a></div><div class="ttdeci">virtual void reset()=0</div><div class="ttdoc">removes all elements from the database. </div></div>
|
|
<div class="ttc" id="structfaiss_1_1Index_html_a1ffe916c958605c38b0b1bfad42485e4"><div class="ttname"><a href="structfaiss_1_1Index.html#a1ffe916c958605c38b0b1bfad42485e4">faiss::Index::train</a></div><div class="ttdeci">virtual void train(idx_t, const float *)</div><div class="ttdef"><b>Definition:</b> <a href="Index_8h_source.html#l00089">Index.h:89</a></div></div>
|
|
<div class="ttc" id="structfaiss_1_1IndexIDMap_html_a2f36fa9150a465b48952afbba856e2a1"><div class="ttname"><a href="structfaiss_1_1IndexIDMap.html#a2f36fa9150a465b48952afbba856e2a1">faiss::IndexIDMap::search</a></div><div class="ttdeci">void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const override</div><div class="ttdef"><b>Definition:</b> <a href="MetaIndexes_8cpp_source.html#l00069">MetaIndexes.cpp:69</a></div></div>
|
|
<div class="ttc" id="structfaiss_1_1IndexShards_html_a1f771d50133494f682f05c9ab9e39ad9"><div class="ttname"><a href="structfaiss_1_1IndexShards.html#a1f771d50133494f682f05c9ab9e39ad9">faiss::IndexShards::add_with_ids</a></div><div class="ttdeci">void add_with_ids(idx_t n, const float *x, const long *xids) override</div><div class="ttdef"><b>Definition:</b> <a href="MetaIndexes_8cpp_source.html#l00437">MetaIndexes.cpp:437</a></div></div>
|
|
<div class="ttc" id="structfaiss_1_1IDSelector_html"><div class="ttname"><a href="structfaiss_1_1IDSelector.html">faiss::IDSelector</a></div><div class="ttdef"><b>Definition:</b> <a href="AuxIndexStructures_8h_source.html#l00052">AuxIndexStructures.h:52</a></div></div>
|
|
<div class="ttc" id="structfaiss_1_1IndexShards_html_a83391295650370e8987358a898bb2cca"><div class="ttname"><a href="structfaiss_1_1IndexShards.html#a83391295650370e8987358a898bb2cca">faiss::IndexShards::search</a></div><div class="ttdeci">void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const override</div><div class="ttdef"><b>Definition:</b> <a href="MetaIndexes_8cpp_source.html#l00498">MetaIndexes.cpp:498</a></div></div>
|
|
<div class="ttc" id="structfaiss_1_1IndexIDMap2_html_a33ff76975891695c2fa45ed26fc8eae9"><div class="ttname"><a href="structfaiss_1_1IndexIDMap2.html#a33ff76975891695c2fa45ed26fc8eae9">faiss::IndexIDMap2::add_with_ids</a></div><div class="ttdeci">void add_with_ids(idx_t n, const float *x, const long *xids) override</div><div class="ttdef"><b>Definition:</b> <a href="MetaIndexes_8cpp_source.html#l00142">MetaIndexes.cpp:142</a></div></div>
|
|
<div class="ttc" id="structfaiss_1_1ScopeDeleter_html"><div class="ttname"><a href="structfaiss_1_1ScopeDeleter.html">faiss::ScopeDeleter</a></div><div class="ttdef"><b>Definition:</b> <a href="FaissException_8h_source.html#l00039">FaissException.h:39</a></div></div>
|
|
<div class="ttc" id="structfaiss_1_1IndexSplitVectors_html_a8047981cb8d10c54107664c4955df402"><div class="ttname"><a href="structfaiss_1_1IndexSplitVectors.html#a8047981cb8d10c54107664c4955df402">faiss::IndexSplitVectors::reset</a></div><div class="ttdeci">void reset() override</div><div class="ttdoc">removes all elements from the database. </div><div class="ttdef"><b>Definition:</b> <a href="MetaIndexes_8cpp_source.html#l00727">MetaIndexes.cpp:727</a></div></div>
|
|
<div class="ttc" id="structfaiss_1_1IndexIDMap_html_a16cce6c1e83d0d5f069c9b0399896450"><div class="ttname"><a href="structfaiss_1_1IndexIDMap.html#a16cce6c1e83d0d5f069c9b0399896450">faiss::IndexIDMap::add</a></div><div class="ttdeci">void add(idx_t n, const float *x) override</div><div class="ttdoc">this will fail. Use add_with_ids </div><div class="ttdef"><b>Definition:</b> <a href="MetaIndexes_8cpp_source.html#l00040">MetaIndexes.cpp:40</a></div></div>
|
|
<div class="ttc" id="structfaiss_1_1IndexSplitVectors_html"><div class="ttname"><a href="structfaiss_1_1IndexSplitVectors.html">faiss::IndexSplitVectors</a></div><div class="ttdef"><b>Definition:</b> <a href="MetaIndexes_8h_source.html#l00134">MetaIndexes.h:134</a></div></div>
|
|
<div class="ttc" id="structfaiss_1_1Index_html_a2a002388d2c081c2dbab8508dcefe73d"><div class="ttname"><a href="structfaiss_1_1Index.html#a2a002388d2c081c2dbab8508dcefe73d">faiss::Index::d</a></div><div class="ttdeci">int d</div><div class="ttdoc">vector dimension </div><div class="ttdef"><b>Definition:</b> <a href="Index_8h_source.html#l00064">Index.h:64</a></div></div>
|
|
<div class="ttc" id="structfaiss_1_1IndexIDMap_html_ad1cc5658f1879f6a2b00d76cca58cf3b"><div class="ttname"><a href="structfaiss_1_1IndexIDMap.html#ad1cc5658f1879f6a2b00d76cca58cf3b">faiss::IndexIDMap::id_map</a></div><div class="ttdeci">std::vector< long > id_map</div><div class="ttdoc">! whether pointers are deleted in destructo </div><div class="ttdef"><b>Definition:</b> <a href="MetaIndexes_8h_source.html#l00029">MetaIndexes.h:29</a></div></div>
|
|
<div class="ttc" id="structfaiss_1_1IndexIDMap_html_a7dbeaf9d44d6783d371f12405e8b7f40"><div class="ttname"><a href="structfaiss_1_1IndexIDMap.html#a7dbeaf9d44d6783d371f12405e8b7f40">faiss::IndexIDMap::range_search</a></div><div class="ttdeci">void range_search(idx_t n, const float *x, float radius, RangeSearchResult *result) const override</div><div class="ttdef"><b>Definition:</b> <a href="MetaIndexes_8cpp_source.html#l00080">MetaIndexes.cpp:80</a></div></div>
|
|
<div class="ttc" id="structfaiss_1_1Index_html_a1b5e9ac70adbce0897dd6c8276ad96f2"><div class="ttname"><a href="structfaiss_1_1Index.html#a1b5e9ac70adbce0897dd6c8276ad96f2">faiss::Index::add</a></div><div class="ttdeci">virtual void add(idx_t n, const float *x)=0</div></div>
|
|
<div class="ttc" id="structfaiss_1_1IndexSplitVectors_html_aafa4c780e325855fe247052453967845"><div class="ttname"><a href="structfaiss_1_1IndexSplitVectors.html#aafa4c780e325855fe247052453967845">faiss::IndexSplitVectors::train</a></div><div class="ttdeci">void train(idx_t n, const float *x) override</div><div class="ttdef"><b>Definition:</b> <a href="MetaIndexes_8cpp_source.html#l00723">MetaIndexes.cpp:723</a></div></div>
|
|
<div class="ttc" id="structfaiss_1_1IndexShards_html_a2e06889521c566525011eb46cc97cda6"><div class="ttname"><a href="structfaiss_1_1IndexShards.html#a2e06889521c566525011eb46cc97cda6">faiss::IndexShards::add</a></div><div class="ttdeci">void add(idx_t n, const float *x) override</div><div class="ttdoc">supported only for sub-indices that implement add_with_ids </div><div class="ttdef"><b>Definition:</b> <a href="MetaIndexes_8cpp_source.html#l00420">MetaIndexes.cpp:420</a></div></div>
|
|
<div class="ttc" id="structfaiss_1_1Index_html_a040c6aed1f224f3ea7bf58eebc0c31a4"><div class="ttname"><a href="structfaiss_1_1Index.html#a040c6aed1f224f3ea7bf58eebc0c31a4">faiss::Index::idx_t</a></div><div class="ttdeci">long idx_t</div><div class="ttdoc">all indices are this type </div><div class="ttdef"><b>Definition:</b> <a href="Index_8h_source.html#l00062">Index.h:62</a></div></div>
|
|
<div class="ttc" id="structfaiss_1_1Index_html_a6970683faa021b7a6f1a0865c0d4eccd"><div class="ttname"><a href="structfaiss_1_1Index.html#a6970683faa021b7a6f1a0865c0d4eccd">faiss::Index::ntotal</a></div><div class="ttdeci">idx_t ntotal</div><div class="ttdoc">total nb of indexed vectors </div><div class="ttdef"><b>Definition:</b> <a href="Index_8h_source.html#l00065">Index.h:65</a></div></div>
|
|
<div class="ttc" id="structfaiss_1_1IndexIDMap2_html_a646302669189e370d9d864fc18f096fe"><div class="ttname"><a href="structfaiss_1_1IndexIDMap2.html#a646302669189e370d9d864fc18f096fe">faiss::IndexIDMap2::construct_rev_map</a></div><div class="ttdeci">void construct_rev_map()</div><div class="ttdoc">make the rev_map from scratch </div><div class="ttdef"><b>Definition:</b> <a href="MetaIndexes_8cpp_source.html#l00151">MetaIndexes.cpp:151</a></div></div>
|
|
<div class="ttc" id="structfaiss_1_1Index_html_a5590d847c5c2b958affd2a05e58a6f23"><div class="ttname"><a href="structfaiss_1_1Index.html#a5590d847c5c2b958affd2a05e58a6f23">faiss::Index::verbose</a></div><div class="ttdeci">bool verbose</div><div class="ttdoc">verbosity level </div><div class="ttdef"><b>Definition:</b> <a href="Index_8h_source.html#l00066">Index.h:66</a></div></div>
|
|
<div class="ttc" id="structfaiss_1_1IndexSplitVectors_html_af2ef88dc72b4a94f1190ecc8cb25881b"><div class="ttname"><a href="structfaiss_1_1IndexSplitVectors.html#af2ef88dc72b4a94f1190ecc8cb25881b">faiss::IndexSplitVectors::add</a></div><div class="ttdeci">void add(idx_t n, const float *x) override</div><div class="ttdef"><b>Definition:</b> <a href="MetaIndexes_8cpp_source.html#l00622">MetaIndexes.cpp:622</a></div></div>
|
|
<div class="ttc" id="structfaiss_1_1Index_html_af8bf7bd97aeed8ad5fc48b242379a68a"><div class="ttname"><a href="structfaiss_1_1Index.html#af8bf7bd97aeed8ad5fc48b242379a68a">faiss::Index::remove_ids</a></div><div class="ttdeci">virtual long remove_ids(const IDSelector &sel)</div><div class="ttdef"><b>Definition:</b> <a href="Index_8cpp_source.html#l00037">Index.cpp:37</a></div></div>
|
|
<div class="ttc" id="structfaiss_1_1IndexIDMap2_html_a87e313ebc8dd62fb734a61a6d95a98f0"><div class="ttname"><a href="structfaiss_1_1IndexIDMap2.html#a87e313ebc8dd62fb734a61a6d95a98f0">faiss::IndexIDMap2::remove_ids</a></div><div class="ttdeci">long remove_ids(const IDSelector &sel) override</div><div class="ttdoc">remove ids adapted to IndexFlat </div><div class="ttdef"><b>Definition:</b> <a href="MetaIndexes_8cpp_source.html#l00160">MetaIndexes.cpp:160</a></div></div>
|
|
<div class="ttc" id="structfaiss_1_1IndexShards_html_a493da171720f2c3f35058f793fb9a6ce"><div class="ttname"><a href="structfaiss_1_1IndexShards.html#a493da171720f2c3f35058f793fb9a6ce">faiss::IndexShards::threaded</a></div><div class="ttdeci">bool threaded</div><div class="ttdoc">should the sub-indexes be deleted along with this? </div><div class="ttdef"><b>Definition:</b> <a href="MetaIndexes_8h_source.html#l00090">MetaIndexes.h:90</a></div></div>
|
|
<div class="ttc" id="structfaiss_1_1Index_html_aced51b1ebc33c47ab3ae15ea906559a7"><div class="ttname"><a href="structfaiss_1_1Index.html#aced51b1ebc33c47ab3ae15ea906559a7">faiss::Index::search</a></div><div class="ttdeci">virtual void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const =0</div></div>
|
|
<div class="ttc" id="structfaiss_1_1IndexShards_html_aa7dc5d020de4a0552058cbf2844cb00f"><div class="ttname"><a href="structfaiss_1_1IndexShards.html#aa7dc5d020de4a0552058cbf2844cb00f">faiss::IndexShards::reset</a></div><div class="ttdeci">void reset() override</div><div class="ttdoc">removes all elements from the database. </div><div class="ttdef"><b>Definition:</b> <a href="MetaIndexes_8cpp_source.html#l00490">MetaIndexes.cpp:490</a></div></div>
|
|
<div class="ttc" id="structfaiss_1_1RangeSearchResult_html"><div class="ttname"><a href="structfaiss_1_1RangeSearchResult.html">faiss::RangeSearchResult</a></div><div class="ttdef"><b>Definition:</b> <a href="AuxIndexStructures_8h_source.html#l00029">AuxIndexStructures.h:29</a></div></div>
|
|
<div class="ttc" id="structfaiss_1_1IndexIDMap_html_a70aca5b0f665c5a109d5dadf1e13b448"><div class="ttname"><a href="structfaiss_1_1IndexIDMap.html#a70aca5b0f665c5a109d5dadf1e13b448">faiss::IndexIDMap::reset</a></div><div class="ttdeci">void reset() override</div><div class="ttdoc">removes all elements from the database. </div><div class="ttdef"><b>Definition:</b> <a href="MetaIndexes_8cpp_source.html#l00053">MetaIndexes.cpp:53</a></div></div>
|
|
<div class="ttc" id="structfaiss_1_1IndexSplitVectors_html_a4a010daf51e44d9717e94400cc23c159"><div class="ttname"><a href="structfaiss_1_1IndexSplitVectors.html#a4a010daf51e44d9717e94400cc23c159">faiss::IndexSplitVectors::search</a></div><div class="ttdeci">void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const override</div><div class="ttdef"><b>Definition:</b> <a href="MetaIndexes_8cpp_source.html#l00666">MetaIndexes.cpp:666</a></div></div>
|
|
<div class="ttc" id="structfaiss_1_1IndexIDMap_html_a699ca4891bf3a437b2a182c04d33d9b9"><div class="ttname"><a href="structfaiss_1_1IndexIDMap.html#a699ca4891bf3a437b2a182c04d33d9b9">faiss::IndexIDMap::remove_ids</a></div><div class="ttdeci">long remove_ids(const IDSelector &sel) override</div><div class="ttdoc">remove ids adapted to IndexFlat </div><div class="ttdef"><b>Definition:</b> <a href="MetaIndexes_8cpp_source.html#l00106">MetaIndexes.cpp:106</a></div></div>
|
|
<div class="ttc" id="structfaiss_1_1Index_html_a8e18f641854b2bde83ecff0a2f9a6f4e"><div class="ttname"><a href="structfaiss_1_1Index.html#a8e18f641854b2bde83ecff0a2f9a6f4e">faiss::Index::metric_type</a></div><div class="ttdeci">MetricType metric_type</div><div class="ttdoc">type of metric this index uses for search </div><div class="ttdef"><b>Definition:</b> <a href="Index_8h_source.html#l00072">Index.h:72</a></div></div>
|
|
<div class="ttc" id="structfaiss_1_1IndexIDMap2_html_a3660adb35a3299bc328988f5ff2199fe"><div class="ttname"><a href="structfaiss_1_1IndexIDMap2.html#a3660adb35a3299bc328988f5ff2199fe">faiss::IndexIDMap2::reconstruct</a></div><div class="ttdeci">void reconstruct(idx_t key, float *recons) const override</div><div class="ttdef"><b>Definition:</b> <a href="MetaIndexes_8cpp_source.html#l00168">MetaIndexes.cpp:168</a></div></div>
|
|
<div class="ttc" id="structfaiss_1_1IndexIDMap_html_ac873cb03aa323a06dca5d6f3af5fe559"><div class="ttname"><a href="structfaiss_1_1IndexIDMap.html#ac873cb03aa323a06dca5d6f3af5fe559">faiss::IndexIDMap::train</a></div><div class="ttdeci">void train(idx_t n, const float *x) override</div><div class="ttdef"><b>Definition:</b> <a href="MetaIndexes_8cpp_source.html#l00047">MetaIndexes.cpp:47</a></div></div>
|
|
<div class="ttc" id="structfaiss_1_1Index_html"><div class="ttname"><a href="structfaiss_1_1Index.html">faiss::Index</a></div><div class="ttdef"><b>Definition:</b> <a href="Index_8h_source.html#l00060">Index.h:60</a></div></div>
|
|
<div class="ttc" id="structfaiss_1_1RangeSearchResult_html_aef57f9db99ca470c01bcdce0366cb6ff"><div class="ttname"><a href="structfaiss_1_1RangeSearchResult.html#aef57f9db99ca470c01bcdce0366cb6ff">faiss::RangeSearchResult::lims</a></div><div class="ttdeci">size_t * lims</div><div class="ttdoc">size (nq + 1) </div><div class="ttdef"><b>Definition:</b> <a href="AuxIndexStructures_8h_source.html#l00031">AuxIndexStructures.h:31</a></div></div>
|
|
<div class="ttc" id="structfaiss_1_1Index_html_a236897832a58dbb205e2d662c1929fb1"><div class="ttname"><a href="structfaiss_1_1Index.html#a236897832a58dbb205e2d662c1929fb1">faiss::Index::range_search</a></div><div class="ttdeci">virtual void range_search(idx_t n, const float *x, float radius, RangeSearchResult *result) const </div><div class="ttdef"><b>Definition:</b> <a href="Index_8cpp_source.html#l00017">Index.cpp:17</a></div></div>
|
|
<div class="ttc" id="structfaiss_1_1Index_html_a6e92732617c4dbe364e7678dd8773a7f"><div class="ttname"><a href="structfaiss_1_1Index.html#a6e92732617c4dbe364e7678dd8773a7f">faiss::Index::is_trained</a></div><div class="ttdeci">bool is_trained</div><div class="ttdoc">set if the Index does not require training, or if training is done already </div><div class="ttdef"><b>Definition:</b> <a href="Index_8h_source.html#l00069">Index.h:69</a></div></div>
|
|
<div class="ttc" id="structfaiss_1_1Index_html_aaef6f4132b1f390121ebb11058baf190"><div class="ttname"><a href="structfaiss_1_1Index.html#aaef6f4132b1f390121ebb11058baf190">faiss::Index::reconstruct</a></div><div class="ttdeci">virtual void reconstruct(idx_t key, float *recons) const </div><div class="ttdef"><b>Definition:</b> <a href="Index_8cpp_source.html#l00043">Index.cpp:43</a></div></div>
|
|
<div class="ttc" id="structfaiss_1_1IndexIDMap_html_aeace75e53648c052d05c357b744e090b"><div class="ttname"><a href="structfaiss_1_1IndexIDMap.html#aeace75e53648c052d05c357b744e090b">faiss::IndexIDMap::add_with_ids</a></div><div class="ttdeci">void add_with_ids(idx_t n, const float *x, const long *xids) override</div><div class="ttdef"><b>Definition:</b> <a href="MetaIndexes_8cpp_source.html#l00060">MetaIndexes.cpp:60</a></div></div>
|
|
<div class="ttc" id="structfaiss_1_1RangeSearchResult_html_aa8ff848474a4cc16c5464cc4ec187824"><div class="ttname"><a href="structfaiss_1_1RangeSearchResult.html#aa8ff848474a4cc16c5464cc4ec187824">faiss::RangeSearchResult::labels</a></div><div class="ttdeci">idx_t * labels</div><div class="ttdoc">result for query i is labels[lims[i]:lims[i+1]] </div><div class="ttdef"><b>Definition:</b> <a href="AuxIndexStructures_8h_source.html#l00035">AuxIndexStructures.h:35</a></div></div>
|
|
<div class="ttc" id="structfaiss_1_1IndexSplitVectors_html_a70ba3367a08722fe4466dc49c21a4d2b"><div class="ttname"><a href="structfaiss_1_1IndexSplitVectors.html#a70ba3367a08722fe4466dc49c21a4d2b">faiss::IndexSplitVectors::IndexSplitVectors</a></div><div class="ttdeci">IndexSplitVectors(idx_t d, bool threaded=false)</div><div class="ttdoc">sum of dimensions seen so far </div><div class="ttdef"><b>Definition:</b> <a href="MetaIndexes_8cpp_source.html#l00592">MetaIndexes.cpp:592</a></div></div>
|
|
<div class="ttc" id="structfaiss_1_1IndexIDMap_html_affa8a764304410c96489c2b2e178bcdb"><div class="ttname"><a href="structfaiss_1_1IndexIDMap.html#affa8a764304410c96489c2b2e178bcdb">faiss::IndexIDMap::own_fields</a></div><div class="ttdeci">bool own_fields</div><div class="ttdoc">! the sub-index </div><div class="ttdef"><b>Definition:</b> <a href="MetaIndexes_8h_source.html#l00028">MetaIndexes.h:28</a></div></div>
|
|
</div><!-- fragment --></div><!-- contents -->
|
|
<!-- start footer part -->
|
|
<hr class="footer"/><address class="footer"><small>
|
|
Generated by  <a href="http://www.doxygen.org/index.html">
|
|
<img class="footer" src="doxygen.png" alt="doxygen"/>
|
|
</a> 1.8.5
|
|
</small></address>
|
|
</body>
|
|
</html>
|