13 #include "Clustering.h"
22 #include "FaissAssert.h"
23 #include "IndexFlat.h"
30 verbose(false), spherical(false),
32 min_points_per_centroid(39),
33 max_points_per_centroid(256),
47 static double imbalance_factor (
int n,
int k,
long *assign) {
48 std::vector<int> hist(k, 0);
49 for (
int i = 0; i < n; i++)
52 double tot = 0, uf = 0;
54 for (
int i = 0 ; i < k ; i++) {
56 uf += hist[i] * (double) hist[i];
58 uf = uf * k / (tot * tot);
67 FAISS_THROW_IF_NOT_MSG (nx >= k,
68 "need at least as many training points as clusters");
74 for (
size_t i = 0; i < nx *
d; i++) {
75 FAISS_THROW_IF_NOT_MSG (finite (x_in[i]),
76 "input contains NaN's or Inf's");
79 const float *x = x_in;
84 printf(
"Sampling a subset of %ld / %ld for training\n",
85 k * max_points_per_centroid, nx);
86 std::vector<int> perm (nx);
87 rand_perm (perm.data (), nx,
seed);
89 float * x_new =
new float [nx *
d];
90 for (idx_t i = 0; i < nx; i++)
91 memcpy (x_new + i * d, x + perm[i] * d,
sizeof(x_new[0]) *
d);
96 "WARNING clustering %ld points to %ld centroids: "
97 "please provide at least %ld training points\n",
98 nx, k, idx_t(k) * min_points_per_centroid);
103 printf(
"Clustering %d points in %ldD to %ld clusters, "
104 "redo %d times, %d iterations\n",
108 idx_t * assign =
new idx_t[nx];
110 float * dis =
new float[nx];
113 float best_err = 1e50;
114 double t_search_tot = 0;
116 printf(
" Preprocessing in %.2f s\n",
121 for (
int redo = 0; redo <
nredo; redo++) {
123 std::vector<float> buf_centroids;
125 std::vector<float> &cur_centroids =
128 if (verbose && nredo > 1) {
129 printf(
"Outer iteration %d / %d\n", redo, nredo);
132 if (cur_centroids.size() == 0) {
134 cur_centroids.resize (d * k);
135 std::vector<int> perm (nx);
137 rand_perm (perm.data(), nx,
seed + 1 + redo * 15486557L);
138 #pragma omp parallel for
139 for (
int i = 0; i <
k ; i++)
140 memcpy (&cur_centroids[i * d], x + perm[i] * d,
143 FAISS_THROW_IF_NOT (cur_centroids.size() == d *
k);
144 FAISS_THROW_IF_NOT_MSG (nredo == 1,
145 "will redo with same initialization");
149 fvec_renorm_L2 (d, k, cur_centroids.data());
152 index.
train (k, cur_centroids.data());
154 FAISS_THROW_IF_NOT (index.
ntotal == 0);
155 index.
add (k, cur_centroids.data());
157 for (
int i = 0; i <
niter; i++) {
159 index.
search (nx, x, 1, dis, assign);
163 for (
int j = 0; j < nx; j++)
171 printf (
" Iteration %d (%.2f s, search %.2f s): "
172 "objective=%g imbalance=%.3f nsplit=%d \r",
175 err, imbalance_factor (nx, k, assign),
181 fvec_renorm_L2 (d, k, cur_centroids.data());
185 index.
train (k, cur_centroids.data());
187 assert (index.
ntotal == 0);
188 index.
add (k, cur_centroids.data());
190 if (verbose) printf(
"\n");
192 if (err < best_err) {
194 printf (
"Objective improved: keep new clusters\n");
209 clus.verbose = d * n * k > (1L << 30);
212 clus.
train (n, x, index);
213 memcpy(centroids, clus.
centroids.data(),
sizeof(*centroids) * d * k);
214 return clus.
obj.back();
int niter
clustering iterations
int km_update_centroids(const float *x, float *centroids, long *assign, size_t d, size_t k, size_t n)
int nredo
redo clustering this many times and keep best
ClusteringParameters()
sets reasonable defaults
virtual void reset()=0
removes all elements from the database.
Clustering(int d, int k)
the only mandatory parameters are k and d
int seed
seed for the random number generator
int min_points_per_centroid
otherwise you get a warning
virtual void add(idx_t n, const float *x)=0
float kmeans_clustering(size_t d, size_t n, size_t k, const float *x, float *centroids)
idx_t ntotal
total nb of indexed vectors
double getmillisecs()
ms elapsed since some arbitrary epoch
std::vector< float > centroids
centroids (k * d)
size_t d
dimension of the vectors
virtual void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const =0
bool update_index
update index after each iteration?
virtual void train(idx_t n, const float *x, faiss::Index &index)
Index is used during the assignment stage.
bool is_trained
set if the Index does not require training, or if training is done already
virtual void train(idx_t n, const float *x)
bool spherical
do we want normalized centroids?
int max_points_per_centroid
to limit size of dataset