Add conda packages metadata + tests. (#769)
+ Add conda packages metadata (now building Faiss using conda's toolchain); + add Dockerfile for building conda packages (for all CUDA versions); + add working Dockerfile building faiss on Centos7; + simplify GPU build; + avoid falling back to CPU-only version (python); + simplify TravisCI config; + update INSTALL.md; + add configure flag for specifying target architectures (--with-cuda-arch); + fix Makefile for gpu tests; + fix various Makefile issues; + remove stale file (gpu/utils/DeviceUtils.cpp).pull/772/head v1.5.1
Lucas Hosseini
GitHub
parent
6c1cb3cd5c
commit
7f5b22b0ff
18
.travis.yml
18
.travis.yml
|
|
@ -15,7 +15,9 @@ matrix:
|
|||
- liblapack-dev
|
||||
- python-numpy
|
||||
- python-dev
|
||||
# - swig
|
||||
# - swig3.0
|
||||
env:
|
||||
- PYTHON_CFLAGS="-I/usr/include/python2.7"
|
||||
- os: linux
|
||||
compiler: gcc
|
||||
addons:
|
||||
|
|
@ -25,7 +27,9 @@ matrix:
|
|||
- liblapack-dev
|
||||
- python-numpy
|
||||
- python-dev
|
||||
# - swig
|
||||
# - swig3.0
|
||||
env:
|
||||
- PYTHON_CFLAGS="-I/usr/include/python2.7"
|
||||
- os: linux
|
||||
compiler: gcc
|
||||
addons:
|
||||
|
|
@ -35,7 +39,9 @@ matrix:
|
|||
- liblapack-dev
|
||||
- python-numpy
|
||||
- python-dev
|
||||
# - swig
|
||||
# - swig3.0
|
||||
env:
|
||||
- PYTHON_CFLAGS="-I/usr/include/python2.7"
|
||||
- os: linux
|
||||
compiler: clang
|
||||
addons:
|
||||
|
|
@ -45,8 +51,9 @@ matrix:
|
|||
- liblapack-dev
|
||||
- python-numpy
|
||||
- python-dev
|
||||
# - swig
|
||||
# - swig3.0
|
||||
env:
|
||||
- PYTHON_CFLAGS="-I/usr/include/python2.7"
|
||||
# NOTE: Hack, c.f. https://github.com/travis-ci/travis-ci/issues/8613
|
||||
- LD_LIBRARY_PATH="/usr/local/clang/lib"
|
||||
- os: osx
|
||||
|
|
@ -69,8 +76,9 @@ install:
|
|||
- ./.travis/install.sh
|
||||
- aclocal
|
||||
- autoconf
|
||||
- ./configure
|
||||
- ./configure --without-cuda
|
||||
- make
|
||||
- make -C python
|
||||
|
||||
script:
|
||||
- make test
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ function installswig() {
|
|||
cd /tmp/ &&
|
||||
wget https://github.com/swig/swig/archive/rel-3.0.12.tar.gz &&
|
||||
tar zxf rel-3.0.12.tar.gz && cd swig-rel-3.0.12 &&
|
||||
./autogen.sh && ./configure --prefix "${HOME}"/swig/ 1>/dev/null &&
|
||||
./autogen.sh && ./configure --prefix "${HOME}" 1>/dev/null &&
|
||||
make >/dev/null &&
|
||||
make install >/dev/null
|
||||
}
|
||||
|
|
|
|||
54
Dockerfile
54
Dockerfile
|
|
@ -1,38 +1,30 @@
|
|||
FROM nvidia/cuda:8.0-devel-ubuntu16.04
|
||||
MAINTAINER Pierre Letessier <pletessier@ina.fr>
|
||||
FROM nvidia/cuda:8.0-devel-centos7
|
||||
|
||||
RUN apt-get update -y
|
||||
RUN apt-get install -y libopenblas-dev python-numpy python-dev swig git python-pip wget
|
||||
# Install MKL
|
||||
RUN yum-config-manager --add-repo https://yum.repos.intel.com/mkl/setup/intel-mkl.repo
|
||||
RUN rpm --import https://yum.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB
|
||||
RUN yum install -y intel-mkl-2019.3-062
|
||||
ENV LD_LIBRARY_PATH /opt/intel/mkl/lib/intel64:$LD_LIBRARY_PATH
|
||||
ENV LIBRARY_PATH /opt/intel/mkl/lib/intel64:$LIBRARY_PATH
|
||||
ENV LD_PRELOAD /usr/lib64/libgomp.so.1:/opt/intel/mkl/lib/intel64/libmkl_def.so:\
|
||||
/opt/intel/mkl/lib/intel64/libmkl_avx2.so:/opt/intel/mkl/lib/intel64/libmkl_core.so:\
|
||||
/opt/intel/mkl/lib/intel64/libmkl_intel_lp64.so:/opt/intel/mkl/lib/intel64/libmkl_gnu_thread.so
|
||||
|
||||
RUN pip install --upgrade pip
|
||||
RUN pip install matplotlib
|
||||
# Install necessary build tools
|
||||
RUN yum install -y gcc-c++ make swig3
|
||||
|
||||
# Install necesary headers/libs
|
||||
RUN yum install -y python-devel numpy
|
||||
|
||||
COPY . /opt/faiss
|
||||
|
||||
WORKDIR /opt/faiss
|
||||
|
||||
ENV BLASLDFLAGS /usr/lib/libopenblas.so.0
|
||||
|
||||
RUN mv example_makefiles/makefile.inc.Linux ./makefile.inc
|
||||
|
||||
RUN make tests/test_blas -j $(nproc) && \
|
||||
make -j $(nproc) && \
|
||||
make demos/demo_sift1M -j $(nproc) && \
|
||||
make py
|
||||
|
||||
RUN cd gpu && \
|
||||
make -j $(nproc) && \
|
||||
make test/demo_ivfpq_indexing_gpu && \
|
||||
make py
|
||||
|
||||
ENV PYTHONPATH $PYTHONPATH:/opt/faiss
|
||||
|
||||
# RUN ./tests/test_blas && \
|
||||
# tests/demo_ivfpq_indexing
|
||||
|
||||
|
||||
# RUN wget ftp://ftp.irisa.fr/local/texmex/corpus/sift.tar.gz && \
|
||||
# tar xf sift.tar.gz && \
|
||||
# mv sift sift1M
|
||||
|
||||
# RUN tests/demo_sift1M
|
||||
# --with-cuda=/usr/local/cuda-8.0
|
||||
RUN ./configure --without-cuda
|
||||
RUN make -j $(nproc)
|
||||
RUN make -C python
|
||||
RUN make test
|
||||
RUN make install
|
||||
RUN make -C demos demo_ivfpq_indexing && \
|
||||
LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH ./demos/demo_ivfpq_indexing
|
||||
|
|
|
|||
178
INSTALL.md
178
INSTALL.md
|
|
@ -8,39 +8,32 @@ INSTALL file for Faiss (Fair AI Similarity Search)
|
|||
Install via Conda
|
||||
-----------------
|
||||
|
||||
The easiest way to install FAISS is from anaconda. We regularly push stable releases to conda channel. FAISS conda package is built with conda gcc, depends on libgcc, mkl and numpy package shipped in conda in runtime.
|
||||
The easiest way to install FAISS is from Anaconda. We regularly push stable releases to the pytorch conda channel.
|
||||
|
||||
Currently we support faiss-cpu on both Linux and OSX platforms. We also provide faiss-gpu compiled with CUDA8.0/CUDA9.0/CUDA9.2 on Linux systems.
|
||||
Currently we support faiss-cpu both on Linux and OSX. We also provide faiss-gpu compiled with CUDA8/CUDA9/CUDA10 on Linux systems.
|
||||
|
||||
You can easily install it by
|
||||
|
||||
```
|
||||
# CPU version only
|
||||
conda install faiss-cpu -c pytorch
|
||||
# Make sure you have CUDA installed before installing faiss-gpu, otherwise it falls back to CPU version
|
||||
conda install faiss-gpu -c pytorch # [DEFAULT]For CUDA8.0
|
||||
conda install faiss-gpu cuda90 -c pytorch # For CUDA9.0
|
||||
conda install faiss-gpu cuda92 -c pytorch # For CUDA9.2
|
||||
# cuda90/cuda91 shown above is a feature, it doesn't install CUDA for you.
|
||||
|
||||
# GPU version
|
||||
conda install faiss-gpu cudatoolkit=8.0 -c pytorch # For CUDA8
|
||||
conda install faiss-gpu cudatoolkit=9.0 -c pytorch # For CUDA9
|
||||
conda install faiss-gpu cudatoolkit=10.0 -c pytorch # For CUDA10
|
||||
```
|
||||
|
||||
Compile from source
|
||||
-------------------
|
||||
|
||||
The Faiss compilation works in 3 steps, from easiest to most
|
||||
involved:
|
||||
The Faiss compilation works in 2 steps:
|
||||
|
||||
1. compile the C++ core and examples
|
||||
|
||||
2. compile the Python interface
|
||||
|
||||
3. compile GPU part
|
||||
|
||||
Steps 2 and 3 depend on 1, but they are otherwise independent.
|
||||
|
||||
Alternatively, all 3 steps above can be run by building a Docker image (see
|
||||
section "Docker instructions" below).
|
||||
|
||||
Steps 2 depends on 1.
|
||||
|
||||
It is also possible to build a pure C interface. This optional process is
|
||||
described separately (please see the [C interface installation file](c_api/INSTALL.md))
|
||||
|
|
@ -48,20 +41,48 @@ described separately (please see the [C interface installation file](c_api/INSTA
|
|||
General compilation instructions
|
||||
================================
|
||||
|
||||
TL;DR: `./configure && make (&& make install)` for the C++ library, and then `cd python; make && make install` for the python interface.
|
||||
|
||||
1. `./configure`
|
||||
|
||||
This generates the system-dependent configuration for the `Makefile`, stored in
|
||||
a file called `makefile.inc`.
|
||||
|
||||
A few useful options:
|
||||
- `./configure --without-cuda` in order to build the CPU part only.
|
||||
- `./configure --with-cuda=/path/to/cuda-10.1` in order to hint to the path of
|
||||
the cudatoolkit.
|
||||
- `./configure --with-cuda-arch="-gencode=arch=compute_75,code=sm_75 -gencode=arch=compute_72,code=sm_72"` for specifying which GPU architectures to build against.
|
||||
- `./configure --with-python=/path/to/python3.7` in order to build a python
|
||||
interface for a different python than the default one.
|
||||
|
||||
2. `make`
|
||||
|
||||
This builds the C++ library (the whole library if a suitable cuda toolkit was
|
||||
found, or the CPU part only otherwise).
|
||||
|
||||
3. `make install` (optional)
|
||||
|
||||
This installs the headers and libraries.
|
||||
|
||||
4. `make -C python` (or `make py`)
|
||||
|
||||
This builds the python interface.
|
||||
|
||||
5. `make -C python install`
|
||||
|
||||
This installs the python library.
|
||||
|
||||
|
||||
Faiss has been tested only on x86_64 machines on Linux and Mac OS.
|
||||
|
||||
Faiss is compiled via a Makefile. The system-dependent configuration
|
||||
of the Makefile is in an include file, makefile.inc. The variables in
|
||||
makefile.inc must be set by hand.
|
||||
|
||||
Faiss requires a C++ compiler that understands:
|
||||
- the Intel intrinsics for SSE instructions
|
||||
- the GCC intrinsic for the popcount instruction
|
||||
- basic OpenMP
|
||||
- the Intel intrinsics for SSE instructions,
|
||||
- the GCC intrinsic for the popcount instruction,
|
||||
- basic OpenMP.
|
||||
|
||||
There are a few models for makefile.inc in the example_makefiles/
|
||||
subdirectory. Copy the relevant one for your system and adjust to your
|
||||
needs. There are also indications for specific configurations in the
|
||||
There are a few examples for makefile.inc in the example_makefiles/
|
||||
subdirectory. There are also indications for specific configurations in the
|
||||
troubleshooting section of the wiki.
|
||||
|
||||
https://github.com/facebookresearch/faiss/wiki/Troubleshooting
|
||||
|
|
@ -70,14 +91,6 @@ Faiss comes as a .a archive, that can be linked with executables or
|
|||
dynamic libraries (useful for the Python wrapper).
|
||||
|
||||
|
||||
Step 1: Compiling the C++ Faiss
|
||||
===============================
|
||||
|
||||
TL;DR: `./configure && make && make install`
|
||||
|
||||
|
||||
The CPU version of Faiss is written in C++11.
|
||||
|
||||
BLAS/Lapack
|
||||
-----------
|
||||
|
||||
|
|
@ -103,17 +116,6 @@ and run
|
|||
|
||||
`./misc/test_blas`
|
||||
|
||||
Building faiss
|
||||
-------------
|
||||
|
||||
Once the proper BLAS flags are set, the library should compile
|
||||
smoothly by running
|
||||
|
||||
`make`
|
||||
|
||||
Then, in order to install the library and the headers, run
|
||||
|
||||
`make install`
|
||||
|
||||
Testing Faiss
|
||||
-------------
|
||||
|
|
@ -122,13 +124,18 @@ A basic usage example is in
|
|||
|
||||
`demos/demo_ivfpq_indexing`
|
||||
|
||||
it makes a small index, stores it and performs some searches. A normal
|
||||
which you can build by calling
|
||||
`make -C demos demo_ivfpq_indexing`
|
||||
|
||||
It makes a small index, stores it and performs some searches. A normal
|
||||
runtime is around 20s. With a fast machine and Intel MKL's BLAS it
|
||||
runs in 2.5s.
|
||||
|
||||
To run the whole test suite:
|
||||
|
||||
`make test`
|
||||
`make test` (for the CPU part)
|
||||
|
||||
`make test_gpu` (for the GPU part)
|
||||
|
||||
|
||||
A real-life benchmark
|
||||
|
|
@ -154,16 +161,12 @@ setting a different index_key to find the indexing structure that
|
|||
gives the best performance.
|
||||
|
||||
|
||||
Step 2: Compiling the Python interface
|
||||
The Python interface
|
||||
======================================
|
||||
|
||||
The Python interface is compiled with
|
||||
|
||||
`make py`
|
||||
|
||||
If you want to compile it for another python version than the default
|
||||
Python 2.7, in particular Python 3, the PYTHONCFLAGS must be adjusted in
|
||||
makefile.inc, see the examples.
|
||||
`make -C python` (or `make py`)
|
||||
|
||||
How it works
|
||||
------------
|
||||
|
|
@ -172,18 +175,7 @@ The Python interface is provided via SWIG (Simple Wrapper and
|
|||
Interface Generator) and an additional level of manual wrappers (in python/faiss.py).
|
||||
|
||||
SWIG generates two wrapper files: a Python file (`python/swigfaiss.py`) and a
|
||||
C++ file that must be compiled to a dynamic library (`python/_swigfaiss.so`). These
|
||||
files are included in the repository, so running swig is only required when
|
||||
the C++ headers of Faiss are changed.
|
||||
|
||||
The C++ compilation to the dynamic library requires to set:
|
||||
|
||||
- `SHAREDFLAGS`: system-specific flags to generate a dynamic library
|
||||
|
||||
- `PYTHONCFLAGS`: include flags for Python
|
||||
|
||||
See the example makefile.inc's on how to set the flags.
|
||||
|
||||
C++ file that must be compiled to a dynamic library (`python/_swigfaiss.so`).
|
||||
|
||||
Testing the Python wrapper
|
||||
--------------------------
|
||||
|
|
@ -227,23 +219,21 @@ operating points. You can play around with the types of indexes.
|
|||
Step 3: Compiling the GPU implementation
|
||||
========================================
|
||||
|
||||
There is a GPU-specific Makefile in the `gpu/` directory. It depends on
|
||||
the same ../makefile.inc for system-specific variables. You need
|
||||
libfaiss.a from Step 1 for this to work.
|
||||
|
||||
The GPU version is a superset of the CPU version. In addition it
|
||||
requires the cuda compiler and related libraries (Cublas)
|
||||
|
||||
See the example makefile on how to set the flags.
|
||||
|
||||
The nvcc-specific flags to pass to the compiler, based on your desired
|
||||
compute capability. Only compute capability 3.5+ is supported. For
|
||||
example, we enable by default:
|
||||
compute capability can be customized by providing the `--with-cuda-arch` to
|
||||
`./configure`. Only compute capability 3.5+ is supported. For example, we enable
|
||||
by default:
|
||||
|
||||
```
|
||||
-gencode arch=compute_35,code="compute_35"
|
||||
-gencode arch=compute_52,code="compute_52"
|
||||
-gencode arch=compute_60,code="compute_60"
|
||||
-gencode=arch=compute_35,code=compute_35
|
||||
-gencode=arch=compute_52,code=compute_52
|
||||
-gencode=arch=compute_60,code=compute_60
|
||||
-gencode=arch=compute_61,code=compute_61
|
||||
-gencode=arch=compute_70,code=compute_70
|
||||
-gencode=arch=compute_75,code=compute_75
|
||||
```
|
||||
|
||||
However, look at https://developer.nvidia.com/cuda-gpus to determine
|
||||
|
|
@ -256,10 +246,6 @@ nvcc, except some of them that are not recognized and that should be
|
|||
escaped by prefixing them with -Xcompiler. Also link flags that are
|
||||
prefixed with -Wl, should be passed with -Xlinker.
|
||||
|
||||
Then compile with
|
||||
|
||||
`cd gpu; make`
|
||||
|
||||
You may want to add `-j 10` to use 10 threads during compile.
|
||||
|
||||
Testing the GPU implementation
|
||||
|
|
@ -267,30 +253,12 @@ Testing the GPU implementation
|
|||
|
||||
Compile the example with
|
||||
|
||||
`cd gpu; make tests/demo_ivfpq_indexing_gpu`
|
||||
`make -C gpu/test demo_ivfpq_indexing_gpu`
|
||||
|
||||
This produce the GPU code equivalent to the CPU
|
||||
demo_ivfpq_indexing. It also shows how to translate indexed from/to
|
||||
the GPU.
|
||||
|
||||
Compiling the Python interface with GPU support
|
||||
-----------------------------------------------
|
||||
|
||||
Given step 2, adding support of the GPU from Python is quite
|
||||
straightforward. Run
|
||||
|
||||
`cd python; make _swigfaiss_gpu.so`
|
||||
|
||||
The import is the same for the GPU version and the CPU-only
|
||||
version.
|
||||
|
||||
`python -c "import faiss"`
|
||||
|
||||
Faiss tries to load the GPU version first, and in case of failure,
|
||||
loads the CPU-only version. To investigate more closely the cause of
|
||||
a failure, you can run:
|
||||
|
||||
`python -c "import _swigfaiss_gpu"`
|
||||
|
||||
Python example with GPU support
|
||||
-------------------------------
|
||||
|
|
@ -357,7 +325,7 @@ libfaiss.so (or libfaiss.dylib)
|
|||
the executable should be linked to one of these. If you use
|
||||
the static version (.a), add the LDFLAGS used in the Makefile.
|
||||
|
||||
For binary-only distributions, the include files should be under
|
||||
For binary-only distributions, the headers should be under
|
||||
a `faiss/` directory, so that they can be included as
|
||||
|
||||
```c++
|
||||
|
|
@ -371,12 +339,12 @@ Python
|
|||
To import Faiss in your own Python project, you need the files
|
||||
|
||||
```
|
||||
faiss.py
|
||||
swigfaiss.py / swigfaiss_gpu.py
|
||||
_swigfaiss.so / _swigfaiss_gpu.so
|
||||
__init__.py
|
||||
swigfaiss.py
|
||||
_swigfaiss.so
|
||||
```
|
||||
|
||||
to be visible in the PYTHONPATH or in the current directory.
|
||||
to be present in a `faiss/` directory visible in the PYTHONPATH or in the
|
||||
current directory.
|
||||
Then Faiss can be used in python with
|
||||
|
||||
```python
|
||||
|
|
|
|||
4
Index.h
4
Index.h
|
|
@ -18,8 +18,8 @@
|
|||
#include <sstream>
|
||||
|
||||
#define FAISS_VERSION_MAJOR 1
|
||||
#define FAISS_VERSION_MINOR 4
|
||||
#define FAISS_VERSION_PATCH 0
|
||||
#define FAISS_VERSION_MINOR 5
|
||||
#define FAISS_VERSION_PATCH 1
|
||||
|
||||
/**
|
||||
* @namespace faiss
|
||||
|
|
|
|||
73
Makefile
73
Makefile
|
|
@ -6,19 +6,35 @@
|
|||
|
||||
-include makefile.inc
|
||||
|
||||
SRC=$(wildcard *.cpp)
|
||||
OBJ=$(SRC:.cpp=.o)
|
||||
HEADERS = $(wildcard *.h)
|
||||
SRC = $(wildcard *.cpp)
|
||||
OBJ = $(SRC:.cpp=.o)
|
||||
INSTALLDIRS = $(DESTDIR)$(libdir) $(DESTDIR)$(includedir)/faiss
|
||||
|
||||
GPU_HEADERS = $(wildcard gpu/*.h gpu/impl/*.h gpu/utils/*.h)
|
||||
GPU_CPPSRC = $(wildcard gpu/*.cpp gpu/impl/*.cpp gpu/utils/*.cpp)
|
||||
GPU_CUSRC = $(wildcard gpu/*.cu gpu/impl/*.cu gpu/utils/*.cu \
|
||||
gpu/utils/nvidia/*.cu gpu/utils/blockselect/*.cu gpu/utils/warpselect/*.cu)
|
||||
GPU_SRC = $(GPU_CPPSRC) $(GPU_CUSRC)
|
||||
GPU_CPPOBJ = $(GPU_CPPSRC:.cpp=.o)
|
||||
GPU_CUOBJ = $(GPU_CUSRC:.cu=.o)
|
||||
GPU_OBJ = $(GPU_CPPOBJ) $(GPU_CUOBJ)
|
||||
GPU_INSTALLDIRS = $(DESTDIR)$(includedir)/faiss/gpu/{impl,utils}
|
||||
|
||||
ifneq ($(strip $(NVCC)),)
|
||||
OBJ += $(GPU_OBJ)
|
||||
INSTALLDIRS += $(GPU_INSTALLDIRS)
|
||||
HEADERS += $(GPU_HEADERS)
|
||||
endif
|
||||
|
||||
|
||||
############################
|
||||
# Building
|
||||
|
||||
default: libfaiss.a
|
||||
|
||||
all: libfaiss.a libfaiss.$(SHAREDEXT)
|
||||
|
||||
libfaiss.a: $(OBJ)
|
||||
ar r $@ $^
|
||||
$(AR) r $@ $^
|
||||
|
||||
libfaiss.$(SHAREDEXT): $(OBJ)
|
||||
$(CXX) $(SHAREDFLAGS) $(LDFLAGS) -o $@ $^ $(LIBS)
|
||||
|
|
@ -26,8 +42,11 @@ libfaiss.$(SHAREDEXT): $(OBJ)
|
|||
%.o: %.cpp
|
||||
$(CXX) $(CPPFLAGS) $(CXXFLAGS) $(CPUFLAGS) -c $< -o $@
|
||||
|
||||
%.o: %.cu
|
||||
$(NVCC) $(NVCCFLAGS) -g -O3 -c $< -o $@
|
||||
|
||||
clean:
|
||||
rm -f libfaiss.*
|
||||
rm -f libfaiss.{a,$(SHAREDEXT)}
|
||||
rm -f $(OBJ)
|
||||
|
||||
|
||||
|
|
@ -35,15 +54,14 @@ clean:
|
|||
# Installing
|
||||
|
||||
install: libfaiss.a libfaiss.$(SHAREDEXT) installdirs
|
||||
cp libfaiss.a libfaiss.$(SHAREDEXT) $(DESTDIR)$(libdir)
|
||||
cp *.h $(DESTDIR)$(includedir)/faiss/
|
||||
cp libfaiss.{a,$(SHAREDEXT)} $(DESTDIR)$(libdir)
|
||||
tar cf - $(HEADERS) | tar xf - -C $(DESTDIR)$(includedir)/faiss/
|
||||
|
||||
installdirs:
|
||||
$(MKDIR_P) $(DESTDIR)$(libdir) $(DESTDIR)$(includedir)/faiss
|
||||
$(MKDIR_P) $(INSTALLDIRS)
|
||||
|
||||
uninstall:
|
||||
rm $(DESTDIR)$(libdir)/libfaiss.a
|
||||
rm $(DESTDIR)$(libdir)/libfaiss.$(SHAREDEXT)
|
||||
rm -f $(DESTDIR)$(libdir)/libfaiss.{a,$(SHAREDEXT)}
|
||||
rm -rf $(DESTDIR)$(includedir)/faiss
|
||||
|
||||
|
||||
|
|
@ -52,41 +70,44 @@ uninstall:
|
|||
|
||||
-include depend
|
||||
|
||||
# The above makefile.dep is generated by the following target:
|
||||
depend:
|
||||
for i in $(SRC); do \
|
||||
$(CXXCPP) $(CPPFLAGS) -MM $$i; \
|
||||
depend: $(SRC) $(GPU_SRC)
|
||||
for i in $^; do \
|
||||
$(CXXCPP) $(CPPFLAGS) -x c++ -MM $$i; \
|
||||
done > depend
|
||||
|
||||
|
||||
#############################
|
||||
# Python
|
||||
|
||||
py: libfaiss.a
|
||||
$(MAKE) -C python
|
||||
|
||||
|
||||
#############################
|
||||
# Tests
|
||||
|
||||
test: libfaiss.a py
|
||||
make -C tests run
|
||||
$(MAKE) -C tests run
|
||||
PYTHONPATH=./python/build/`ls python/build | grep lib` \
|
||||
$(PYTHON) -m unittest discover tests/ -v
|
||||
|
||||
test_gpu: libfaiss.a
|
||||
$(MAKE) -C gpu/test run
|
||||
PYTHONPATH=./python/build/`ls python/build | grep lib` \
|
||||
$(PYTHON) -m unittest discover gpu/test/ -v
|
||||
|
||||
#############################
|
||||
# Demos
|
||||
|
||||
demos: libfaiss.a
|
||||
make -C demos
|
||||
$(MAKE) -C demos
|
||||
|
||||
|
||||
#############################
|
||||
# Misc
|
||||
|
||||
misc/test_blas: misc/test_blas.cpp
|
||||
$(CXX) $(CXXFLAG) $(LDFLAGS) -o $@ $^ $(LIBS)
|
||||
$(CXX) $(CPPFLAGS) $(CXXFLAGS) $(LDFLAGS) -o $@ $^ $(LIBS)
|
||||
|
||||
|
||||
#############################
|
||||
# Python
|
||||
|
||||
py:
|
||||
$(MAKE) -C python build
|
||||
|
||||
|
||||
.PHONY: all clean default demos install installdirs py test uninstall
|
||||
.PHONY: all clean demos install installdirs py test gpu_test uninstall
|
||||
|
|
|
|||
|
|
@ -1,28 +1,52 @@
|
|||
AC_DEFUN([FA_CHECK_CUDA], [
|
||||
|
||||
AC_ARG_WITH(cuda,
|
||||
[AS_HELP_STRING([--with-cuda=<prefix>], [prefix of the CUDA installation])])
|
||||
case $with_cuda in
|
||||
"") cuda_prefix=/usr/local/cuda ;;
|
||||
*) cuda_prefix="$with_cuda"
|
||||
esac
|
||||
[AS_HELP_STRING([--with-cuda=<prefix>], [prefix of the CUDA installation])])
|
||||
AC_ARG_WITH(cuda-arch,
|
||||
[AS_HELP_STRING([--with-cuda-arch=<gencodes>], [device specific -gencode flags])],
|
||||
[],
|
||||
[with_cuda_arch=default])
|
||||
|
||||
AC_CHECK_PROG(NVCC, "nvcc", "$cuda_prefix/bin/nvcc", "", "$cuda_prefix/bin")
|
||||
fa_nvcc_bin=$NVCC
|
||||
if test x$with_cuda != xno; then
|
||||
if test x$with_cuda != x; then
|
||||
cuda_prefix=$with_cuda
|
||||
AC_CHECK_PROG(NVCC, [nvcc], [$cuda_prefix/bin/nvcc], [], [$cuda_prefix/bin])
|
||||
NVCC_CPPFLAGS="-I$cuda_prefix/include"
|
||||
NVCC_LDFLAGS="-L$cuda_prefix/lib64"
|
||||
else
|
||||
AC_CHECK_PROGS(NVCC, [nvcc /usr/local/cuda/bin/nvcc], [])
|
||||
if test "x$NVCC" == "x/usr/local/cuda/bin/nvcc"; then
|
||||
cuda_prefix="/usr/local/cuda"
|
||||
NVCC_CPPFLAGS="-I$cuda_prefix/include"
|
||||
NVCC_LDFLAGS="-L$cuda_prefix/lib64"
|
||||
else
|
||||
cuda_prefix=""
|
||||
NVCC_CPPFLAGS=""
|
||||
NVCC_LDFLAGS=""
|
||||
fi
|
||||
fi
|
||||
|
||||
if test "x$NVCC" == x; then
|
||||
AC_MSG_ERROR([Couldn't find nvcc])
|
||||
fi
|
||||
|
||||
if test "x$with_cuda_arch" == xdefault; then
|
||||
with_cuda_arch="-gencode=arch=compute_35,code=compute_35 \\
|
||||
-gencode=arch=compute_52,code=compute_52 \\
|
||||
-gencode=arch=compute_60,code=compute_60 \\
|
||||
-gencode=arch=compute_61,code=compute_61 \\
|
||||
-gencode=arch=compute_70,code=compute_70 \\
|
||||
-gencode=arch=compute_75,code=compute_75"
|
||||
fi
|
||||
|
||||
if test x$fa_nvcc_bin != x; then
|
||||
fa_save_CPPFLAGS="$CPPFLAGS"
|
||||
fa_save_LDFLAGS="$LDFLAGS"
|
||||
fa_save_LIBS="$LIBS"
|
||||
|
||||
NVCC_CPPFLAGS="-I$cuda_prefix/include"
|
||||
NVCC_LDFLAGS="-L$cuda_prefix/lib64"
|
||||
|
||||
CPPFLAGS="$NVCC_CPPFLAGS $CPPFLAGS"
|
||||
LDFLAGS="$NVCC_LDFLAGS $LDFLAGS"
|
||||
|
||||
AC_CHECK_HEADER([cuda.h], [], AC_MSG_FAILURE([Couldn't find cuda.h]))
|
||||
AC_CHECK_LIB([cuda], [cuInit], [], AC_MSG_FAILURE([Couldn't find libcuda]))
|
||||
AC_CHECK_LIB([cublas], [cublasAlloc], [], AC_MSG_FAILURE([Couldn't find libcublas]))
|
||||
AC_CHECK_LIB([cudart], [cudaSetDevice], [], AC_MSG_FAILURE([Couldn't find libcudart]))
|
||||
|
||||
|
|
@ -32,8 +56,6 @@ if test x$fa_nvcc_bin != x; then
|
|||
CPPFLAGS="$fa_save_CPPFLAGS"
|
||||
LDFLAGS="$fa_save_LDFLAGS"
|
||||
LIBS="$fa_save_LIBS"
|
||||
else
|
||||
cuda_prefix=""
|
||||
fi
|
||||
|
||||
AC_SUBST(NVCC)
|
||||
|
|
@ -41,4 +63,5 @@ AC_SUBST(NVCC_CPPFLAGS)
|
|||
AC_SUBST(NVCC_LDFLAGS)
|
||||
AC_SUBST(NVCC_LIBS)
|
||||
AC_SUBST(CUDA_PREFIX, $cuda_prefix)
|
||||
AC_SUBST(CUDA_ARCH, $with_cuda_arch)
|
||||
])
|
||||
|
|
|
|||
|
|
@ -3,10 +3,9 @@ AC_DEFUN([FA_PROG_SWIG], [
|
|||
AC_ARG_WITH(swig,
|
||||
[AS_HELP_STRING([--with-swig=<bin>], [use SWIG binary <bin>])])
|
||||
case $with_swig in
|
||||
"") SWIG_BIN=swig ;;
|
||||
*) SWIG_BIN="$with_swig"
|
||||
"") AC_CHECK_PROG(SWIG, swig, swig);;
|
||||
*) SWIG="$with_swig"
|
||||
esac
|
||||
|
||||
AC_CHECK_PROG(SWIG, $SWIG_BIN, $SWIG_BIN)
|
||||
AC_SUBST(SWIG)
|
||||
])
|
||||
|
|
|
|||
|
|
@ -10,36 +10,12 @@ esac
|
|||
AC_CHECK_PROG(PYTHON, $PYTHON_BIN, $PYTHON_BIN)
|
||||
fa_python_bin=$PYTHON
|
||||
|
||||
AC_ARG_WITH(python-config,
|
||||
[AS_HELP_STRING([--with-python-config=<bin>], [use Python config binary <bin>])])
|
||||
case $with_python_config in
|
||||
"") PYTHON_CFG_BIN="${PYTHON_BIN}-config" ;;
|
||||
*) PYTHON_CFG_BIN="$with_python_config"
|
||||
esac
|
||||
AC_MSG_CHECKING([for Python C flags])
|
||||
fa_python_cflags=`$PYTHON -c "
|
||||
import sysconfig
|
||||
paths = [['-I' + sysconfig.get_path(p) for p in ['include', 'platinclude']]]
|
||||
print(' '.join(paths))"`
|
||||
AC_MSG_RESULT($fa_python_cflags)
|
||||
AC_SUBST(PYTHON_CFLAGS, "$PYTHON_CFLAGS $fa_python_cflags")
|
||||
|
||||
AC_CHECK_PROG(PYTHON_CFG, $PYTHON_CFG_BIN, $PYTHON_CFG_BIN)
|
||||
fa_python_cfg_bin=$PYTHON_CFG
|
||||
|
||||
if test x$fa_python_cfg_bin != x; then
|
||||
AC_MSG_CHECKING([for Python C flags])
|
||||
fa_python_cflags=`${PYTHON_CFG} --includes`
|
||||
if test x"$fa_python_cflags" == x; then
|
||||
AC_MSG_RESULT([not found])
|
||||
AC_MSG_WARN([You won't be able to build the python interface.])
|
||||
else
|
||||
AC_MSG_RESULT($fa_python_cflags)
|
||||
AC_SUBST(PYTHON_CFLAGS, $fa_python_cflags)
|
||||
fi
|
||||
|
||||
AC_MSG_CHECKING([for Python ld flags])
|
||||
fa_python_ldflags=`${PYTHON_CFG} --ldflags`
|
||||
if test x"$fa_python_ldflags" == x; then
|
||||
AC_MSG_RESULT([not found])
|
||||
else
|
||||
AC_MSG_RESULT($fa_python_ldflags)
|
||||
AC_SUBST(PYTHON_LDFLAGS, $fa_python_ldflags)
|
||||
fi
|
||||
else
|
||||
AC_MSG_WARN([You won't be able to build the python interface.])
|
||||
fi
|
||||
])
|
||||
])dnl FA_PYTHON
|
||||
|
|
|
|||
|
|
@ -0,0 +1,33 @@
|
|||
FROM soumith/conda-cuda:latest
|
||||
|
||||
COPY ./ faiss
|
||||
WORKDIR /faiss/conda
|
||||
|
||||
ENV FAISS_BUILD_VERSION 1.5.1
|
||||
ENV FAISS_BUILD_NUMBER 1
|
||||
RUN conda build faiss --no-anaconda-upload -c pytorch
|
||||
RUN CUDA_ROOT=/usr/local/cuda-8.0 \
|
||||
CUDA_ARCH="-gencode=arch=compute_35,code=compute_35 \
|
||||
-gencode=arch=compute_52,code=compute_52 \
|
||||
-gencode=arch=compute_60,code=compute_60 \
|
||||
-gencode=arch=compute_61,code=compute_61" \
|
||||
conda build faiss-gpu --variants '{ "cudatoolkit": "8.0" }' \
|
||||
--no-anaconda-upload -c pytorch --no-test
|
||||
RUN CUDA_ROOT=/usr/local/cuda-9.0 \
|
||||
CUDA_ARCH="-gencode=arch=compute_35,code=compute_35 \
|
||||
-gencode=arch=compute_52,code=compute_52 \
|
||||
-gencode=arch=compute_60,code=compute_60 \
|
||||
-gencode=arch=compute_61,code=compute_61 \
|
||||
-gencode=arch=compute_70,code=compute_70" \
|
||||
conda build faiss-gpu --variants '{ "cudatoolkit": "9.0" }' \
|
||||
--no-anaconda-upload -c pytorch --no-test
|
||||
RUN CUDA_ROOT=/usr/local/cuda-10.0 \
|
||||
CUDA_ARCH="-gencode=arch=compute_35,code=compute_35 \
|
||||
-gencode=arch=compute_52,code=compute_52 \
|
||||
-gencode=arch=compute_60,code=compute_60 \
|
||||
-gencode=arch=compute_61,code=compute_61 \
|
||||
-gencode=arch=compute_70,code=compute_70 \
|
||||
-gencode=arch=compute_72,code=compute_72 \
|
||||
-gencode=arch=compute_75,code=compute_75" \
|
||||
conda build faiss-gpu --variants '{ "cudatoolkit": "10.0" }' \
|
||||
--no-anaconda-upload -c pytorch --no-test
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
CONDA_BUILD_SYSROOT:
|
||||
- /opt/MacOSX10.9.sdk # [osx]
|
||||
python:
|
||||
- 2.7
|
||||
- 3.5
|
||||
- 3.6
|
||||
- 3.7
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
./configure --with-cuda=$CUDA_ROOT --with-cuda-arch="$CUDA_ARCH"
|
||||
make -j $CPU_COUNT
|
||||
cd python
|
||||
make
|
||||
$PYTHON setup.py install --single-version-externally-managed --record=record.txt
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
cxx_compiler_version:
|
||||
- 5.4
|
||||
cudatoolkit:
|
||||
- 8.0
|
||||
- 9.0
|
||||
- 9.2
|
||||
- 10.0
|
||||
# - 10.1 # cudatoolkit 10.1 packages are not yet available on Anaconda.
|
||||
pin_run_as_build:
|
||||
cudatoolkit:
|
||||
max_pin: x.x
|
||||
|
|
@ -0,0 +1,41 @@
|
|||
package:
|
||||
name: faiss-gpu
|
||||
version: "{{ FAISS_BUILD_VERSION }}"
|
||||
|
||||
source:
|
||||
git_url: ../../
|
||||
|
||||
requirements:
|
||||
build:
|
||||
- {{ compiler('cxx') }}
|
||||
- llvm-openmp # [osx]
|
||||
- setuptools
|
||||
- swig
|
||||
|
||||
host:
|
||||
- python {{ python }}
|
||||
- intel-openmp # [osx]
|
||||
- numpy 1.11.*
|
||||
- mkl >=2018
|
||||
- cudatoolkit {{ cudatoolkit }}
|
||||
|
||||
run:
|
||||
- python {{ python }}
|
||||
- intel-openmp # [osx]
|
||||
- numpy >=1.11
|
||||
- mkl >=2018
|
||||
- blas=*=mkl
|
||||
- {{ pin_compatible('cudatoolkit') }}
|
||||
|
||||
build:
|
||||
number: {{ FAISS_BUILD_NUMBER }}
|
||||
script_env:
|
||||
- CUDA_ROOT
|
||||
- CUDA_ARCH
|
||||
|
||||
about:
|
||||
home: https://github.com/facebookresearch/faiss
|
||||
license: BSD 3-Clause
|
||||
license_family: BSD
|
||||
license_file: LICENSE
|
||||
summary: A library for efficient similarity search and clustering of dense vectors.
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
import faiss
|
||||
import numpy as np
|
||||
|
||||
d = 128
|
||||
n = 100
|
||||
|
||||
rs = np.random.RandomState(1337)
|
||||
x = rs.rand(n, d).astype(np.float32)
|
||||
|
||||
index = faiss.IndexFlatL2(d)
|
||||
|
||||
res = faiss.StandardGpuResources()
|
||||
gpu_index = faiss.index_cpu_to_gpu(res, 0, index)
|
||||
gpu_index.add(x)
|
||||
|
||||
D, I = index.search(x, 10)
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
./configure --without-cuda
|
||||
make -j $CPU_COUNT
|
||||
cd python
|
||||
make
|
||||
$PYTHON setup.py install --single-version-externally-managed --record=record.txt
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
package:
|
||||
name: faiss-cpu
|
||||
version: "{{ FAISS_BUILD_VERSION }}"
|
||||
|
||||
source:
|
||||
git_url: ../../
|
||||
|
||||
requirements:
|
||||
build:
|
||||
- {{ compiler('cxx') }}
|
||||
- llvm-openmp # [osx]
|
||||
- setuptools
|
||||
- swig
|
||||
|
||||
host:
|
||||
- python {{ python }}
|
||||
- intel-openmp # [osx]
|
||||
- numpy 1.11.*
|
||||
- mkl >=2018
|
||||
|
||||
run:
|
||||
- python {{ python }}
|
||||
- intel-openmp # [osx]
|
||||
- numpy >=1.11
|
||||
- blas=*=mkl
|
||||
- mkl >=2018
|
||||
|
||||
build:
|
||||
number: {{ FAISS_BUILD_NUMBER }}
|
||||
|
||||
about:
|
||||
home: https://github.com/facebookresearch/faiss
|
||||
license: BSD 3-Clause
|
||||
license_family: BSD
|
||||
license_file: LICENSE
|
||||
summary: A library for efficient similarity search and clustering of dense vectors.
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
import faiss
|
||||
import numpy as np
|
||||
|
||||
d = 128
|
||||
# NOTE: BLAS kicks in only when n > distance_compute_blas_threshold = 20
|
||||
n = 100
|
||||
|
||||
rs = np.random.RandomState(1337)
|
||||
x = rs.rand(n, d).astype(np.float32)
|
||||
|
||||
index = faiss.IndexFlatL2(d)
|
||||
index.add(x)
|
||||
|
||||
D, I = index.search(x, 10)
|
||||
|
|
@ -645,6 +645,7 @@ build_cpu
|
|||
build
|
||||
OPENMP_CXXFLAGS
|
||||
LIBOBJS
|
||||
CUDA_ARCH
|
||||
CUDA_PREFIX
|
||||
NVCC_LIBS
|
||||
NVCC_LDFLAGS
|
||||
|
|
@ -655,9 +656,7 @@ CXXCPP
|
|||
NVCC
|
||||
SWIG
|
||||
NUMPY_INCLUDE
|
||||
PYTHON_LDFLAGS
|
||||
PYTHON_CFLAGS
|
||||
PYTHON_CFG
|
||||
PYTHON
|
||||
MKDIR_P
|
||||
SET_MAKE
|
||||
|
|
@ -692,7 +691,6 @@ infodir
|
|||
docdir
|
||||
oldincludedir
|
||||
includedir
|
||||
runstatedir
|
||||
localstatedir
|
||||
sharedstatedir
|
||||
sysconfdir
|
||||
|
|
@ -716,9 +714,9 @@ ac_subst_files=''
|
|||
ac_user_opts='
|
||||
enable_option_checking
|
||||
with_python
|
||||
with_python_config
|
||||
with_swig
|
||||
with_cuda
|
||||
with_cuda_arch
|
||||
enable_openmp
|
||||
with_blas
|
||||
with_lapack
|
||||
|
|
@ -774,7 +772,6 @@ datadir='${datarootdir}'
|
|||
sysconfdir='${prefix}/etc'
|
||||
sharedstatedir='${prefix}/com'
|
||||
localstatedir='${prefix}/var'
|
||||
runstatedir='${localstatedir}/run'
|
||||
includedir='${prefix}/include'
|
||||
oldincludedir='/usr/include'
|
||||
docdir='${datarootdir}/doc/${PACKAGE_TARNAME}'
|
||||
|
|
@ -1027,15 +1024,6 @@ do
|
|||
| -silent | --silent | --silen | --sile | --sil)
|
||||
silent=yes ;;
|
||||
|
||||
-runstatedir | --runstatedir | --runstatedi | --runstated \
|
||||
| --runstate | --runstat | --runsta | --runst | --runs \
|
||||
| --run | --ru | --r)
|
||||
ac_prev=runstatedir ;;
|
||||
-runstatedir=* | --runstatedir=* | --runstatedi=* | --runstated=* \
|
||||
| --runstate=* | --runstat=* | --runsta=* | --runst=* | --runs=* \
|
||||
| --run=* | --ru=* | --r=*)
|
||||
runstatedir=$ac_optarg ;;
|
||||
|
||||
-sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb)
|
||||
ac_prev=sbindir ;;
|
||||
-sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \
|
||||
|
|
@ -1173,7 +1161,7 @@ fi
|
|||
for ac_var in exec_prefix prefix bindir sbindir libexecdir datarootdir \
|
||||
datadir sysconfdir sharedstatedir localstatedir includedir \
|
||||
oldincludedir docdir infodir htmldir dvidir pdfdir psdir \
|
||||
libdir localedir mandir runstatedir
|
||||
libdir localedir mandir
|
||||
do
|
||||
eval ac_val=\$$ac_var
|
||||
# Remove trailing slashes.
|
||||
|
|
@ -1326,7 +1314,6 @@ Fine tuning of the installation directories:
|
|||
--sysconfdir=DIR read-only single-machine data [PREFIX/etc]
|
||||
--sharedstatedir=DIR modifiable architecture-independent data [PREFIX/com]
|
||||
--localstatedir=DIR modifiable single-machine data [PREFIX/var]
|
||||
--runstatedir=DIR modifiable per-process data [LOCALSTATEDIR/run]
|
||||
--libdir=DIR object code libraries [EPREFIX/lib]
|
||||
--includedir=DIR C header files [PREFIX/include]
|
||||
--oldincludedir=DIR C header files for non-gcc [/usr/include]
|
||||
|
|
@ -1367,10 +1354,10 @@ Optional Packages:
|
|||
--with-PACKAGE[=ARG] use PACKAGE [ARG=yes]
|
||||
--without-PACKAGE do not use PACKAGE (same as --with-PACKAGE=no)
|
||||
--with-python=<bin> use Python binary <bin>
|
||||
--with-python-config=<bin>
|
||||
use Python config binary <bin>
|
||||
--with-swig=<bin> use SWIG binary <bin>
|
||||
--with-cuda=<prefix> prefix of the CUDA installation
|
||||
--with-cuda-arch=<gencodes>
|
||||
device specific -gencode flags
|
||||
--with-blas=<lib> use BLAS library <lib>
|
||||
--with-lapack=<lib> use LAPACK library <lib>
|
||||
|
||||
|
|
@ -4143,88 +4130,17 @@ fi
|
|||
|
||||
fa_python_bin=$PYTHON
|
||||
|
||||
|
||||
# Check whether --with-python-config was given.
|
||||
if test "${with_python_config+set}" = set; then :
|
||||
withval=$with_python_config;
|
||||
fi
|
||||
|
||||
case $with_python_config in
|
||||
"") PYTHON_CFG_BIN="${PYTHON_BIN}-config" ;;
|
||||
*) PYTHON_CFG_BIN="$with_python_config"
|
||||
esac
|
||||
|
||||
# Extract the first word of "$PYTHON_CFG_BIN", so it can be a program name with args.
|
||||
set dummy $PYTHON_CFG_BIN; ac_word=$2
|
||||
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
|
||||
$as_echo_n "checking for $ac_word... " >&6; }
|
||||
if ${ac_cv_prog_PYTHON_CFG+:} false; then :
|
||||
$as_echo_n "(cached) " >&6
|
||||
else
|
||||
if test -n "$PYTHON_CFG"; then
|
||||
ac_cv_prog_PYTHON_CFG="$PYTHON_CFG" # Let the user override the test.
|
||||
else
|
||||
as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
|
||||
for as_dir in $PATH
|
||||
do
|
||||
IFS=$as_save_IFS
|
||||
test -z "$as_dir" && as_dir=.
|
||||
for ac_exec_ext in '' $ac_executable_extensions; do
|
||||
if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
|
||||
ac_cv_prog_PYTHON_CFG="$PYTHON_CFG_BIN"
|
||||
$as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
|
||||
break 2
|
||||
fi
|
||||
done
|
||||
done
|
||||
IFS=$as_save_IFS
|
||||
|
||||
fi
|
||||
fi
|
||||
PYTHON_CFG=$ac_cv_prog_PYTHON_CFG
|
||||
if test -n "$PYTHON_CFG"; then
|
||||
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $PYTHON_CFG" >&5
|
||||
$as_echo "$PYTHON_CFG" >&6; }
|
||||
else
|
||||
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
|
||||
$as_echo "no" >&6; }
|
||||
fi
|
||||
|
||||
|
||||
fa_python_cfg_bin=$PYTHON_CFG
|
||||
|
||||
if test x$fa_python_cfg_bin != x; then
|
||||
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for Python C flags" >&5
|
||||
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for Python C flags" >&5
|
||||
$as_echo_n "checking for Python C flags... " >&6; }
|
||||
fa_python_cflags=`${PYTHON_CFG} --includes`
|
||||
if test x"$fa_python_cflags" == x; then
|
||||
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: not found" >&5
|
||||
$as_echo "not found" >&6; }
|
||||
{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: You won't be able to build the python interface." >&5
|
||||
$as_echo "$as_me: WARNING: You won't be able to build the python interface." >&2;}
|
||||
else
|
||||
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $fa_python_cflags" >&5
|
||||
fa_python_cflags=`$PYTHON -c "
|
||||
import sysconfig
|
||||
paths = ['-I' + sysconfig.get_path(p) for p in ['include', 'platinclude']]
|
||||
print(' '.join(paths))"`
|
||||
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $fa_python_cflags" >&5
|
||||
$as_echo "$fa_python_cflags" >&6; }
|
||||
PYTHON_CFLAGS=$fa_python_cflags
|
||||
PYTHON_CFLAGS="$PYTHON_CFLAGS $fa_python_cflags"
|
||||
|
||||
fi
|
||||
|
||||
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for Python ld flags" >&5
|
||||
$as_echo_n "checking for Python ld flags... " >&6; }
|
||||
fa_python_ldflags=`${PYTHON_CFG} --ldflags`
|
||||
if test x"$fa_python_ldflags" == x; then
|
||||
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: not found" >&5
|
||||
$as_echo "not found" >&6; }
|
||||
else
|
||||
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $fa_python_ldflags" >&5
|
||||
$as_echo "$fa_python_ldflags" >&6; }
|
||||
PYTHON_LDFLAGS=$fa_python_ldflags
|
||||
|
||||
fi
|
||||
else
|
||||
{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: You won't be able to build the python interface." >&5
|
||||
$as_echo "$as_me: WARNING: You won't be able to build the python interface." >&2;}
|
||||
fi
|
||||
|
||||
|
||||
if test x$PYTHON != x; then
|
||||
|
|
@ -4266,12 +4182,8 @@ if test "${with_swig+set}" = set; then :
|
|||
fi
|
||||
|
||||
case $with_swig in
|
||||
"") SWIG_BIN=swig ;;
|
||||
*) SWIG_BIN="$with_swig"
|
||||
esac
|
||||
|
||||
# Extract the first word of "$SWIG_BIN", so it can be a program name with args.
|
||||
set dummy $SWIG_BIN; ac_word=$2
|
||||
"") # Extract the first word of "swig", so it can be a program name with args.
|
||||
set dummy swig; ac_word=$2
|
||||
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
|
||||
$as_echo_n "checking for $ac_word... " >&6; }
|
||||
if ${ac_cv_prog_SWIG+:} false; then :
|
||||
|
|
@ -4287,7 +4199,7 @@ do
|
|||
test -z "$as_dir" && as_dir=.
|
||||
for ac_exec_ext in '' $ac_executable_extensions; do
|
||||
if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
|
||||
ac_cv_prog_SWIG="$SWIG_BIN"
|
||||
ac_cv_prog_SWIG="swig"
|
||||
$as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
|
||||
break 2
|
||||
fi
|
||||
|
|
@ -4306,6 +4218,9 @@ else
|
|||
$as_echo "no" >&6; }
|
||||
fi
|
||||
|
||||
;;
|
||||
*) SWIG="$with_swig"
|
||||
esac
|
||||
|
||||
|
||||
|
||||
|
|
@ -4711,13 +4626,20 @@ if test "${with_cuda+set}" = set; then :
|
|||
withval=$with_cuda;
|
||||
fi
|
||||
|
||||
case $with_cuda in
|
||||
"") cuda_prefix=/usr/local/cuda ;;
|
||||
*) cuda_prefix="$with_cuda"
|
||||
esac
|
||||
|
||||
# Extract the first word of ""nvcc"", so it can be a program name with args.
|
||||
set dummy "nvcc"; ac_word=$2
|
||||
# Check whether --with-cuda-arch was given.
|
||||
if test "${with_cuda_arch+set}" = set; then :
|
||||
withval=$with_cuda_arch;
|
||||
else
|
||||
with_cuda_arch=default
|
||||
fi
|
||||
|
||||
|
||||
if test x$with_cuda != xno; then
|
||||
if test x$with_cuda != x; then
|
||||
cuda_prefix=$with_cuda
|
||||
# Extract the first word of "nvcc", so it can be a program name with args.
|
||||
set dummy nvcc; ac_word=$2
|
||||
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
|
||||
$as_echo_n "checking for $ac_word... " >&6; }
|
||||
if ${ac_cv_prog_NVCC+:} false; then :
|
||||
|
|
@ -4727,13 +4649,13 @@ else
|
|||
ac_cv_prog_NVCC="$NVCC" # Let the user override the test.
|
||||
else
|
||||
as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
|
||||
for as_dir in "$cuda_prefix/bin"
|
||||
for as_dir in $cuda_prefix/bin
|
||||
do
|
||||
IFS=$as_save_IFS
|
||||
test -z "$as_dir" && as_dir=.
|
||||
for ac_exec_ext in '' $ac_executable_extensions; do
|
||||
if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
|
||||
ac_cv_prog_NVCC=""$cuda_prefix/bin/nvcc""
|
||||
ac_cv_prog_NVCC="$cuda_prefix/bin/nvcc"
|
||||
$as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
|
||||
break 2
|
||||
fi
|
||||
|
|
@ -4741,7 +4663,6 @@ done
|
|||
done
|
||||
IFS=$as_save_IFS
|
||||
|
||||
test -z "$ac_cv_prog_NVCC" && ac_cv_prog_NVCC=""""
|
||||
fi
|
||||
fi
|
||||
NVCC=$ac_cv_prog_NVCC
|
||||
|
|
@ -4754,16 +4675,79 @@ $as_echo "no" >&6; }
|
|||
fi
|
||||
|
||||
|
||||
fa_nvcc_bin=$NVCC
|
||||
NVCC_CPPFLAGS="-I$cuda_prefix/include"
|
||||
NVCC_LDFLAGS="-L$cuda_prefix/lib64"
|
||||
else
|
||||
for ac_prog in nvcc /usr/local/cuda/bin/nvcc
|
||||
do
|
||||
# Extract the first word of "$ac_prog", so it can be a program name with args.
|
||||
set dummy $ac_prog; ac_word=$2
|
||||
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
|
||||
$as_echo_n "checking for $ac_word... " >&6; }
|
||||
if ${ac_cv_prog_NVCC+:} false; then :
|
||||
$as_echo_n "(cached) " >&6
|
||||
else
|
||||
if test -n "$NVCC"; then
|
||||
ac_cv_prog_NVCC="$NVCC" # Let the user override the test.
|
||||
else
|
||||
as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
|
||||
for as_dir in $PATH
|
||||
do
|
||||
IFS=$as_save_IFS
|
||||
test -z "$as_dir" && as_dir=.
|
||||
for ac_exec_ext in '' $ac_executable_extensions; do
|
||||
if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
|
||||
ac_cv_prog_NVCC="$ac_prog"
|
||||
$as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
|
||||
break 2
|
||||
fi
|
||||
done
|
||||
done
|
||||
IFS=$as_save_IFS
|
||||
|
||||
fi
|
||||
fi
|
||||
NVCC=$ac_cv_prog_NVCC
|
||||
if test -n "$NVCC"; then
|
||||
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $NVCC" >&5
|
||||
$as_echo "$NVCC" >&6; }
|
||||
else
|
||||
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
|
||||
$as_echo "no" >&6; }
|
||||
fi
|
||||
|
||||
|
||||
test -n "$NVCC" && break
|
||||
done
|
||||
|
||||
if test "x$NVCC" == "x/usr/local/cuda/bin/nvcc"; then
|
||||
cuda_prefix="/usr/local/cuda"
|
||||
NVCC_CPPFLAGS="-I$cuda_prefix/include"
|
||||
NVCC_LDFLAGS="-L$cuda_prefix/lib64"
|
||||
else
|
||||
cuda_prefix=""
|
||||
NVCC_CPPFLAGS=""
|
||||
NVCC_LDFLAGS=""
|
||||
fi
|
||||
fi
|
||||
|
||||
if test "x$NVCC" == x; then
|
||||
as_fn_error $? "Couldn't find nvcc" "$LINENO" 5
|
||||
fi
|
||||
|
||||
if test "x$with_cuda_arch" == xdefault; then
|
||||
with_cuda_arch="-gencode=arch=compute_35,code=compute_35 \\
|
||||
-gencode=arch=compute_52,code=compute_52 \\
|
||||
-gencode=arch=compute_60,code=compute_60 \\
|
||||
-gencode=arch=compute_61,code=compute_61 \\
|
||||
-gencode=arch=compute_70,code=compute_70 \\
|
||||
-gencode=arch=compute_75,code=compute_75"
|
||||
fi
|
||||
|
||||
if test x$fa_nvcc_bin != x; then
|
||||
fa_save_CPPFLAGS="$CPPFLAGS"
|
||||
fa_save_LDFLAGS="$LDFLAGS"
|
||||
fa_save_LIBS="$LIBS"
|
||||
|
||||
NVCC_CPPFLAGS="-I$cuda_prefix/include"
|
||||
NVCC_LDFLAGS="-L$cuda_prefix/lib64"
|
||||
|
||||
CPPFLAGS="$NVCC_CPPFLAGS $CPPFLAGS"
|
||||
LDFLAGS="$NVCC_LDFLAGS $LDFLAGS"
|
||||
|
||||
|
|
@ -4778,56 +4762,6 @@ See \`config.log' for more details" "$LINENO" 5; }
|
|||
fi
|
||||
|
||||
|
||||
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for cuInit in -lcuda" >&5
|
||||
$as_echo_n "checking for cuInit in -lcuda... " >&6; }
|
||||
if ${ac_cv_lib_cuda_cuInit+:} false; then :
|
||||
$as_echo_n "(cached) " >&6
|
||||
else
|
||||
ac_check_lib_save_LIBS=$LIBS
|
||||
LIBS="-lcuda $LIBS"
|
||||
cat confdefs.h - <<_ACEOF >conftest.$ac_ext
|
||||
/* end confdefs.h. */
|
||||
|
||||
/* Override any GCC internal prototype to avoid an error.
|
||||
Use char because int might match the return type of a GCC
|
||||
builtin and then its argument prototype would still apply. */
|
||||
#ifdef __cplusplus
|
||||
extern "C"
|
||||
#endif
|
||||
char cuInit ();
|
||||
int
|
||||
main ()
|
||||
{
|
||||
return cuInit ();
|
||||
;
|
||||
return 0;
|
||||
}
|
||||
_ACEOF
|
||||
if ac_fn_cxx_try_link "$LINENO"; then :
|
||||
ac_cv_lib_cuda_cuInit=yes
|
||||
else
|
||||
ac_cv_lib_cuda_cuInit=no
|
||||
fi
|
||||
rm -f core conftest.err conftest.$ac_objext \
|
||||
conftest$ac_exeext conftest.$ac_ext
|
||||
LIBS=$ac_check_lib_save_LIBS
|
||||
fi
|
||||
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_cuda_cuInit" >&5
|
||||
$as_echo "$ac_cv_lib_cuda_cuInit" >&6; }
|
||||
if test "x$ac_cv_lib_cuda_cuInit" = xyes; then :
|
||||
cat >>confdefs.h <<_ACEOF
|
||||
#define HAVE_LIBCUDA 1
|
||||
_ACEOF
|
||||
|
||||
LIBS="-lcuda $LIBS"
|
||||
|
||||
else
|
||||
{ { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
|
||||
$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
|
||||
as_fn_error $? "Couldn't find libcuda
|
||||
See \`config.log' for more details" "$LINENO" 5; }
|
||||
fi
|
||||
|
||||
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for cublasAlloc in -lcublas" >&5
|
||||
$as_echo_n "checking for cublasAlloc in -lcublas... " >&6; }
|
||||
if ${ac_cv_lib_cublas_cublasAlloc+:} false; then :
|
||||
|
|
@ -4935,8 +4869,6 @@ fi
|
|||
CPPFLAGS="$fa_save_CPPFLAGS"
|
||||
LDFLAGS="$fa_save_LDFLAGS"
|
||||
LIBS="$fa_save_LIBS"
|
||||
else
|
||||
cuda_prefix=""
|
||||
fi
|
||||
|
||||
|
||||
|
|
@ -4945,6 +4877,8 @@ fi
|
|||
|
||||
CUDA_PREFIX=$cuda_prefix
|
||||
|
||||
CUDA_ARCH=$with_cuda_arch
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
@ -5474,8 +5408,6 @@ main ()
|
|||
if (*(data + i) != *(data3 + i))
|
||||
return 14;
|
||||
close (fd);
|
||||
free (data);
|
||||
free (data3);
|
||||
return 0;
|
||||
}
|
||||
_ACEOF
|
||||
|
|
|
|||
|
|
@ -15,8 +15,8 @@ all: $(DEMOS)
|
|||
clean:
|
||||
rm -f $(DEMOS)
|
||||
|
||||
%: %.cpp ../libfaiss.a
|
||||
$(CXX) -o $@ $(CXXFLAGS) $^ $(LDFLAGS) $(LIBS)
|
||||
%: %.cpp
|
||||
$(CXX) $(CPPFLAGS) $(CXXFLAGS) $(CPUFLAGS) -o $@ $^ $(LDFLAGS) $(LIBS) -lfaiss
|
||||
|
||||
|
||||
.PHONY: all clean
|
||||
|
|
|
|||
100
gpu/Makefile
100
gpu/Makefile
|
|
@ -1,100 +0,0 @@
|
|||
# Copyright (c) 2015-present, Facebook, Inc.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under the BSD+Patents license found in the
|
||||
# LICENSE file in the root directory of this source tree.
|
||||
|
||||
-include ../makefile.inc
|
||||
|
||||
all: libgpufaiss.a libgpufaiss.$(SHAREDEXT)
|
||||
|
||||
CPPOBJ = GpuResources.o \
|
||||
IndexProxy.o \
|
||||
StandardGpuResources.o \
|
||||
GpuAutoTune.o \
|
||||
GpuClonerOptions.o \
|
||||
impl/RemapIndices.o \
|
||||
utils/DeviceMemory.o \
|
||||
utils/StackDeviceMemory.o \
|
||||
utils/DeviceUtils.o \
|
||||
utils/Timer.o \
|
||||
utils/MemorySpace.o \
|
||||
utils/WorkerThread.o
|
||||
|
||||
|
||||
INS = 1 32 64 128 256 F512 T512 F1024 T1024
|
||||
|
||||
CUOBJ = impl/BinaryDistance.o \
|
||||
impl/BinaryFlatIndex.o \
|
||||
impl/BroadcastSum.o \
|
||||
impl/Distance.o \
|
||||
impl/FlatIndex.o \
|
||||
impl/InvertedListAppend.o \
|
||||
impl/IVFBase.o \
|
||||
impl/IVFFlat.o \
|
||||
impl/IVFFlatScan.o \
|
||||
impl/IVFPQ.o \
|
||||
impl/IVFUtils.o \
|
||||
impl/IVFUtilsSelect1.o \
|
||||
impl/IVFUtilsSelect2.o \
|
||||
impl/L2Norm.o \
|
||||
impl/L2Select.o \
|
||||
impl/PQCodeDistances.o \
|
||||
impl/PQScanMultiPassNoPrecomputed.o \
|
||||
impl/PQScanMultiPassPrecomputed.o \
|
||||
impl/VectorResidual.o \
|
||||
GpuDistance.o \
|
||||
GpuIndex.o \
|
||||
GpuIndexBinaryFlat.o \
|
||||
GpuIndexFlat.o \
|
||||
GpuIndexIVF.o \
|
||||
GpuIndexIVFFlat.o \
|
||||
GpuIndexIVFPQ.o \
|
||||
utils/Float16.o \
|
||||
utils/MatrixMult.o \
|
||||
utils/BlockSelectFloat.o \
|
||||
utils/BlockSelectHalf.o \
|
||||
utils/WarpSelectFloat.o \
|
||||
utils/WarpSelectHalf.o \
|
||||
utils/nvidia/fp16_emu.o \
|
||||
$(foreach v,$(INS), \
|
||||
utils/blockselect/BlockSelectHalf$(v).o \
|
||||
utils/blockselect/BlockSelectFloat$(v).o \
|
||||
utils/warpselect/WarpSelectHalf$(v).o \
|
||||
utils/warpselect/WarpSelectFloat$(v).o \
|
||||
)
|
||||
|
||||
%.o: %.cpp
|
||||
$(CXX) $(CXXFLAGS) $(CPUFLAGS) -c $< -o $@ $(CUDACFLAGS)
|
||||
|
||||
%.o: %.cu
|
||||
$(NVCC) $(NVCCFLAGS) -g -O3 -c $< -o $@
|
||||
|
||||
libgpufaiss.a: $(CPPOBJ) $(CUOBJ)
|
||||
ar r $@ $^
|
||||
|
||||
libgpufaiss.$(SHAREDEXT): $(CPPOBJ) $(CUOBJ)
|
||||
$(CXX) $(SHAREDFLAGS) $(LDFLAGS) $(NVCCLDFLAGS) \
|
||||
-o libgpufaiss.$(SHAREDEXT) $^ $(LIBS) $(NVCCLIBS)
|
||||
|
||||
clean:
|
||||
rm -rf *.o impl/*.o utils/*.o libgpufaiss.a \
|
||||
libgpufaiss.$(SHAREDEXT) \
|
||||
|
||||
-include depend
|
||||
|
||||
depend:
|
||||
for i in $(patsubst %.o,%.cpp,$(CPPOBJ)) \
|
||||
$(patsubst %.o,%.cu,$(CUOBJ)); do \
|
||||
$(CXXCPP) $(CPPFLAGS) -x c++ -MM $$i; \
|
||||
done > depend
|
||||
|
||||
install: libgpufaiss.a libgpufaiss.$(SHAREDEXT) installdirs
|
||||
cp libgpufaiss.a libgpufaiss.$(SHAREDEXT) $(DESTDIR)$(libdir)
|
||||
cp *.h $(DESTDIR)$(includedir)/faiss/gpu
|
||||
cp --parents **/**.h $(DESTDIR)$(includedir)/faiss/gpu
|
||||
|
||||
installdirs:
|
||||
$(MKDIR_P) $(DESTDIR)$(libdir) $(DESTDIR)$(includedir)/faiss/gpu
|
||||
|
||||
.PHONY: all clean
|
||||
|
|
@ -6,13 +6,45 @@
|
|||
|
||||
-include ../../makefile.inc
|
||||
|
||||
%.o: %.cpp
|
||||
$(CXX) $(CUDACFLAGS) -o $@ -c $^
|
||||
TESTS_SRC = TestGpuIndexFlat.cpp TestGpuIndexIVFPQ.cpp \
|
||||
TestGpuIndexBinaryFlat.cpp TestGpuIndexIVFFlat.cpp TestGpuMemoryException.cpp
|
||||
CUDA_TESTS_SRC = TestGpuSelect.cu
|
||||
|
||||
demo_ivfpq_indexing_gpu: demo_ivfpq_indexing_gpu.o ../libgpufaiss.a ../../libfaiss.a
|
||||
$(CXX) $(LDFLAGS) $(NVCCLDFLAGS) -o $@ $^ $(LIBS) $(NVCCLIBS)
|
||||
TESTS_BIN = $(TESTS_SRC:.cpp=)
|
||||
CUDA_TESTS_BIN = $(CUDA_TESTS_SRC:.cu=)
|
||||
|
||||
|
||||
# test_gpu_index.py test_pytorch_faiss.py
|
||||
|
||||
run: $(TESTS_BIN) $(CUDA_TESTS_BIN)
|
||||
for t in $(TESTS_BIN) $(CUDA_TESTS_BIN); do ./$$t; done
|
||||
|
||||
%.o: %.cu gtest
|
||||
$(NVCC) $(NVCCFLAGS) -g -O3 -o $@ -c $< -Igtest/include
|
||||
|
||||
%.o: %.cpp gtest
|
||||
$(CXX) $(CPPFLAGS) $(CXXFLAGS) $(CPUFLAGS) -o $@ -c $< -Igtest/include
|
||||
|
||||
%: %.o TestUtils.o ../../libfaiss.a gtest/make/gtest.a
|
||||
$(CXX) -o $@ $^ $(LDFLAGS) $(LIBS)
|
||||
|
||||
demo_ivfpq_indexing_gpu: demo_ivfpq_indexing_gpu.o ../../libfaiss.a
|
||||
$(CXX) -o $@ $^ $(LDFLAGS) $(LIBS)
|
||||
|
||||
demo_ivfpq_indexing_gpu.o: demo_ivfpq_indexing_gpu.cpp
|
||||
$(CXX) $(CPPFLAGS) $(CXXFLAGS) $(CPUFLAGS) -o $@ -c $^
|
||||
|
||||
gtest/make/gtest.a: gtest
|
||||
$(MAKE) -C gtest/make CXX="$(CXX)" CXXFLAGS="$(CXXFLAGS)" gtest.a
|
||||
|
||||
gtest:
|
||||
curl -L https://github.com/google/googletest/archive/release-1.8.0.tar.gz | tar xz && \
|
||||
mv googletest-release-1.8.0/googletest gtest && \
|
||||
rm -rf googletest-release-1.8.0
|
||||
|
||||
clean:
|
||||
rm -f *.o demo_ivfpq_indexing_gpu
|
||||
rm -f *.o $(TESTS_BIN)
|
||||
rm -rf gtest
|
||||
rm -f demo_ivfpq_indexing_gpu
|
||||
|
||||
.PHONY: clean
|
||||
|
|
|
|||
|
|
@ -1,175 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) 2015-present, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under the BSD+Patents license found in the
|
||||
* LICENSE file in the root directory of this source tree.
|
||||
*/
|
||||
|
||||
|
||||
#include "DeviceUtils.h"
|
||||
#include "../../FaissAssert.h"
|
||||
#include <mutex>
|
||||
#include <unordered_map>
|
||||
|
||||
namespace faiss { namespace gpu {
|
||||
|
||||
int getCurrentDevice() {
|
||||
int dev = -1;
|
||||
CUDA_VERIFY(cudaGetDevice(&dev));
|
||||
FAISS_ASSERT(dev != -1);
|
||||
|
||||
return dev;
|
||||
}
|
||||
|
||||
void setCurrentDevice(int device) {
|
||||
CUDA_VERIFY(cudaSetDevice(device));
|
||||
}
|
||||
|
||||
int getNumDevices() {
|
||||
int numDev = -1;
|
||||
CUDA_VERIFY(cudaGetDeviceCount(&numDev));
|
||||
FAISS_ASSERT(numDev != -1);
|
||||
|
||||
return numDev;
|
||||
}
|
||||
|
||||
void synchronizeAllDevices() {
|
||||
for (int i = 0; i < getNumDevices(); ++i) {
|
||||
DeviceScope scope(i);
|
||||
|
||||
CUDA_VERIFY(cudaDeviceSynchronize());
|
||||
}
|
||||
}
|
||||
|
||||
const cudaDeviceProp& getDeviceProperties(int device) {
|
||||
static std::mutex mutex;
|
||||
static std::unordered_map<int, cudaDeviceProp> properties;
|
||||
|
||||
std::lock_guard<std::mutex> guard(mutex);
|
||||
|
||||
auto it = properties.find(device);
|
||||
if (it == properties.end()) {
|
||||
cudaDeviceProp prop;
|
||||
CUDA_VERIFY(cudaGetDeviceProperties(&prop, device));
|
||||
|
||||
properties[device] = prop;
|
||||
it = properties.find(device);
|
||||
}
|
||||
|
||||
return it->second;
|
||||
}
|
||||
|
||||
const cudaDeviceProp& getCurrentDeviceProperties() {
|
||||
return getDeviceProperties(getCurrentDevice());
|
||||
}
|
||||
|
||||
int getMaxThreads(int device) {
|
||||
return getDeviceProperties(device).maxThreadsPerBlock;
|
||||
}
|
||||
|
||||
int getMaxThreadsCurrentDevice() {
|
||||
return getMaxThreads(getCurrentDevice());
|
||||
}
|
||||
|
||||
size_t getMaxSharedMemPerBlock(int device) {
|
||||
return getDeviceProperties(device).sharedMemPerBlock;
|
||||
}
|
||||
|
||||
size_t getMaxSharedMemPerBlockCurrentDevice() {
|
||||
return getMaxSharedMemPerBlock(getCurrentDevice());
|
||||
}
|
||||
|
||||
int getDeviceForAddress(const void* p) {
|
||||
if (!p) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
cudaPointerAttributes att;
|
||||
cudaError_t err = cudaPointerGetAttributes(&att, p);
|
||||
FAISS_ASSERT(err == cudaSuccess ||
|
||||
err == cudaErrorInvalidValue);
|
||||
|
||||
if (err == cudaErrorInvalidValue) {
|
||||
// Make sure the current thread error status has been reset
|
||||
err = cudaGetLastError();
|
||||
FAISS_ASSERT(err == cudaErrorInvalidValue);
|
||||
return -1;
|
||||
} else if (att.memoryType == cudaMemoryTypeHost) {
|
||||
return -1;
|
||||
} else {
|
||||
return att.device;
|
||||
}
|
||||
}
|
||||
|
||||
bool getFullUnifiedMemSupport(int device) {
|
||||
const auto& prop = getDeviceProperties(device);
|
||||
return (prop.major >= 6);
|
||||
}
|
||||
|
||||
bool getFullUnifiedMemSupportCurrentDevice() {
|
||||
return getFullUnifiedMemSupport(getCurrentDevice());
|
||||
}
|
||||
|
||||
DeviceScope::DeviceScope(int device) {
|
||||
prevDevice_ = getCurrentDevice();
|
||||
|
||||
if (prevDevice_ != device) {
|
||||
setCurrentDevice(device);
|
||||
} else {
|
||||
prevDevice_ = -1;
|
||||
}
|
||||
}
|
||||
|
||||
DeviceScope::~DeviceScope() {
|
||||
if (prevDevice_ != -1) {
|
||||
setCurrentDevice(prevDevice_);
|
||||
}
|
||||
}
|
||||
|
||||
CublasHandleScope::CublasHandleScope() {
|
||||
auto blasStatus = cublasCreate(&blasHandle_);
|
||||
FAISS_ASSERT(blasStatus == CUBLAS_STATUS_SUCCESS);
|
||||
}
|
||||
|
||||
CublasHandleScope::~CublasHandleScope() {
|
||||
auto blasStatus = cublasDestroy(blasHandle_);
|
||||
FAISS_ASSERT(blasStatus == CUBLAS_STATUS_SUCCESS);
|
||||
}
|
||||
|
||||
CudaEvent::CudaEvent(cudaStream_t stream)
|
||||
: event_(0) {
|
||||
CUDA_VERIFY(cudaEventCreateWithFlags(&event_, cudaEventDisableTiming));
|
||||
CUDA_VERIFY(cudaEventRecord(event_, stream));
|
||||
}
|
||||
|
||||
CudaEvent::CudaEvent(CudaEvent&& event) noexcept
|
||||
: event_(std::move(event.event_)) {
|
||||
event.event_ = 0;
|
||||
}
|
||||
|
||||
CudaEvent::~CudaEvent() {
|
||||
if (event_) {
|
||||
CUDA_VERIFY(cudaEventDestroy(event_));
|
||||
}
|
||||
}
|
||||
|
||||
CudaEvent&
|
||||
CudaEvent::operator=(CudaEvent&& event) noexcept {
|
||||
event_ = std::move(event.event_);
|
||||
event.event_ = 0;
|
||||
|
||||
return *this;
|
||||
}
|
||||
|
||||
void
|
||||
CudaEvent::streamWaitOnEvent(cudaStream_t stream) {
|
||||
CUDA_VERIFY(cudaStreamWaitEvent(stream, event_, 0));
|
||||
}
|
||||
|
||||
void
|
||||
CudaEvent::cpuWaitOnEvent() {
|
||||
CUDA_VERIFY(cudaEventSynchronize(event_));
|
||||
}
|
||||
|
||||
} } // namespace
|
||||
|
|
@ -6,27 +6,20 @@
|
|||
|
||||
CXX = @CXX@
|
||||
CXXCPP = @CXXCPP@
|
||||
# TODO: Investigate the LAPACKE wrapper for LAPACK, which defines the correct
|
||||
# type for FORTRAN integers.
|
||||
CPPFLAGS = -DFINTEGER=int @CPPFLAGS@
|
||||
CXXFLAGS = -fPIC @OPENMP_CXXFLAGS@ @ARCH_CXXFLAGS@ -Wno-sign-compare @CXXFLAGS@
|
||||
CPPFLAGS = -DFINTEGER=int @CPPFLAGS@ @OPENMP_CXXFLAGS@ @NVCC_CPPFLAGS@
|
||||
CXXFLAGS = -fPIC @ARCH_CXXFLAGS@ -Wno-sign-compare @CXXFLAGS@
|
||||
CPUFLAGS = @ARCH_CPUFLAGS@
|
||||
LDFLAGS = @OPENMP_LDFLAGS@ @LDFLAGS@
|
||||
LIBS = @BLAS_LIBS@ @LAPACK_LIBS@ @LIBS@
|
||||
LDFLAGS = @OPENMP_CXXFLAGS@ @LDFLAGS@ @NVCC_LDFLAGS@
|
||||
LIBS = @BLAS_LIBS@ @LAPACK_LIBS@ @LIBS@ @NVCC_LIBS@
|
||||
PYTHONCFLAGS = @PYTHON_CFLAGS@ -I@NUMPY_INCLUDE@
|
||||
|
||||
NVCC = @NVCC@
|
||||
NVCCLDFLAGS = @NVCC_LDFLAGS@
|
||||
NVCCLIBS = @NVCC_LIBS@
|
||||
CUDAROOT = @CUDA_PREFIX@
|
||||
CUDACFLAGS = @NVCC_CPPFLAGS@
|
||||
NVCCFLAGS = -I $(CUDAROOT)/targets/x86_64-linux/include/ \
|
||||
CUDA_ROOT = @CUDA_PREFIX@
|
||||
CUDA_ARCH = @CUDA_ARCH@
|
||||
NVCCFLAGS = -I $(CUDA_ROOT)/targets/x86_64-linux/include/ \
|
||||
-Xcompiler -fPIC \
|
||||
-Xcudafe --diag_suppress=unrecognized_attribute \
|
||||
-gencode arch=compute_35,code="compute_35" \
|
||||
-gencode arch=compute_52,code="compute_52" \
|
||||
-gencode arch=compute_60,code="compute_60" \
|
||||
-gencode arch=compute_61,code="compute_61" \
|
||||
$(CUDA_ARCH) \
|
||||
-lineinfo \
|
||||
-ccbin $(CXX) -DFAISS_USE_FLOAT16
|
||||
|
||||
|
|
@ -43,6 +36,7 @@ endif
|
|||
MKDIR_P = @MKDIR_P@
|
||||
PYTHON = @PYTHON@
|
||||
SWIG = @SWIG@
|
||||
AR ?= ar
|
||||
|
||||
prefix ?= @prefix@
|
||||
exec_prefix ?= @exec_prefix@
|
||||
|
|
|
|||
|
|
@ -6,52 +6,33 @@
|
|||
|
||||
-include ../makefile.inc
|
||||
|
||||
HEADERS = $(wildcard ../*.h)
|
||||
ifneq ($(strip $(NVCC)),)
|
||||
SWIGFLAGS = -DGPU_WRAPPER
|
||||
endif
|
||||
|
||||
all: cpu build
|
||||
|
||||
#############################
|
||||
# CPU
|
||||
|
||||
cpu: _swigfaiss.so
|
||||
all: build
|
||||
|
||||
# Also silently generates swigfaiss.py.
|
||||
swigfaiss_wrap.cpp: swigfaiss.swig $(HEADERS)
|
||||
$(SWIG) -python -c++ -Doverride= -I../ -o $@ $<
|
||||
swigfaiss.cpp: swigfaiss.swig ../libfaiss.a
|
||||
$(SWIG) -python -c++ -Doverride= -I../ $(SWIGFLAGS) -o $@ $<
|
||||
|
||||
swigfaiss_wrap.o: swigfaiss_wrap.cpp
|
||||
$(CXX) $(CXXFLAGS) $(CPUFLAGS) $(PYTHONCFLAGS) -I../ -c $< -o $@
|
||||
swigfaiss.o: swigfaiss.cpp
|
||||
$(CXX) $(CPPFLAGS) $(CXXFLAGS) $(CPUFLAGS) $(PYTHONCFLAGS) \
|
||||
-I../ -c $< -o $@
|
||||
|
||||
# Extension is .so even on OSX.
|
||||
_swigfaiss.so: swigfaiss_wrap.o ../libfaiss.a
|
||||
$(CXX) $(SHAREDFLAGS) $(LDFLAGS) -o $@ $^ $(LIBS)
|
||||
_swigfaiss.so: swigfaiss.o ../libfaiss.a
|
||||
$(CXX) $(SHAREDFLAGS) $(LDFLAGS) -o $@ $^ $(LIBS)
|
||||
|
||||
|
||||
#############################
|
||||
# GPU
|
||||
|
||||
gpu: _swigfaiss_gpu.so
|
||||
|
||||
# Also silently generates swigfaiss.py.
|
||||
swigfaiss_gpu_wrap.cpp: swigfaiss.swig
|
||||
$(SWIG) -python -c++ -Doverride= -I../ -DGPU_WRAPPER -o $@ $<
|
||||
|
||||
swigfaiss_gpu_wrap.o: swigfaiss_gpu_wrap.cpp
|
||||
$(NVCC) $(NVCCFLAGS) $(PYTHONCFLAGS) -I../ -c $< -o $@
|
||||
|
||||
_swigfaiss_gpu.so: swigfaiss_gpu_wrap.o ../gpu/libgpufaiss.a ../libfaiss.a
|
||||
$(CXX) $(SHAREDFLAGS) $(NVCCLDFLAGS) $(LDFLAGS) -o $@ $^ $(NVCCLIBS) $(LIBS)
|
||||
|
||||
|
||||
build: cpu
|
||||
build: _swigfaiss.so faiss.py
|
||||
$(PYTHON) setup.py build
|
||||
|
||||
install: build
|
||||
$(PYTHON) setup.py install
|
||||
|
||||
clean:
|
||||
rm -f swigfaiss_wrap.cpp swigfaiss_gpu_wrap.cpp
|
||||
rm -f swigfaiss.py swigfaiss_gpu.py
|
||||
rm -f _swigfaiss.so _swigfaiss_gpu.so
|
||||
rm -f swigfaiss.{cpp,o,py}
|
||||
rm -f _swigfaiss.so
|
||||
rm -rf build/
|
||||
|
||||
.PHONY: all build clean cpu gpu install
|
||||
.PHONY: all build clean install
|
||||
|
|
|
|||
|
|
@ -16,16 +16,7 @@ import pdb
|
|||
|
||||
|
||||
# we import * so that the symbol X can be accessed as faiss.X
|
||||
|
||||
try:
|
||||
from swigfaiss_gpu import *
|
||||
except ImportError as e:
|
||||
|
||||
if 'No module named' not in e.args[0]:
|
||||
# swigfaiss_gpu is there but failed to load: Warn user about it.
|
||||
sys.stderr.write("Failed to load GPU Faiss: %s\n" % e.args[0])
|
||||
sys.stderr.write("Faiss falling back to CPU-only.\n")
|
||||
from swigfaiss import *
|
||||
from .swigfaiss import *
|
||||
|
||||
__version__ = "%d.%d.%d" % (FAISS_VERSION_MAJOR,
|
||||
FAISS_VERSION_MINOR,
|
||||
|
|
|
|||
|
|
@ -8,8 +8,7 @@ here = os.path.abspath(os.path.dirname(__file__))
|
|||
check_fpath = os.path.join("_swigfaiss.so")
|
||||
if not os.path.exists(check_fpath):
|
||||
print("Could not find {}".format(check_fpath))
|
||||
print("Have you run `make` and `make py` "
|
||||
"(and optionally `cd gpu && make && make py && cd ..`)?")
|
||||
print("Have you run `make` and `make -C python`?")
|
||||
|
||||
# make the faiss python package dir
|
||||
shutil.rmtree("faiss", ignore_errors=True)
|
||||
|
|
@ -17,27 +16,22 @@ os.mkdir("faiss")
|
|||
shutil.copyfile("faiss.py", "faiss/__init__.py")
|
||||
shutil.copyfile("swigfaiss.py", "faiss/swigfaiss.py")
|
||||
shutil.copyfile("_swigfaiss.so", "faiss/_swigfaiss.so")
|
||||
try:
|
||||
shutil.copyfile("_swigfaiss_gpu.so", "faiss/_swigfaiss_gpu.so")
|
||||
shutil.copyfile("swigfaiss_gpu.py", "faiss/swigfaiss_gpu.py")
|
||||
except:
|
||||
pass
|
||||
|
||||
long_description="""
|
||||
Faiss is a library for efficient similarity search and clustering of dense
|
||||
Faiss is a library for efficient similarity search and clustering of dense
|
||||
vectors. It contains algorithms that search in sets of vectors of any size,
|
||||
up to ones that possibly do not fit in RAM. It also contains supporting
|
||||
code for evaluation and parameter tuning. Faiss is written in C++ with
|
||||
complete wrappers for Python/numpy. Some of the most useful algorithms
|
||||
up to ones that possibly do not fit in RAM. It also contains supporting
|
||||
code for evaluation and parameter tuning. Faiss is written in C++ with
|
||||
complete wrappers for Python/numpy. Some of the most useful algorithms
|
||||
are implemented on the GPU. It is developed by Facebook AI Research.
|
||||
"""
|
||||
setup(
|
||||
name='faiss',
|
||||
version='0.1',
|
||||
version='1.5.1',
|
||||
description='A library for efficient similarity search and clustering of dense vectors',
|
||||
long_description=long_description,
|
||||
url='https://github.com/facebookresearch/faiss',
|
||||
author='Matthijs Douze, Jeff Johnson, Herve Jegou',
|
||||
author='Matthijs Douze, Jeff Johnson, Herve Jegou, Lucas Hosseini',
|
||||
author_email='matthijs@fb.com',
|
||||
license='BSD',
|
||||
keywords='search nearest neighbors',
|
||||
|
|
|
|||
6173
python/swigfaiss.py
6173
python/swigfaiss.py
File diff suppressed because it is too large
Load Diff
|
|
@ -15,11 +15,7 @@
|
|||
// SWIGPYTHON: Python-specific code
|
||||
// GPU_WRAPPER: also compile interfaces for GPU.
|
||||
|
||||
#ifdef GPU_WRAPPER
|
||||
%module swigfaiss_gpu;
|
||||
#else
|
||||
%module swigfaiss;
|
||||
#endif
|
||||
|
||||
// fbode SWIG fails on warnings, so make them non fatal
|
||||
#pragma SWIG nowarn=321
|
||||
|
|
@ -328,16 +324,6 @@ int get_num_gpus()
|
|||
%include "gpu/GpuIndexBinaryFlat.h"
|
||||
%include "gpu/GpuDistance.h"
|
||||
|
||||
#ifdef SWIGLUA
|
||||
|
||||
/// in Lua, swigfaiss_gpu is known as swigfaiss
|
||||
%luacode {
|
||||
local swigfaiss = swigfaiss_gpu
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
|
||||
#endif
|
||||
|
||||
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
|
|
@ -7,21 +7,22 @@
|
|||
-include ../makefile.inc
|
||||
|
||||
TESTS_SRC = $(wildcard *.cpp)
|
||||
TESTS = $(TESTS_SRC:.cpp=.o)
|
||||
TESTS_OBJ = $(TESTS_SRC:.cpp=.o)
|
||||
|
||||
|
||||
all: run
|
||||
|
||||
run: tests
|
||||
./tests
|
||||
|
||||
tests: $(TESTS) ../libfaiss.a gtest/make/gtest_main.a
|
||||
tests: $(TESTS_OBJ) ../libfaiss.a gtest/make/gtest_main.a
|
||||
$(CXX) -o $@ $^ $(LDFLAGS) $(LIBS)
|
||||
|
||||
%.o: %.cpp gtest
|
||||
$(CXX) $(CPPFLAGS) $(CXXFLAGS) $(CPUFLAGS) -c -o $@ $< -Igtest/include -I../..
|
||||
|
||||
gtest/make/gtest_main.a: gtest
|
||||
make -C gtest/make CXX="$(CXX)" CXXFLAGS="$(CXXFLAGS)" gtest_main.a
|
||||
$(MAKE) -C gtest/make CXX="$(CXX)" CXXFLAGS="$(CXXFLAGS)" gtest_main.a
|
||||
|
||||
gtest:
|
||||
curl -L https://github.com/google/googletest/archive/release-1.8.0.tar.gz | tar xz && \
|
||||
|
|
@ -29,8 +30,8 @@ gtest:
|
|||
rm -rf googletest-release-1.8.0
|
||||
|
||||
clean:
|
||||
rm -f test_runner
|
||||
rm -f *.o
|
||||
rm -f tests
|
||||
rm -f $(TESTS_OBJ)
|
||||
rm -rf gtest
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue