mirrors
/
faiss
mirror of https://github.com/facebookresearch/faiss.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Update conda packaging. (#1363) 

Summary: Pull Request resolved: https://github.com/facebookresearch/faiss/pull/1363

Test Plan: Imported from OSS

Reviewed By: mdouze

Differential Revision: D23314731

Pulled By: beauby

fbshipit-source-id: b57a66cf5c44788837ee379fbb7f73adbdbf1274
pull/1380/head
Lucas Hosseini 2020-08-25 16:56:01 -07:00 committed by Facebook GitHub Bot
parent 24d00ccc5b
commit c689bd010c
14 changed files with 316 additions and 134 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
conda/conda_build_config.yaml
View File

@ -2,6 +2,6 @@ CONDA_BUILD_SYSROOT:
- /opt/MacOSX10.9.sdk # [osx]
python:
- 2.7
- 3.5
- 3.6
- 3.7
- 3.8

38
conda/faiss-gpu/build-lib.sh 100755
View File

@ -0,0 +1,38 @@
#!/bin/sh
# Copyright (c) Facebook, Inc. and its affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
set -e
# Build libfaiss_avx2.so.
cmake -B _build_avx2 \
-DBUILD_SHARED_LIBS=ON \
-DBUILD_TESTING=OFF \
-DFAISS_OPT_LEVEL=avx2 \
-DFAISS_ENABLE_GPU=ON \
-DFAISS_ENABLE_PYTHON=OFF \
-DBLA_VENDOR=Intel10_64lp \
-DCMAKE_BUILD_TYPE=Release .
cmake --build _build_avx2 -j $CPU_COUNT
cmake --install _build_avx2 --prefix $PREFIX
cmake --install _build_avx2 --prefix _libfaiss_avx2_stage/
# Build libfaiss.so.
cmake -B _build_generic \
-DBUILD_SHARED_LIBS=ON \
-DBUILD_TESTING=OFF \
-DFAISS_ENABLE_GPU=ON \
-DFAISS_ENABLE_PYTHON=OFF \
-DBLA_VENDOR=Intel10_64lp \
-DCMAKE_BUILD_TYPE=Release .
cmake --build _build_generic -j $CPU_COUNT
cmake --install _build_generic --prefix $PREFIX
cmake --install _build_generic --prefix _libfaiss_stage/

36
conda/faiss-gpu/build-pkg.sh 100755
View File

@ -0,0 +1,36 @@
#!/bin/sh
# Copyright (c) Facebook, Inc. and its affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
set -e
# Build avx2 version.
cmake -B _build_python_${PY_VER}_avx2 \
-Dfaiss_ROOT=_libfaiss_avx2_stage/ \
-DFAISS_ENABLE_GPU=ON \
-DCMAKE_BUILD_TYPE=Release \
-DPython_EXECUTABLE=$PYTHON \
faiss/python
cmake --build _build_python_${PY_VER}_avx2 -j $CPU_COUNT
# Build vanilla version (no avx).
cmake -B _build_python_${PY_VER} \
-Dfaiss_ROOT=_libfaiss_stage/ \
-DFAISS_ENABLE_GPU=ON \
-DCMAKE_BUILD_TYPE=Release \
-DPython_EXECUTABLE=$PYTHON \
faiss/python
cmake --build _build_python_${PY_VER} -j $CPU_COUNT
# Build actual python module.
cp _build_python_${PY_VER}_avx2/swigfaiss.py _build_python_${PY_VER}/swigfaiss_avx2.py
cp _build_python_${PY_VER}_avx2/_swigfaiss.so _build_python_${PY_VER}/_swigfaiss_avx2.so
cd _build_python_${PY_VER}/
$PYTHON setup.py install --single-version-externally-managed --record=record.txt --prefix=$PREFIX

16
conda/faiss-gpu/build.sh
View File

@ -1,16 +0,0 @@
# Build avx2 version
CXXFLAGS="-mavx2 -mf16c" ./configure --with-cuda=$CUDA_ROOT --with-cuda-arch="$CUDA_ARCH"
make -j $CPU_COUNT
make -C python _swigfaiss_avx2.so
make clean
# Build vanilla version (no avx)
./configure --with-cuda=$CUDA_ROOT --with-cuda-arch="$CUDA_ARCH"
make -j $CPU_COUNT
make -C python _swigfaiss.so
make -C python build
cd python
$PYTHON setup.py install --single-version-externally-managed --record=record.txt

11
conda/faiss-gpu/conda_build_config.yaml
View File

@ -1,11 +0,0 @@
cxx_compiler_version:
- 5.4
cudatoolkit:
- 8.0
- 9.0
- 9.2
- 10.0
- 10.1
pin_run_as_build:
cudatoolkit:
max_pin: x.x

10
conda/faiss-gpu/install-cmake.sh 100755
View File

@ -0,0 +1,10 @@
#!/bin/sh
# Copyright (c) Facebook, Inc. and its affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
set -e
wget -O - https://github.com/Kitware/CMake/releases/download/v3.17.1/cmake-3.17.1-Linux-x86_64.tar.gz | tar xzf -
cp -R cmake-3.17.1-Linux-x86_64/* $PREFIX

106
conda/faiss-gpu/meta.yaml
View File

@ -1,37 +1,17 @@
# Copyright (c) Facebook, Inc. and its affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
{% set version = environ.get('GIT_DESCRIBE_TAG') %}
{% set number = environ.get('GIT_DESCRIBE_NUMBER') %}
package:
name: faiss-gpu
version: "{{ FAISS_BUILD_VERSION }}"
source:
git_url: ../../
requirements:
build:
- {{ compiler('cxx') }}
- llvm-openmp # [osx]
- setuptools
- swig
host:
- python {{ python }}
- intel-openmp # [osx]
- numpy 1.11.*
- mkl >=2018
- cudatoolkit {{ cudatoolkit }}
run:
- python {{ python }}
- intel-openmp # [osx]
- numpy >=1.11
- mkl >=2018
- blas=*=mkl
- {{ pin_compatible('cudatoolkit') }}
name: faiss-pkg
version: {{ version }}
build:
number: {{ FAISS_BUILD_NUMBER }}
script_env:
- CUDA_ROOT
- CUDA_ARCH
number: {{ number }}
about:
home: https://github.com/facebookresearch/faiss
@ -39,3 +19,67 @@ about:
license_family: MIT
license_file: LICENSE
summary: A library for efficient similarity search and clustering of dense vectors.
source:
git_url: ../../
outputs:
- name: libfaiss
script: build-lib.sh
build:
string: "h{{ PKG_HASH }}_{{ number }}_cuda{{ cudatoolkit }}"
run_exports:
- {{ pin_compatible('libfaiss', exact=True) }}
requirements:
build:
- {{ compiler('cxx') }}
- llvm-openmp # [osx]
- cmake-local
host:
- mkl >=2018
- blas =*=mkl
- cudatoolkit {{ cudatoolkit }}
run:
- mkl >=2018
- blas =*=mkl
- {{ pin_compatible('cudatoolkit') }}
test:
commands:
- test -f $PREFIX/lib/libfaiss.so # [linux]
- test -f $PREFIX/lib/libfaiss.dylib # [osx]
- conda inspect linkages -p $PREFIX $PKG_NAME # [not win]
- conda inspect objects -p $PREFIX $PKG_NAME # [osx]
- name: faiss-gpu
script: build-pkg.sh
build:
string: "py{{ PY_VER }}_h{{ PKG_HASH }}_{{ number }}_cuda{{ cudatoolkit }}"
requirements:
build:
- {{ compiler('cxx') }}
- swig
- cmake-local
host:
- python {{ python }}
- numpy 1.11.*
- {{ pin_subpackage('libfaiss', exact=True) }}
run:
- python {{ python }}
- {{ pin_compatible('numpy') }}
- {{ pin_subpackage('libfaiss', exact=True) }}
test:
requires:
- numpy
- scipy
commands:
- python -m unittest discover tests/
- python -m unittest discover faiss/gpu/test/
source_files:
- tests/
- name: cmake-local
version: 3.17
script: install-cmake.sh
requirements:
build:
- {{ compiler('cxx') }}

16
conda/faiss-gpu/run_test.py
View File

@ -1,16 +0,0 @@
import faiss
import numpy as np
d = 128
n = 100
rs = np.random.RandomState(1337)
x = rs.rand(n, d).astype(np.float32)
index = faiss.IndexFlatL2(d)
res = faiss.StandardGpuResources()
gpu_index = faiss.index_cpu_to_gpu(res, 0, index)
gpu_index.add(x)
D, I = index.search(x, 10)

38
conda/faiss/build-lib.sh 100755
View File

@ -0,0 +1,38 @@
#!/bin/sh
# Copyright (c) Facebook, Inc. and its affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
set -e
# Build libfaiss_avx2.so.
cmake -B _build_avx2 \
-DBUILD_SHARED_LIBS=ON \
-DBUILD_TESTING=OFF \
-DFAISS_OPT_LEVEL=avx2 \
-DFAISS_ENABLE_GPU=OFF \
-DFAISS_ENABLE_PYTHON=OFF \
-DBLA_VENDOR=Intel10_64lp \
-DCMAKE_BUILD_TYPE=Release .
cmake --build _build_avx2 -j $CPU_COUNT
cmake --install _build_avx2 --prefix $PREFIX
cmake --install _build_avx2 --prefix _libfaiss_avx2_stage/
# Build libfaiss.so.
cmake -B _build_generic \
-DBUILD_SHARED_LIBS=ON \
-DBUILD_TESTING=OFF \
-DFAISS_ENABLE_GPU=OFF \
-DFAISS_ENABLE_PYTHON=OFF \
-DBLA_VENDOR=Intel10_64lp \
-DCMAKE_BUILD_TYPE=Release .
cmake --build _build_generic -j $CPU_COUNT
cmake --install _build_generic --prefix $PREFIX
cmake --install _build_generic --prefix _libfaiss_stage/

36
conda/faiss/build-pkg.sh 100755
View File

@ -0,0 +1,36 @@
#!/bin/sh
# Copyright (c) Facebook, Inc. and its affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
set -e
# Build avx2 version.
cmake -B _build_python_${PY_VER}_avx2 \
-Dfaiss_ROOT=_libfaiss_avx2_stage/ \
-DFAISS_ENABLE_GPU=OFF \
-DCMAKE_BUILD_TYPE=Release \
-DPython_EXECUTABLE=$PYTHON \
faiss/python
cmake --build _build_python_${PY_VER}_avx2 -j $CPU_COUNT
# Build vanilla version (no avx).
cmake -B _build_python_${PY_VER} \
-Dfaiss_ROOT=_libfaiss_stage/ \
-DFAISS_ENABLE_GPU=OFF \
-DCMAKE_BUILD_TYPE=Release \
-DPython_EXECUTABLE=$PYTHON \
faiss/python
cmake --build _build_python_${PY_VER} -j $CPU_COUNT
# Build actual python module.
cp _build_python_${PY_VER}_avx2/swigfaiss.py _build_python_${PY_VER}/swigfaiss_avx2.py
cp _build_python_${PY_VER}_avx2/_swigfaiss.so _build_python_${PY_VER}/_swigfaiss_avx2.so
cd _build_python_${PY_VER}/
$PYTHON setup.py install --single-version-externally-managed --record=record.txt --prefix=$PREFIX

16
conda/faiss/build.sh
View File

@ -1,16 +0,0 @@
# Build avx2 version
CXXFLAGS="-mavx2 -mf16c" ./configure --without-cuda
make -j $CPU_COUNT
make -C python _swigfaiss_avx2.so
make clean
# Build vanilla version (no avx)
./configure --without-cuda
make -j $CPU_COUNT
make -C python _swigfaiss.so
make -C python build
cd python
$PYTHON setup.py install --single-version-externally-managed --record=record.txt

10
conda/faiss/install-cmake.sh 100755
View File

@ -0,0 +1,10 @@
#!/bin/sh#
# Copyright (c) Facebook, Inc. and its affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
set -e
wget -O - https://github.com/Kitware/CMake/releases/download/v3.17.1/cmake-3.17.1-Linux-x86_64.tar.gz | tar xzf -
cp -R cmake-3.17.1-Linux-x86_64/* $PREFIX

101
conda/faiss/meta.yaml
View File

@ -1,35 +1,17 @@
# Copyright (c) Facebook, Inc. and its affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
{% set version = environ.get('GIT_DESCRIBE_TAG') %}
{% set number = environ.get('GIT_DESCRIBE_NUMBER') %}
package:
name: faiss-cpu
version: "{{ FAISS_BUILD_VERSION }}"
source:
git_url: ../../
requirements:
build:
- {{ compiler('cxx') }}
- python {{ python }}
- llvm-openmp # [osx]
- setuptools
- swig
- mkl >=2018
host:
- python {{ python }}
- intel-openmp # [osx]
- numpy 1.11.*
- blas=*=mkl
- mkl >=2018
run:
- python {{ python }}
- intel-openmp # [osx]
- numpy >=1.11
- blas=*=mkl
- mkl >=2018
name: faiss-pkg
version: {{ version }}
build:
number: {{ FAISS_BUILD_NUMBER }}
number: {{ number }}
about:
home: https://github.com/facebookresearch/faiss
@ -37,3 +19,64 @@ about:
license_family: MIT
license_file: LICENSE
summary: A library for efficient similarity search and clustering of dense vectors.
source:
git_url: ../../
outputs:
- name: libfaiss
script: build-lib.sh
build:
string: "h{{ PKG_HASH }}_{{ number }}_cpu"
run_exports:
- {{ pin_compatible('libfaiss', exact=True) }}
requirements:
build:
- {{ compiler('cxx') }}
- llvm-openmp # [osx]
- cmake-local
host:
- mkl >=2018
- blas =*=mkl
run:
- mkl >=2018
- blas =*=mkl
test:
commands:
- test -f $PREFIX/lib/libfaiss.so # [linux]
- test -f $PREFIX/lib/libfaiss.dylib # [osx]
- conda inspect linkages -p $PREFIX $PKG_NAME # [not win]
- conda inspect objects -p $PREFIX $PKG_NAME # [osx]
- name: faiss-cpu
script: build-pkg.sh
build:
string: "py{{ PY_VER }}_h{{ PKG_HASH }}_{{ number }}_cpu"
requirements:
build:
- {{ compiler('cxx') }}
- swig
- cmake-local
host:
- python {{ python }}
- numpy 1.11.*
- {{ pin_subpackage('libfaiss', exact=True) }}
run:
- python {{ python }}
- {{ pin_compatible('numpy') }}
- {{ pin_subpackage('libfaiss', exact=True) }}
test:
requires:
- numpy
- scipy
commands:
- python -m unittest discover tests/
source_files:
- tests/
- name: cmake-local
version: 3.17
script: install-cmake.sh
requirements:
build:
- {{ compiler('cxx') }}

14
conda/faiss/run_test.py
View File

@ -1,14 +0,0 @@
import faiss
import numpy as np
d = 128
# NOTE: BLAS kicks in only when n > distance_compute_blas_threshold = 20
n = 100
rs = np.random.RandomState(1337)
x = rs.rand(n, d).astype(np.float32)
index = faiss.IndexFlatL2(d)
index.add(x)
D, I = index.search(x, 10)