Add GPU CI. (#1378)

Summary: Pull Request resolved: https://github.com/facebookresearch/faiss/pull/1378

Reviewed By: mdouze

Differential Revision: D23370605

Pulled By: beauby

fbshipit-source-id: 6a9de47b6167ba8be12cce2735b1132991a190ab

.circleci/Dockerfile.cpu

@@ -0,0 +1,11 @@
+FROM cimg/base:stable-20.04
+
+# Install python3, swig, and MKL.
+RUN sudo apt-get update && \
+sudo apt-get install -y python3-dev python3-pip swig libmkl-dev
+
+# Install recent CMake.
+RUN wget -nv -O - https://github.com/Kitware/CMake/releases/download/v3.17.1/cmake-3.17.1-Linux-x86_64.tar.gz | sudo tar xzf - --strip-components=1 -C /usr
+
+# Install numpy/scipy for python tests.
+RUN pip3 install numpy scipy

.circleci/Dockerfile.cuda10.2

@@ -0,0 +1,13 @@
+FROM nvidia/cuda:10.2-devel-ubuntu18.04
+
+# Install python3, swig, and openblas.
+RUN apt-get update && \
+        apt-get install -y python3-dev python3-pip swig libopenblas-dev
+
+# Install recent CMake.
+RUN apt-get install -y wget && \
+        wget -nv -O - https://github.com/Kitware/CMake/releases/download/v3.17.1/cmake-3.17.1-Linux-x86_64.tar.gz | tar xzf - --strip-components=1 -C /usr && \
+        apt remove -y wget
+
+# Install numpy/scipy for python tests.
+RUN pip3 install numpy scipy

.circleci/Dockerfile.faiss_gpu

@@ -0,0 +1,14 @@
+FROM beauby/faiss-circleci:cuda10.2
+
+COPY . /faiss
+
+WORKDIR /faiss
+
+RUN cmake -B build \
+        -DFAISS_ENABLE_GPU=ON \
+        -DFAISS_ENABLE_PYTHON=ON \
+        -DBUILD_TESTING=ON \
+        -DCMAKE_CUDA_FLAGS="-gencode arch=compute_61,code=sm_61" \
+        .
+
+RUN make -C build -j20

.circleci/config.yml

@@ -6,66 +6,40 @@ orbs:
 jobs:
   build_linux:
     docker:
-      - image: circleci/python:3.7
+      - image: beauby/faiss-circleci:cpu
+    environment:
+      OMP_NUM_THREADS: 10
+      MKL_THREADING_LAYER: GNU
     steps:
       - checkout
-      - run:
-          name: Register Intel APT repository for MKL
-          command: |
-            cd /tmp
-            wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB
-            sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB
-            rm GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB
-            sudo sh -c 'echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list'
-      - run:
-          name: Install apt packages
-          command: |
-            sudo apt-get update
-            sudo apt-get install -y swig python3-dev intel-mkl-64bit-2020.0.088
-            echo "source /opt/intel/bin/compilervars.sh intel64" >> $BASH_ENV
-      - run:
-          name: Install numpy/scipy
-          command: |
-            pip install numpy scipy
-      - run:
-          name: Install cmake
-          command: |
-            wget -O - https://github.com/Kitware/CMake/releases/download/v3.17.1/cmake-3.17.1-Linux-x86_64.tar.gz | tar xzf -
-            sudo cp -R cmake-3.17.1-Linux-x86_64/* /usr
       - run:
           name: Build faiss library
           command: |
-            source $BASH_ENV
             cmake -B build -DBUILD_TESTING=ON -DFAISS_ENABLE_GPU=OFF \
                   -DCMAKE_BUILD_TYPE=Release -DBLA_VENDOR=Intel10_64_dyn .
             make -C build -j faiss
       - run:
           name: Test faiss library
           command: |
-            source $BASH_ENV
             make -C build -j faiss_test
             make -C build test
-          environment:
-            OMP_NUM_THREADS: 10
       - run:
           name: Build python extension
           command: |
-            source $BASH_ENV
             make -C build -j swigfaiss
             cd build/faiss/python
-            python setup.py build
+            python3 setup.py build
       - run:
           name: Test python extension
           command: |
-            source $BASH_ENV
             export PYTHONPATH="$(ls -d ./build/faiss/python/build/lib*/)"
-            python -m unittest discover ./tests
-          environment:
-            OMP_NUM_THREADS: 10
+            python3 -m unittest discover ./tests
 
   build_osx:
     macos:
       xcode: 11.6.0
+    environment:
+      OMP_NUM_THREADS: 10
     steps:
       - checkout
       - run:
@@ -91,8 +65,6 @@ jobs:
           command: |
             make -C build -j faiss_test
             make -C build test
-          environment:
-            OMP_NUM_THREADS: 10
       - run:
           name: Build python extension
           command: |
@@ -103,8 +75,6 @@ jobs:
           name: Test python extension
           command: |
             PYTHONPATH="$(ls -d ./build/faiss/python/build/lib*/)" python3 -m unittest discover ./tests
-          environment:
-            OMP_NUM_THREADS: 10
 
   build_windows:
     executor:
@@ -119,10 +89,26 @@ jobs:
             cd conda
             conda build faiss --python 3.7
 
+  build_linux_gpu:
+    machine:
+      resource_class: gpu.nvidia.small
+      image: ubuntu-1604-cuda-10.1:201909-23
+      docker_layer_caching: true
+    steps:
+      - checkout
+      - run:
+          name: Build/test
+          command: |
+            docker build -t faiss -f .circleci/Dockerfile.faiss_gpu .
+            docker run --gpus all faiss make -C build test
+
 workflows:
   version: 2
   build:
     jobs:
       - build_linux
+      - build_linux_gpu:
+          requires:
+            - build_linux
       - build_osx
       - build_windows

faiss/gpu/CMakeLists.txt

@@ -180,4 +180,4 @@ endforeach()
 
 find_package(CUDAToolkit REQUIRED)
 
-target_link_libraries(faiss PRIVATE CUDA::cuda_driver CUDA::cudart CUDA::cublas)
+target_link_libraries(faiss PRIVATE CUDA::cudart CUDA::cublas)

faiss/gpu/test/CMakeLists.txt

@@ -10,7 +10,7 @@ find_package(CUDAToolkit REQUIRED)
 include(GoogleTest)
 
 add_library(faiss_gpu_test_helper TestUtils.cpp)
-target_link_libraries(faiss_gpu_test_helper PUBLIC faiss gtest CUDA::cuda_driver)
+target_link_libraries(faiss_gpu_test_helper PUBLIC faiss gtest CUDA::cudart)
 
 macro(faiss_gpu_test file)
   get_filename_component(test_name ${file} NAME_WE)