faiss/.circleci/config.yml

version: 2.1

executors:
  linux-x86_64-cpu:
    docker:
      - image: continuumio/miniconda3
    resource_class: large
  linux-x86_64-gpu:
    environment:
      CONDA_ARCH: Linux-x86_64
    machine:
      image: linux-cuda-11:2023.02.1
    resource_class: gpu.nvidia.medium
  linux-arm64-cpu:
    environment:
      CONDA_ARCH: Linux-aarch64
    machine:
      image: ubuntu-2004:current
    resource_class: arm.medium
  macosx-x86_64-cpu:
    environment:
      CONDA_ARCH: MacOSX-x86_64
    macos:
      xcode: 11.7.0  # max supported for conda build, https://circleci.com/docs/using-macos#supported-xcode-versions
  macosx-arm64-cpu:
    environment:
      CONDA_ARCH: MacOSX-arm64
    macos:
      xcode: 14.2.0 # minimum supported for M1
    resource_class: macos.m1.large.gen1
  windows-x86_64-cpu:
    machine:
      image: windows-server-2019-vs2019:stable
      shell: bash.exe
    resource_class: windows.medium

jobs:
  format:
    docker:
      - image: ubuntu:22.04
    steps:
      - checkout
      - run:
          name: Install clang-format
          command: |
            apt-get update
            apt-get install -y git-core clang-format-11
      - run:
          name: Verify clang-format
          command: |
             git ls-files | grep -E  '\.(cpp|h|cu|cuh)$' | xargs clang-format-11 -i
             if git diff --quiet; then
               echo "Formatting OK!"
             else
               echo "Formatting not OK!"
               echo "------------------"
               git --no-pager diff --color
               exit 1
             fi

  build_conda:
    parameters:
      label:
        type: string
        default: ""
      cuda:
        type: string
        default: ""
      cuda_archs:
        type: string
        default: ""
      compiler_version:
        type: string
        default: ""
      exec:
        type: executor
    executor: << parameters.exec >>
    environment:
      OMP_NUM_THREADS: 10
      PACKAGE_TYPE: <<parameters.label>>
      CUDA_ARCHS: <<parameters.cuda_archs>>
    steps:
      - checkout
      - run:
          name: Install conda
          command: |
            if [ -n "${CONDA_ARCH}" ]
            then
              curl https://repo.anaconda.com/miniconda/Miniconda3-latest-${CONDA_ARCH}.sh --output miniconda.sh
              bash miniconda.sh -b -p $HOME/miniconda
              ~/miniconda/bin/conda init
            fi
      - run:
          name: Install conda build tools
          command: |
            conda update -y -q conda
            conda install -y -q conda-build
      - when:
          condition: << parameters.label >>
          steps:
            - run:
                name: Enable anaconda uploads
                command: |
                  conda install -y -q anaconda-client
                  conda config --set anaconda_upload yes
      - when:
          condition:
              not: << parameters.label >>
          steps:
            - run:
                name: Conda build (CPU)
                no_output_timeout: 30m
                command: |
                  cd conda
                  conda build faiss --python 3.10 -c pytorch -c pkgs/main -c conda-forge
      - when:
          condition:
            and:
              - << parameters.label >>
              - not: << parameters.cuda >>
          steps:
            - run:
                name: Conda build (CPU) w/ anaconda upload
                no_output_timeout: 30m
                command: |
                  cd conda
                  conda build faiss --user pytorch --label <<parameters.label>> -c pytorch -c pkgs/main -c conda-forge
      - when:
          condition:
            and:
              - << parameters.label >>
              - << parameters.cuda >>
          steps:
            - run:
                name: Conda build (GPU) w/ anaconda upload
                no_output_timeout: 60m
                command: |
                  sudo update-alternatives --set cuda /usr/local/cuda-<<parameters.cuda>>
                  cd conda
                  conda build faiss-gpu --variants '{ "cudatoolkit": "<<parameters.cuda>>", "c_compiler_version": "<<parameters.compiler_version>>", "cxx_compiler_version": "<<parameters.compiler_version>>" }' \
                      --user pytorch --label <<parameters.label>> -c pytorch -c nvidia -c pkgs/main -c conda-forge

  build_cmake:
    parameters:
      exec:
        type: executor
      opt_level:
        type: string
        default: generic
      gpu:
        type: string
        default: "OFF"
      raft:
        type: string
        default: "OFF"
    executor: << parameters.exec >>
    environment:
      OMP_NUM_THREADS: 10
      MKL_THREADING_LAYER: GNU
    steps:
      - checkout
      - run:
          name: Install conda
          command: |
            if [ -n "${CONDA_ARCH}" ]
            then
              curl https://repo.anaconda.com/miniconda/Miniconda3-latest-${CONDA_ARCH}.sh --output miniconda.sh
              bash miniconda.sh -b -p $HOME/miniconda
              ~/miniconda/bin/conda init
            fi
      - when:
          condition:
            equal: [ "ON", << parameters.gpu >> ]
          steps:
            - run:
                name: Configure CUDA
                command: sudo update-alternatives --set cuda /usr/local/cuda-11.4
      - run:
          name: Set up environment
          command: |
            conda update -y -q conda
            conda install -y -q cmake=3.23.1 make swig mkl=2023 mkl-devel=2023 numpy scipy pytest gxx_linux-64 sysroot_linux-64=2.17 -c pkgs/main -c conda-forge
      - run:
          name: Build all targets
          no_output_timeout: 30m
          command: |
            eval "$(conda shell.bash hook)"
            conda activate
            cmake -B build \
                  -DBUILD_TESTING=ON \
                  -DBUILD_SHARED_LIBS=OFF \
                  -DFAISS_ENABLE_GPU=<< parameters.gpu >> \
                  -DFAISS_ENABLE_RAFT=<< parameters.raft >> \
                  -DFAISS_OPT_LEVEL=<< parameters.opt_level >> \
                  -DFAISS_ENABLE_C_API=ON \
                  -DPYTHON_EXECUTABLE=$(which python) \
                  -DCMAKE_BUILD_TYPE=Release \
                  -DBLA_VENDOR=Intel10_64_dyn \
                  -DCMAKE_CUDA_FLAGS="-gencode arch=compute_75,code=sm_75" \
                  .
            make -k -C build -j$(nproc)
      - run:
          name: C++ tests
          command: |
            export GTEST_OUTPUT="xml:$(realpath .)/test-results/googletest/"
            make -C build test
      - run:
          name: Install Python extension
          command: |
            cd build/faiss/python
            python setup.py install
      - when:
          condition:
            equal: [ "OFF", << parameters.gpu >> ]
          steps:
            - run:
                name: Python tests (CPU only)
                command: |
                  conda install -y -q pytorch -c pytorch
                  pytest --junitxml=test-results/pytest/results.xml tests/test_*.py
                  pytest --junitxml=test-results/pytest/results-torch.xml tests/torch_*.py
      - when:
          condition:
            equal: [ "ON", << parameters.gpu >> ]
          steps:
            - run:
                name: Python tests (CPU + GPU)
                command: |
                  conda install -y -q pytorch pytorch-cuda -c pytorch -c nvidia
                  pytest --junitxml=test-results/pytest/results.xml tests/test_*.py
                  pytest --junitxml=test-results/pytest/results-torch.xml tests/torch_*.py
                  cp tests/common_faiss_tests.py faiss/gpu/test
                  pytest --junitxml=test-results/pytest/results-gpu.xml faiss/gpu/test/test_*.py
                  pytest --junitxml=test-results/pytest/results-gpu-torch.xml faiss/gpu/test/torch_*.py
      - when:
          condition:
            equal: [ "avx2", << parameters.opt_level >> ]
          steps:
            - run:
                name: Test avx2 loading
                command: |
                  FAISS_DISABLE_CPU_FEATURES=AVX2 LD_DEBUG=libs python -c "import faiss" 2>&1 | grep faiss.so
                  LD_DEBUG=libs python -c "import faiss" 2>&1 | grep faiss_avx2.so
      - store_test_results:
          path: test-results

workflows:
  version: 2
  build:
    jobs:
      - format:
          name: Format
      - build_cmake:
          name: Linux x86_64 (cmake)
          exec: linux-x86_64-cpu
      - build_cmake:
          name: Linux x86_64 AVX2 (cmake)
          exec: linux-x86_64-cpu
          opt_level: "avx2"
      - build_cmake:
          name: Linux x86_64 GPU (cmake)
          exec: linux-x86_64-gpu
          gpu: "ON"
          requires:
            - Linux x86_64 (cmake)
      - build_cmake:
          name: Linux x86_64 GPU w/ RAFT (cmake)
          exec: linux-x86_64-gpu
          gpu: "ON"
          raft: "ON"
          requires:
            - Linux x86_64 (cmake)
      - build_conda:
          name: Linux x86_64 (conda)
          exec: linux-x86_64-cpu
      - build_conda:
          name: OSX x86_64 (conda)
          exec: macosx-x86_64-cpu
      - build_conda:
          name: Windows x86_64 (conda)
          exec: windows-x86_64-cpu
      - build_conda:
          name: Linux arm64 (conda)
          exec: linux-arm64-cpu
      - build_conda:
          name: Linux x86_64 packages
          exec: linux-x86_64-cpu
          label: main
          filters:
            tags:
              only: /^v.*/
            branches:
              ignore: /.*/
      - build_conda:
          name: Linux x86_64 GPU packages (CUDA 11.4)
          exec: linux-x86_64-gpu
          label: main
          cuda: "11.4"
          cuda_archs: "60;61;70;72;75;80;86"
          compiler_version: "11.2"
          filters:
            tags:
              only: /^v.*/
            branches:
              ignore: /.*/
      - build_conda:
          name: Windows x86_64 packages
          exec: windows-x86_64-cpu
          label: main
          filters:
            tags:
              only: /^v.*/
            branches:
              ignore: /.*/
      - build_conda:
          name: OSX x86_64 packages
          exec: macosx-x86_64-cpu
          label: main
          filters:
            tags:
              only: /^v.*/
            branches:
              ignore: /.*/
      - build_conda:
          name: OSX arm64 packages
          exec: macosx-arm64-cpu
          label: main
          filters:
            tags:
              only: /^v.*/
            branches:
              ignore: /.*/
      - build_conda:
          name: Linux arm64 packages
          exec: linux-arm64-cpu
          label: main
          filters:
            tags:
              only: /^v.*/
            branches:
              ignore: /.*/

  nightly:
    triggers:
      - schedule:
          cron: "0 0 * * *"
          filters:
            branches:
              only:
                - main
    jobs:
      - build_conda:
          name: Linux x86_64 nightlies
          exec: linux-x86_64-cpu
          label: nightly
      - build_conda:
          name: Linux x86_64 GPU nightlies (CUDA 11.4)
          exec: linux-x86_64-gpu
          cuda: "11.4"
          cuda_archs: "60;61;70;72;75;80;86"
          compiler_version: "11.2"
          label: nightly
      - build_conda:
          name: Windows x86_64 nightlies
          exec: windows-x86_64-cpu
          label: nightly
      - build_conda:
          name: OSX x86_64 nightlies
          exec: macosx-x86_64-cpu
          label: nightly
      - build_conda:
          name: OSX arm64 nightlies
          exec: macosx-arm64-cpu
          label: nightly
      - build_conda:
          name: Linux arm64 nightlies
          exec: linux-arm64-cpu
          label: nightly