From a6ea630e626362e6ee6af7af38013276aa4e0797 Mon Sep 17 00:00:00 2001 From: Miao Zheng <76149310+MeowZheng@users.noreply.github.com> Date: Tue, 10 Jan 2023 17:04:59 +0800 Subject: [PATCH] [Fix]Fix circile ci env (#2467) ## Motivation 1. circle ci failed without zlib1g-dev installation 3. add approve button ## Modification 1. .circleci/ --- .circleci/config.yml | 179 +++++----------------------- .circleci/docker/Dockerfile | 11 ++ .circleci/scripts/get_mmcv_var.sh | 19 +++ .circleci/test.yml | 192 ++++++++++++++++++++++++++++++ 4 files changed, 250 insertions(+), 151 deletions(-) create mode 100644 .circleci/docker/Dockerfile create mode 100644 .circleci/scripts/get_mmcv_var.sh create mode 100644 .circleci/test.yml diff --git a/.circleci/config.yml b/.circleci/config.yml index eccf5e295..b036f11f1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,157 +1,34 @@ version: 2.1 -jobs: - lint: - docker: - - image: cimg/python:3.7.4 - steps: - - checkout - - run: - name: Install dependencies - command: | - sudo apt-add-repository ppa:brightbox/ruby-ng -y - sudo apt-get update - sudo apt-get install -y ruby2.7 - - run: - name: Install pre-commit hook - command: | - pip install pre-commit - pre-commit install - - run: - name: Linting - command: pre-commit run --all-files - - run: - name: Check docstring coverage - command: | - pip install interrogate - interrogate -v --ignore-init-method --ignore-module --ignore-nested-functions --ignore-regex "__repr__" --fail-under 50 mmseg +# this allows you to use CircleCI's dynamic configuration feature +setup: true - build_cpu: - parameters: - # The python version must match available image tags in - # https://circleci.com/developer/images/image/cimg/python - python: - type: string - default: "3.7.4" - torch: - type: string - torchvision: - type: string - docker: - - image: cimg/python:<< parameters.python >> - resource_class: large - steps: - - checkout - - run: - name: Install Libraries - command: | - sudo apt-get update - sudo apt-get install -y ninja-build libglib2.0-0 libsm6 libxrender-dev libxext6 libgl1-mesa-glx libjpeg-dev zlib1g-dev libtinfo-dev libncurses5 - - run: - name: Configure Python & pip - command: | - python -m pip install --upgrade pip - python -m pip install wheel - - run: - name: Install PyTorch - command: | - python -V - python -m pip install torch==<< parameters.torch >>+cpu torchvision==<< parameters.torchvision >>+cpu -f https://download.pytorch.org/whl/torch_stable.html - - run: - name: Install mmseg dependencies - command: | - python -m pip install mmcv-full -f https://download.openmmlab.com/mmcv/dist/cpu/torch<< parameters.torch >>/index.html - python -m pip install -r requirements.txt - - run: - name: Build and install - command: | - python -m pip install -e . - - run: - name: Run unittests but skip timm unittests - command: | - python -m coverage run --branch --source mmseg -m pytest tests/ --ignore tests/test_models/test_backbones/test_timm_backbone.py - python -m coverage xml - python -m coverage report -m - - build_cu101: - machine: - image: ubuntu-1604-cuda-10.1:201909-23 - resource_class: gpu.nvidia.small - steps: - - checkout - - run: - name: Install Libraries - command: | - sudo apt-get update - sudo apt-get install -y git ninja-build libglib2.0-0 libsm6 libxrender-dev libxext6 libgl1-mesa-glx - - run: - name: Configure Python & pip - command: | - pyenv global 3.7.0 - python -m pip install --upgrade pip - python -m pip install wheel - - run: - name: Install PyTorch - command: | - python -V - python -m pip install torch==1.6.0+cu101 torchvision==0.7.0+cu101 -f https://download.pytorch.org/whl/torch_stable.html - - run: - name: Install mmseg dependencies - # python -m pip install mmcv-full -f https://download.openmmlab.com/mmcv/dist/cu101/torch${{matrix.torch_version}}/index.html - command: | - python -m pip install mmcv-full -f https://download.openmmlab.com/mmcv/dist/cu101/torch1.6.0/index.html - python -m pip install -r requirements.txt - - run: - name: Build and install - command: | - python setup.py check -m -s - TORCH_CUDA_ARCH_LIST=7.0 python -m pip install -e . - - run: - name: Run unittests but skip timm unittests - command: | - python -m pytest tests/ --ignore tests/test_models/test_backbones/test_timm_backbone.py +# the path-filtering orb is required to continue a pipeline based on +# the path of an updated fileset +orbs: + path-filtering: circleci/path-filtering@0.1.2 workflows: - unit_tests: + # the always-run workflow is always triggered, regardless of the pipeline parameters. + always-run: jobs: - - lint - - build_cpu: - name: build_cpu_th1.6 - torch: 1.6.0 - torchvision: 0.7.0 - requires: - - lint - - build_cpu: - name: build_cpu_th1.7 - torch: 1.7.0 - torchvision: 0.8.1 - requires: - - lint - - build_cpu: - name: build_cpu_th1.8_py3.9 - torch: 1.8.0 - torchvision: 0.9.0 - python: "3.9.0" - requires: - - lint - - build_cpu: - name: build_cpu_th1.9_py3.8 - torch: 1.9.0 - torchvision: 0.10.0 - python: "3.8.0" - requires: - - lint - - build_cpu: - name: build_cpu_th1.9_py3.9 - torch: 1.9.0 - torchvision: 0.10.0 - python: "3.9.0" - requires: - - lint - - build_cu101: - requires: - - build_cpu_th1.6 - - build_cpu_th1.7 - - build_cpu_th1.8_py3.9 - - build_cpu_th1.9_py3.8 - - build_cpu_th1.9_py3.9 + # the path-filtering/filter job determines which pipeline + # parameters to update. + - path-filtering/filter: + name: check-updated-files + # 3-column, whitespace-delimited mapping. One mapping per + # line: + # + mapping: | + mmseg/.* lint_only false + requirements/.* lint_only false + tests/.* lint_only false + tools/.* lint_only false + configs/.* lint_only false + .circleci/.* lint_only false + base-revision: master + # this is the path of the configuration we should trigger once + # path filtering and pipeline parameter value updates are + # complete. In this case, we are using the parent dynamic + # configuration itself. + config-path: .circleci/test.yml diff --git a/.circleci/docker/Dockerfile b/.circleci/docker/Dockerfile new file mode 100644 index 000000000..d9cf8cc77 --- /dev/null +++ b/.circleci/docker/Dockerfile @@ -0,0 +1,11 @@ +ARG PYTORCH="1.8.1" +ARG CUDA="10.2" +ARG CUDNN="7" + +FROM pytorch/pytorch:${PYTORCH}-cuda${CUDA}-cudnn${CUDNN}-devel + +# To fix GPG key error when running apt-get update +RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/3bf863cc.pub +RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64/7fa2af80.pub + +RUN apt-get update && apt-get install -y ninja-build libglib2.0-0 libsm6 libxrender-dev libxext6 libgl1-mesa-glx diff --git a/.circleci/scripts/get_mmcv_var.sh b/.circleci/scripts/get_mmcv_var.sh new file mode 100644 index 000000000..552ff871a --- /dev/null +++ b/.circleci/scripts/get_mmcv_var.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +TORCH=$1 +CUDA=$2 + +# 10.2 -> cu102 +MMCV_CUDA="cu`echo ${CUDA} | tr -d '.'`" + +# MMCV only provides pre-compiled packages for torch 1.x.0 +# which works for any subversions of torch 1.x. +# We force the torch version to be 1.x.0 to ease package searching +# and avoid unnecessary rebuild during MMCV's installation. +TORCH_VER_ARR=(${TORCH//./ }) +TORCH_VER_ARR[2]=0 +printf -v MMCV_TORCH "%s." "${TORCH_VER_ARR[@]}" +MMCV_TORCH=${MMCV_TORCH%?} # Remove the last dot + +echo "export MMCV_CUDA=${MMCV_CUDA}" >> $BASH_ENV +echo "export MMCV_TORCH=${MMCV_TORCH}" >> $BASH_ENV diff --git a/.circleci/test.yml b/.circleci/test.yml new file mode 100644 index 000000000..d77170611 --- /dev/null +++ b/.circleci/test.yml @@ -0,0 +1,192 @@ + + +version: 2.1 + +# the default pipeline parameters, which will be updated according to +# the results of the path-filtering orb + +parameters: + lint_only: + type: boolean + default: true + +jobs: + lint: + docker: + - image: cimg/python:3.7.4 + steps: + - checkout + - run: + name: Install dependencies + command: | + sudo apt-add-repository ppa:brightbox/ruby-ng -y + sudo apt-get update + sudo apt-get install -y ruby2.7 + - run: + name: Install pre-commit hook + command: | + pip install pre-commit + pre-commit install + - run: + name: Linting + command: pre-commit run --all-files + - run: + name: Check docstring coverage + command: | + pip install interrogate + interrogate -v --ignore-init-method --ignore-module --ignore-nested-functions --ignore-regex "__repr__" --fail-under 50 mmseg + + build_cpu: + parameters: + # The python version must match available image tags in + # https://circleci.com/developer/images/image/cimg/python + python: + type: string + default: "3.7.4" + torch: + type: string + torchvision: + type: string + docker: + - image: cimg/python:<< parameters.python >> + resource_class: large + steps: + - checkout + - run: + name: Get MMCV_TORCH as environment variables + command: | + . .circleci/scripts/get_mmcv_var.sh << parameters.torch >> + source $BASH_ENV + - run: + name: Install Libraries + command: | + sudo apt-get update + sudo apt-get install -y ninja-build libglib2.0-0 libsm6 libxrender-dev libxext6 libgl1-mesa-glx libjpeg-dev zlib1g-dev libtinfo-dev libncurses5 + - run: + name: Configure Python & pip + command: | + python -m pip install --upgrade pip + python -m pip install wheel + - run: + name: Install PyTorch + command: | + python -V + python -m pip install torch==<< parameters.torch >>+cpu torchvision==<< parameters.torchvision >>+cpu -f https://download.pytorch.org/whl/torch_stable.html + - run: + name: Install mmseg dependencies + command: | + python -m pip install mmcv-full -f https://download.openmmlab.com/mmcv/dist/cpu/torch<< parameters.torch >>/index.html + python -m pip install -r requirements.txt + - run: + name: Build and install + command: | + python -m pip install -e . + - run: + name: Run unittests but skip timm unittests + command: | + python -m coverage run --branch --source mmseg -m pytest tests/ --ignore tests/test_models/test_backbones/test_timm_backbone.py + python -m coverage xml + python -m coverage report -m + + build_cuda: + parameters: + torch: + type: string + cuda: + type: enum + enum: ["10.1", "10.2", "11.1"] + cudnn: + type: integer + default: 7 + machine: + image: ubuntu-2004-cuda-11.4:202110-01 + docker_layer_caching: true + resource_class: gpu.nvidia.small + steps: + - checkout + - run: + name: Get MMCV_TORCH and MMCV_CUDA as environment variables + command: | + . .circleci/scripts/get_mmcv_var.sh << parameters.torch >> << parameters.cuda >> + source $BASH_ENV + - run: + name: Build Docker image + command: | + docker build .circleci/docker -t mmseg:gpu --build-arg PYTORCH=<< parameters.torch >> --build-arg CUDA=<< parameters.cuda >> --build-arg CUDNN=<< parameters.cudnn >> + docker run --gpus all -t -d -v /home/circleci/project:/mmseg -w /mmseg --name mmseg mmseg:gpu + - run: + name: Install mmseg dependencies + command: | + docker exec mmseg pip install mmcv-full -f https://download.openmmlab.com/mmcv/dist/${MMCV_CUDA}/torch${MMCV_TORCH}/index.html + docker exec mmseg pip install -r requirements.txt + - run: + name: Build and install + command: | + docker exec mmseg pip install -e . + - run: + name: Run unittests but skip timm unittests + command: | + docker exec mmseg python -m pytest tests/ --ignore tests/test_models/test_backbones/test_timm_backbone.py + +workflows: + pr_stage_lint: + when: << pipeline.parameters.lint_only >> + jobs: + - lint: + name: lint + filters: + branches: + ignore: + - master + pr_stage_test: + when: + not: + << pipeline.parameters.lint_only >> + jobs: + - lint: + name: lint + filters: + branches: + ignore: + - master + - build_cpu: + name: minimum_version_cpu + torch: 1.6.0 + torchvision: 0.7.0 + python: 3.7.7 + requires: + - lint + - build_cpu: + name: maximum_version_cpu + torch: 1.9.0 + torchvision: 0.10.0 + python: 3.8.0 + requires: + - minimum_version_cpu + - hold: + type: approval + requires: + - maximum_version_cpu + - build_cuda: + name: mainstream_version_gpu + torch: 1.8.1 + # Use double quotation mark to explicitly specify its type + # as string instead of number + cuda: "10.2" + requires: + - hold + merge_stage_test: + when: + not: + << pipeline.parameters.lint_only >> + jobs: + - build_cuda: + name: minimum_version_gpu + torch: 1.6.0 + # Use double quotation mark to explicitly specify its type + # as string instead of number + cuda: "10.1" + filters: + branches: + only: + - master