mirrors
/
mmselfsup
mirror of https://github.com/open-mmlab/mmselfsup.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[CI] update CI package (#476) 

* update circleci

* update github action

* fix windows error

* update

* update pr and merge state

* fix lint

* update

* update

* update circle ci

* fix

* fix ut
pull/567/head
Yixiao Fang 2022-09-16 16:33:08 +08:00
parent 8b19432ac3
commit ff5f5ae2a9
10 changed files with 348 additions and 159 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
.circleci/test.yml
View File

@ -25,7 +25,7 @@ jobs:
name: Check docstring coverage
command: |
pip install interrogate
interrogate -v --ignore-init-method --ignore-module --ignore-nested-functions --ignore-magic --ignore-regex "__repr__" --fail-under 80 mmselfsup
interrogate -v --ignore-init-method --ignore-module --ignore-nested-functions --ignore-magic --ignore-regex "__repr__" --fail-under 90 mmselfsup
build_cpu:
parameters:
# The python version must match available image tags in
@ -36,8 +36,6 @@ jobs:
type: string
torchvision:
type: string
mmcv:
type: string
docker:
- image: cimg/python:<< parameters.python >>
resource_class: large
@ -51,30 +49,30 @@ jobs:
- run:
name: Configure Python & pip
command: |
python -m pip install --upgrade pip
python -m pip install wheel
pip install --upgrade pip
pip install wheel
- run:
name: Install PyTorch
command: |
python -V
python -m pip install torch==<< parameters.torch >>+cpu torchvision==<< parameters.torchvision >>+cpu -f https://download.pytorch.org/whl/torch_stable.html
pip install torch==<< parameters.torch >>+cpu torchvision==<< parameters.torchvision >>+cpu -f https://download.pytorch.org/whl/torch_stable.html
- run:
name: Install mmselfsup dependencies
command: |
python -m pip install git+ssh://git@github.com/open-mmlab/mmengine.git@main
python -m pip install << parameters.mmcv >>
python -m pip install git+ssh://git@github.com/open-mmlab/mmclassification.git@dev-1.x
python -m pip install -r requirements.txt
pip install git+https://github.com/open-mmlab/mmengine.git@main
pip install -U openmim
mim install 'mmcv >= 2.0.0rc1'
pip install -r requirements/tests.txt
- run:
name: Build and install
command: |
python -m pip install -e .
pip install -e .
- run:
name: Run unittests
command: |
python -m coverage run --branch --source mmselfsup -m pytest tests/
python -m coverage xml
python -m coverage report -m
coverage run --branch --source mmselfsup -m pytest tests/
coverage xml
coverage report -m
build_cuda:
parameters:
torch:
@ -85,8 +83,6 @@ jobs:
cudnn:
type: integer
default: 7
mmcv:
type: string
machine:
image: ubuntu-2004-cuda-11.4:202110-01
# docker_layer_caching: true
@ -97,8 +93,8 @@ jobs:
# CLoning repos in VM since Docker doesn't have access to the private key
name: Clone Repos
command: |
git clone -b main --depth 1 ssh://git@github.com/open-mmlab/mmengine.git /home/circleci/mmengine
git clone -b dev-1.x --depth 1 ssh://git@github.com/open-mmlab/mmclassification.git /home/circleci/mmclassification
git clone -b main --depth 1 https://github.com/open-mmlab/mmengine.git /home/circleci/mmengine
git clone -b dev-1.x --depth 1 https://github.com/open-mmlab/mmclassification.git /home/circleci/mmclassification
- run:
name: Build Docker image
command: |
@ -108,9 +104,9 @@ jobs:
name: Install mmselfsup dependencies
command: |
docker exec mmselfsup pip install -e /mmengine
docker exec mmselfsup pip install << parameters.mmcv >>
docker exec mmselfsup pip install -e /mmclassification
docker exec mmselfsup pip install -r requirements.txt
docker exec mmselfsup pip install -U openmim
docker exec mmselfsup mim install 'mmcv >= 2.0.0rc1'
docker exec mmselfsup pip install -r requirements/tests.txt
- run:
name: Build and install
command: |
@ -118,7 +114,8 @@ jobs:
- run:
name: Run unittests
command: |
docker exec mmselfsup python -m pytest tests/
docker exec mmselfsup pytest tests/
workflows:
pr_stage_lint:
when: << pipeline.parameters.lint_only >>
@ -129,6 +126,7 @@ workflows:
branches:
ignore:
- dev-1.x
- 1.x
pr_stage_test:
when:
not:
@ -145,7 +143,6 @@ workflows:
torch: 1.6.0
torchvision: 0.7.0
python: 3.6.9 # The lowest python 3.6.x version available on CircleCI images
mmcv: https://download.openmmlab.com/mmcv/dev-2.x/cpu/torch1.6.0/mmcv_full-2.0.0rc0-cp36-cp36m-manylinux1_x86_64.whl
requires:
- lint
- build_cpu:
@ -153,7 +150,6 @@ workflows:
torch: 1.9.0
torchvision: 0.10.0
python: 3.9.0
mmcv: https://download.openmmlab.com/mmcv/dev-2.x/cpu/torch1.9.0/mmcv_full-2.0.0rc0-cp39-cp39-manylinux1_x86_64.whl
requires:
- minimum_version_cpu
- hold:
@ -166,7 +162,6 @@ workflows:
# Use double quotation mark to explicitly specify its type
# as string instead of number
cuda: "10.2"
mmcv: https://download.openmmlab.com/mmcv/dev-2.x/cu102/torch1.8.0/mmcv_full-2.0.0rc0-cp37-cp37m-manylinux1_x86_64.whl
requires:
- hold
merge_stage_test:
@ -179,7 +174,6 @@ workflows:
torch: 1.6.0
# Use double quotation mark to explicitly specify its type
# as string instead of number
mmcv: https://download.openmmlab.com/mmcv/dev-2.x/cu101/torch1.6.0/mmcv_full-2.0.0rc0-cp37-cp37m-manylinux1_x86_64.whl
cuda: "10.1"
filters:
branches:

2
.github/ISSUE_TEMPLATE/config.yml vendored
View File

@ -2,5 +2,5 @@ blank_issues_enabled: false
contact_links:
- name: MMSelfSup Documentation
url: https://mmselfsup.readthedocs.io/en/latest/
url: https://mmselfsup.readthedocs.io/en/1.x/
about: Check if your question is answered in docs

130
.github/workflows/build.yml vendored
View File

@ -1,130 +0,0 @@
# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
name: build
on:
push:
paths-ignore:
- 'README.md'
- 'README_zh-CN.md'
- 'docs/**'
- 'model-index.yml'
- 'configs/**.md'
pull_request:
paths-ignore:
- 'README.md'
- 'README_zh-CN.md'
- 'docs/**'
- 'model-index.yml'
- 'configs/**.md'
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
# jobs:
# build:
# runs-on: ubuntu-latest
# env:
# UBUNTU_VERSION: ubuntu1804
# strategy:
# matrix:
# python-version: [3.7]
# torch: [1.6.0, 1.7.0, 1.8.0, 1.9.0]
# include:
# - torch: 1.6.0
# torch_version: 1.6
# torchvision: 0.7.0
# - torch: 1.7.0
# torch_version: 1.7
# torchvision: 0.8.1
# - torch: 1.7.0
# torch_version: 1.7
# torchvision: 0.8.1
# python-version: 3.8
# - torch: 1.8.0
# torch_version: 1.8
# torchvision: 0.9.0
# - torch: 1.8.0
# torch_version: 1.8
# torchvision: 0.9.0
# python-version: 3.8
# - torch: 1.9.0
# torch_version: 1.9
# torchvision: 0.10.0
# - torch: 1.9.0
# torch_version: 1.9
# torchvision: 0.10.0
# python-version: 3.8
# steps:
# - uses: actions/checkout@v2
# - name: Set up Python ${{ matrix.python-version }}
# uses: actions/setup-python@v2
# with:
# python-version: ${{ matrix.python-version }}
# - name: Install PyTorch
# run: pip install --use-deprecated=legacy-resolver torch==${{matrix.torch}}+cpu torchvision==${{matrix.torchvision}}+cpu -f https://download.pytorch.org/whl/torch_stable.html
# - name: Install MMCV
# run: |
# pip install --use-deprecated=legacy-resolver mmcv-full -f https://download.openmmlab.com/mmcv/dist/cpu/torch${{matrix.torch_version}}/index.html
# python -c 'import mmcv; print(mmcv.__version__)'
# - name: Install mmselfsup dependencies
# run: |
# pip install -r requirements.txt
# - name: Build and install
# run: |
# rm -rf .eggs
# pip install -e . -U
# - name: Run unittests and generate coverage report
# run: |
# coverage run --branch --source mmselfsup -m pytest tests/
# coverage xml
# coverage report -m
# # Only upload coverage report for python3.8 && pytorch1.9.0
# - name: Upload coverage to Codecov
# if: ${{matrix.torch == '1.9.0' && matrix.python-version == '3.8'}}
# uses: codecov/codecov-action@v2
# with:
# file: ./coverage.xml
# flags: unittests
# env_vars: OS,PYTHON
# name: codecov-umbrella
# fail_ci_if_error: false
# test_windows:
# runs-on: ${{ matrix.os }}
# strategy:
# matrix:
# os: [windows-2022]
# python: [3.8]
# platform: [cpu, cu111]
# steps:
# - uses: actions/checkout@v2
# - name: Set up Python ${{ matrix.python }}
# uses: actions/setup-python@v2
# with:
# python-version: ${{ matrix.python }}
# - name: Install PyTorch
# # As a complement to Linux CI, we test on PyTorch LTS version
# run: pip install torch==1.8.2+${{ matrix.platform }} torchvision==0.9.2+${{ matrix.platform }} -f https://download.pytorch.org/whl/lts/1.8/torch_lts.html
# - name: Install OpenCV
# run: |
# pip install opencv-python>=3
# - name: Install MMCV
# run: |
# pip install mmcv-full -f https://download.openmmlab.com/mmcv/dist/cpu/torch1.8/index.html --only-binary mmcv-full
# - name: Install unittest dependencies
# run: |
# pip install -r requirements.txt
# - name: Build and install
# run: pip install -e .
# - name: Run unittests
# run: |
# coverage run --branch --source mmselfsup -m pytest tests/
# - name: Generate coverage report
# run: |
# coverage xml
# coverage report -m --omit="mmselfsup/apis/*"

179
.github/workflows/merge_stage_test.yml vendored 100644
View File

@ -0,0 +1,179 @@
name: merge_stage_test
on:
push:
paths-ignore:
- 'README.md'
- 'README_zh-CN.md'
- 'docs/**'
- 'demo/**'
- '.dev_scripts/**'
- '.circleci/**'
branches:
- dev-1.x
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
build_cpu_py:
runs-on: ubuntu-18.04
strategy:
matrix:
python-version: [3.6, 3.8, 3.9]
torch: [1.8.1]
include:
- torch: 1.8.1
torchvision: 0.9.1
steps:
- uses: actions/checkout@v2
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Upgrade pip
run: pip install pip --upgrade
- name: Install PyTorch
run: pip install torch==${{matrix.torch}}+cpu torchvision==${{matrix.torchvision}}+cpu -f https://download.pytorch.org/whl/torch_stable.html
- name: Install MMEngine
run: pip install git+https://github.com/open-mmlab/mmengine.git@main
- name: Install MMCV
run: |
pip install -U openmim
mim install 'mmcv >= 2.0.0rc1'
- name: Install other dependencies
run: pip install -r requirements/tests.txt
- name: Build and install
run: rm -rf .eggs && pip install -e .
- name: Run unittests and generate coverage report
run: |
coverage run --branch --source mmselfsup -m pytest tests/
coverage xml
coverage report -m
build_cpu_pt:
runs-on: ubuntu-18.04
strategy:
matrix:
python-version: [3.7]
torch: [1.6.0, 1.7.1, 1.8.1, 1.9.1, 1.10.1, 1.11.0, 1.12.1]
include:
- torch: 1.6.0
torchvision: 0.7.0
- torch: 1.7.1
torchvision: 0.8.2
- torch: 1.8.1
torchvision: 0.9.1
- torch: 1.9.1
torchvision: 0.10.1
- torch: 1.10.1
torchvision: 0.11.2
- torch: 1.11.0
torchvision: 0.12.0
- torch: 1.12.1
torchvision: 0.13.1
steps:
- uses: actions/checkout@v2
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Install PyTorch
run: pip install torch==${{matrix.torch}}+cpu torchvision==${{matrix.torchvision}}+cpu -f https://download.pytorch.org/whl/torch_stable.html
- name: Install MMEngine
run: pip install git+https://github.com/open-mmlab/mmengine.git@main
- name: Install MMCV
run: |
pip install -U openmim
mim install 'mmcv >= 2.0.0rc1'
- name: Install other dependencies
run: pip install -r requirements/tests.txt
- name: Build and install
run: rm -rf .eggs && pip install -e .
- name: Run unittests and generate coverage report
run: |
coverage run --branch --source mmselfsup -m pytest tests/
coverage xml
coverage report -m
# Only upload coverage report for python3.8 && pytorch1.9.0
- name: Upload coverage to Codecov
if: ${{matrix.torch == '1.8.1' && matrix.python-version == '3.7'}}
uses: codecov/codecov-action@v2
with:
file: ./coverage.xml
flags: unittests
env_vars: OS,PYTHON
name: codecov-umbrella
fail_ci_if_error: false
build_cu102:
runs-on: ubuntu-18.04
container:
image: pytorch/pytorch:1.8.1-cuda10.2-cudnn7-devel
strategy:
matrix:
python-version: [3.7]
include:
- torch: 1.8.1
cuda: 10.2
steps:
- uses: actions/checkout@v2
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Upgrade pip
run: pip install pip --upgrade
- name: Fetch GPG keys
run: |
apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/3bf863cc.pub
apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64/7fa2af80.pub
- name: Install Python-dev
run: apt-get update && apt-get install -y python${{matrix.python-version}}-dev
if: ${{matrix.python-version != 3.9}}
- name: Install system dependencies
run: |
apt-get update && apt-get install -y ffmpeg libsm6 libxext6 git ninja-build libglib2.0-0 libsm6 libxrender-dev libxext6
- name: Install mmselfsup dependencies
run: |
pip install git+https://github.com/open-mmlab/mmengine.git@main
pip install -U openmim
mim install 'mmcv >= 2.0.0rc1'
pip install -r requirements/tests.txt
- name: Build and install
run: |
python setup.py check -m -s
TORCH_CUDA_ARCH_LIST=7.0 pip install -e .
build_windows:
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [windows-2022]
python: [3.7]
platform: [cpu, cu111]
steps:
- uses: actions/checkout@v2
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Upgrade pip
run: pip install pip --upgrade
- name: Install lmdb
run: pip install lmdb
- name: Install PyTorch
run: pip install torch==1.8.1+${{matrix.platform}} torchvision==0.9.1+${{matrix.platform}} -f https://download.pytorch.org/whl/lts/1.8/torch_lts.html
- name: Install mmselfsup dependencies
run: |
pip install git+https://github.com/open-mmlab/mmengine.git@main
pip install -U openmim
mim install 'mmcv >= 2.0.0rc1'
pip install -r requirements/tests.txt
- name: Build and install
run: |
pip install -e .
- name: Run unittests and generate coverage report
run: |
pytest tests/

128
.github/workflows/pr_stage_test.yml vendored 100644
View File

@ -0,0 +1,128 @@
name: pr_stage_test
on:
pull_request:
paths-ignore:
- 'README.md'
- 'README_zh-CN.md'
- 'docs/**'
- 'demo/**'
- '.dev_scripts/**'
- '.circleci/**'
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
build_cpu:
runs-on: ubuntu-18.04
strategy:
matrix:
python-version: [3.7]
include:
- torch: 1.8.1
torchvision: 0.9.1
steps:
- uses: actions/checkout@v2
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Upgrade pip
run: pip install pip --upgrade
- name: Install PyTorch
run: pip install torch==${{matrix.torch}}+cpu torchvision==${{matrix.torchvision}}+cpu -f https://download.pytorch.org/whl/torch_stable.html
- name: Install MMEngine
run: pip install git+https://github.com/open-mmlab/mmengine.git@main
- name: Install MMCV
run: |
pip install -U openmim
mim install 'mmcv >= 2.0.0rc1'
- name: Install other dependencies
run: pip install -r requirements/tests.txt
- name: Build and install
run: rm -rf .eggs && pip install -e .
- name: Run unittests and generate coverage report
run: |
coverage run --branch --source mmselfsup -m pytest tests/
coverage xml
coverage report -m
# Upload coverage report for python3.7 && pytorch1.8.1 cpu
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v1.0.14
with:
file: ./coverage.xml
flags: unittests
env_vars: OS,PYTHON
name: codecov-umbrella
fail_ci_if_error: false
build_cu102:
runs-on: ubuntu-18.04
container:
image: pytorch/pytorch:1.8.1-cuda10.2-cudnn7-devel
strategy:
matrix:
python-version: [3.7]
steps:
- uses: actions/checkout@v2
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Upgrade pip
run: pip install pip --upgrade
- name: Fetch GPG keys
run: |
apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/3bf863cc.pub
apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64/7fa2af80.pub
- name: Install Python-dev
run: apt-get update && apt-get install -y python${{matrix.python-version}}-dev
if: ${{matrix.python-version != 3.9}}
- name: Install system dependencies
run: |
apt-get update
apt-get install -y ffmpeg libsm6 libxext6 git ninja-build libglib2.0-0 libxrender-dev
- name: Install mmselfsup dependencies
run: |
pip install git+https://github.com/open-mmlab/mmengine.git@main
pip install -U openmim
mim install 'mmcv >= 2.0.0rc1'
pip install -r requirements/tests.txt
- name: Build and install
run: |
python setup.py check -m -s
TORCH_CUDA_ARCH_LIST=7.0 pip install -e .
build_windows:
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [windows-2022]
python: [3.7]
platform: [cpu, cu111]
steps:
- uses: actions/checkout@v2
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Upgrade pip
run: pip install pip --upgrade
- name: Install lmdb
run: pip install lmdb
- name: Install PyTorch
run: pip install torch==1.8.1+${{matrix.platform}} torchvision==0.9.1+${{matrix.platform}} -f https://download.pytorch.org/whl/lts/1.8/torch_lts.html
- name: Install mmselfsup dependencies
run: |
pip install git+https://github.com/open-mmlab/mmengine.git@main
pip install -U openmim
mim install 'mmcv >= 2.0.0rc1'
pip install -r requirements/tests.txt
- name: Build and install
run: |
pip install -e .
- name: Run unittests and generate coverage report
run: |
pytest tests/

6
tests/test_engine/test_hooks/test_densecl_hook.py
View File

@ -1,9 +1,11 @@
# Copyright (c) OpenMMLab. All rights reserved.
import logging
import tempfile
from unittest import TestCase
import torch
import torch.nn as nn
from mmengine.logging import MMLogger
from mmengine.model import BaseModule
from mmengine.optim import OptimWrapper
from mmengine.runner import Runner
@ -70,6 +72,10 @@ class TestDenseCLHook(TestCase):
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
# `FileHandler` should be closed in Windows, otherwise we cannot
# delete the temporary directory
logging.shutdown()
MMLogger._instance_dict.clear()
self.temp_dir.cleanup()
def test_densecl_hook(self):

6
tests/test_engine/test_hooks/test_simsiam_hook.py
View File

@ -1,9 +1,11 @@
# Copyright (c) OpenMMLab. All rights reserved.
import logging
import tempfile
from unittest import TestCase
import torch
import torch.nn as nn
from mmengine.logging import MMLogger
from mmengine.model import BaseModule
from mmengine.runner import Runner
from mmengine.structures import LabelData
@ -67,6 +69,10 @@ class TestSimSiamHook(TestCase):
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
# `FileHandler` should be closed in Windows, otherwise we cannot
# delete the temporary directory
logging.shutdown()
MMLogger._instance_dict.clear()
self.temp_dir.cleanup()
def test_simsiam_hook(self):

6
tests/test_engine/test_hooks/test_swav_hook.py
View File

@ -1,9 +1,11 @@
# Copyright (c) OpenMMLab. All rights reserved.
import logging
import tempfile
from unittest import TestCase
import torch
import torch.nn as nn
from mmengine.logging import MMLogger
from mmengine.model import BaseModule
from mmengine.optim import OptimWrapper
from mmengine.runner import Runner
@ -77,6 +79,10 @@ class TestSwAVHook(TestCase):
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
# `FileHandler` should be closed in Windows, otherwise we cannot
# delete the temporary directory
logging.shutdown()
MMLogger._instance_dict.clear()
self.temp_dir.cleanup()
def test_swav_hook(self):

2
tests/test_models/test_algorithms/test_deepcluster.py
View File

@ -54,7 +54,7 @@ def test_deepcluster():
'inputs':
[torch.randn((2, 3, 224, 224)),
torch.randn((2, 3, 224, 224))],
'data_sample': [SelfSupDataSample() for _ in range(2)]
'data_sample': [fake_data_sample for _ in range(2)]
}
fake_inputs, fake_data_samples = alg.data_preprocessor(fake_data)

2
tests/test_models/test_algorithms/test_odc.py
View File

@ -64,7 +64,7 @@ def test_odc():
fake_data_sample.sample_idx = fake_sample_idx
fake_data = {
'inputs': [torch.randn((2, 3, 224, 224))],
'data_samples': [SelfSupDataSample() for _ in range(2)]
'data_samples': [fake_data_sample for _ in range(2)]
}
fake_inputs, fake_data_samples = alg.data_preprocessor(fake_data)