diff --git a/.circleci/test.yml b/.circleci/test.yml index 560e344f4..67ce1b279 100644 --- a/.circleci/test.yml +++ b/.circleci/test.yml @@ -31,7 +31,7 @@ jobs: name: Check docstring coverage command: | pip install interrogate - interrogate -v --ignore-init-method --ignore-module --ignore-nested-functions --ignore-regex "__repr__" --fail-under 60 mmcls + interrogate -v --ignore-init-method --ignore-module --ignore-nested-functions --ignore-magic --ignore-regex "__repr__" --fail-under 60 mmcls build_cpu: parameters: # The python version must match available image tags in @@ -42,8 +42,6 @@ jobs: type: string torchvision: type: string - mmcv: - type: string docker: - image: cimg/python:<< parameters.python >> resource_class: large @@ -57,31 +55,32 @@ jobs: - run: name: Configure Python & pip command: | - python -m pip install --upgrade pip - python -m pip install wheel + pip install --upgrade pip + pip install wheel - run: name: Install PyTorch command: | python -V - python -m pip install torch==<< parameters.torch >>+cpu torchvision==<< parameters.torchvision >>+cpu -f https://download.pytorch.org/whl/torch_stable.html + pip install torch==<< parameters.torch >>+cpu torchvision==<< parameters.torchvision >>+cpu -f https://download.pytorch.org/whl/torch_stable.html - run: name: Install mmcls dependencies command: | - python -m pip install git+ssh://git@github.com/open-mmlab/mmengine.git@main - python -m pip install << parameters.mmcv >> - python -m pip install timm - python -m pip install -r requirements.txt + pip install git+https://github.com/open-mmlab/mmengine.git@main + pip install -U openmim + mim install 'mmcv >= 2.0.0rc1' + pip install timm + pip install -r requirements.txt python -c 'import mmcv; print(mmcv.__version__)' - run: name: Build and install command: | - python -m pip install -e . + pip install -e . - run: name: Run unittests command: | - python -m coverage run --branch --source mmcls -m pytest tests/ - python -m coverage xml - python -m coverage report -m + coverage run --branch --source mmcls -m pytest tests/ + coverage xml + coverage report -m build_cuda: machine: @@ -96,15 +95,13 @@ jobs: cudnn: type: integer default: 7 - mmcv: - type: string steps: - checkout - run: # Cloning repos in VM since Docker doesn't have access to the private key name: Clone Repos command: | - git clone -b main --depth 1 ssh://git@github.com/open-mmlab/mmengine.git /home/circleci/mmengine + git clone -b main --depth 1 https://github.com/open-mmlab/mmengine.git /home/circleci/mmengine - run: name: Build Docker image command: | @@ -114,7 +111,8 @@ jobs: name: Install mmcls dependencies command: | docker exec mmcls pip install -e /mmengine - docker exec mmcls pip install << parameters.mmcv >> + docker exec mmcls pip install -U openmim + docker exec mmcls mim install 'mmcv >= 2.0.0rc1' docker exec mmcls pip install -r requirements.txt docker exec mmcls python -c 'import mmcv; print(mmcv.__version__)' - run: @@ -124,7 +122,7 @@ jobs: - run: name: Run unittests command: | - docker exec mmcls python -m pytest tests/ --ignore tests/test_models/test_backbones/test_timm_backbone.py + docker exec mmcls python -m pytest tests/ -k 'not timm' # Invoke jobs via workflows # See: https://circleci.com/docs/2.0/configuration-reference/#workflows @@ -138,6 +136,7 @@ workflows: branches: ignore: - dev-1.x + - 1.x pr_stage_test: when: not: @@ -154,15 +153,13 @@ workflows: torch: 1.6.0 torchvision: 0.7.0 python: 3.6.9 # The lowest python 3.6.x version available on CircleCI images - mmcv: https://download.openmmlab.com/mmcv/dev-2.x/cpu/torch1.6.0/mmcv_full-2.0.0rc0-cp36-cp36m-manylinux1_x86_64.whl requires: - lint - build_cpu: name: maximum_version_cpu - torch: 1.9.0 # TODO: Update the version after mmcv provides more pre-compiled packages. - torchvision: 0.10.0 + torch: 1.12.1 + torchvision: 0.13.1 python: 3.9.0 - mmcv: https://download.openmmlab.com/mmcv/dev-2.x/cpu/torch1.9.0/mmcv_full-2.0.0rc0-cp39-cp39-manylinux1_x86_64.whl requires: - minimum_version_cpu - hold: @@ -175,7 +172,6 @@ workflows: # Use double quotation mark to explicitly specify its type # as string instead of number cuda: "10.2" - mmcv: https://download.openmmlab.com/mmcv/dev-2.x/cu102/torch1.8.0/mmcv_full-2.0.0rc0-cp37-cp37m-manylinux1_x86_64.whl requires: - hold merge_stage_test: @@ -188,7 +184,6 @@ workflows: torch: 1.6.0 # Use double quotation mark to explicitly specify its type # as string instead of number - mmcv: https://download.openmmlab.com/mmcv/dev-2.x/cu101/torch1.6.0/mmcv_full-2.0.0rc0-cp37-cp37m-manylinux1_x86_64.whl cuda: "10.1" filters: branches: diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 000000000..0c1cd14d1 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,27 @@ +name: lint + +on: [push, pull_request] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.7 + uses: actions/setup-python@v2 + with: + python-version: 3.7 + - name: Install pre-commit hook + run: | + pip install pre-commit + pre-commit install + - name: Linting + run: pre-commit run --all-files + - name: Check docstring coverage + run: | + pip install interrogate + interrogate -v --ignore-init-method --ignore-module --ignore-nested-functions --ignore-magic --ignore-regex "__repr__" --fail-under 60 mmcls diff --git a/.github/workflows/pr_stage_test.yml b/.github/workflows/pr_stage_test.yml new file mode 100644 index 000000000..67b154a60 --- /dev/null +++ b/.github/workflows/pr_stage_test.yml @@ -0,0 +1,87 @@ +name: pr_stage_test + +on: + pull_request: + paths-ignore: + - 'README.md' + - 'README_zh-CN.md' + - 'docs/**' + - 'demo/**' + - 'tools/**' + - 'configs/**' + - '.dev_scripts/**' + - '.circleci/**' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + runs-on: ubuntu-18.04 + strategy: + matrix: + python-version: [3.7] + include: + - torch: 1.8.1 + torchvision: 0.9.1 + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Upgrade pip + run: pip install pip --upgrade + - name: Install PyTorch + run: pip install torch==${{matrix.torch}}+cpu torchvision==${{matrix.torchvision}}+cpu -f https://download.pytorch.org/whl/torch_stable.html + - name: Install mmcls dependencies + run: | + pip install git+https://github.com/open-mmlab/mmengine.git@main + pip install -U openmim + mim install 'mmcv >= 2.0.0rc1' + pip install -r requirements.txt + - name: Build and install + run: pip install -e . + - name: Run unittests and generate coverage report + run: | + coverage run --branch --source mmcls -m pytest tests/ -k 'not timm' + coverage xml + coverage report -m + # Upload coverage report for python3.7 && pytorch1.8.1 cpu + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v1.0.14 + with: + file: ./coverage.xml + flags: unittests + env_vars: OS,PYTHON + name: codecov-umbrella + fail_ci_if_error: false + + build_windows: + runs-on: windows-2022 + strategy: + matrix: + python: [3.7] + platform: [cu111] + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Upgrade pip + run: pip install pip --upgrade + - name: Install PyTorch + run: pip install torch==1.8.2+${{matrix.platform}} torchvision==0.9.2+${{matrix.platform}} -f https://download.pytorch.org/whl/lts/1.8/torch_lts.html + - name: Install mmcls dependencies + run: | + pip install git+https://github.com/open-mmlab/mmengine.git@main + pip install -U openmim + mim install 'mmcv >= 2.0.0rc1' + pip install -r requirements.txt + - name: Build and install + run: pip install -e . + - name: Run unittests + run: | + pytest tests/ -k 'not timm' --ignore tests/test_models/test_backbones diff --git a/.github/workflows/publish-to-pypi.yml b/.github/workflows/publish-to-pypi.yml new file mode 100644 index 000000000..08936cb2c --- /dev/null +++ b/.github/workflows/publish-to-pypi.yml @@ -0,0 +1,22 @@ +name: deploy + +on: push + +jobs: + build-n-publish: + runs-on: ubuntu-latest + if: startsWith(github.event.ref, 'refs/tags') + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.7 + uses: actions/setup-python@v2 + with: + python-version: 3.7 + - name: Build MMClassification + run: | + pip install wheel + python setup.py sdist bdist_wheel + - name: Publish distribution to PyPI + run: | + pip install twine + twine upload dist/* -u __token__ -p ${{ secrets.pypi_password }} diff --git a/.github/workflows/test_mim.yml b/.github/workflows/test_mim.yml new file mode 100644 index 000000000..f437e4e52 --- /dev/null +++ b/.github/workflows/test_mim.yml @@ -0,0 +1,44 @@ +name: test-mim + +on: + push: + paths: + - 'model-index.yml' + - 'configs/**' + + pull_request: + paths: + - 'model-index.yml' + - 'configs/**' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build_cpu: + runs-on: ubuntu-18.04 + strategy: + matrix: + python-version: [3.7] + torch: [1.8.0] + include: + - torch: 1.8.0 + torch_version: torch1.8 + torchvision: 0.9.0 + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Upgrade pip + run: pip install pip --upgrade + - name: Install PyTorch + run: pip install torch==${{matrix.torch}}+cpu torchvision==${{matrix.torchvision}}+cpu -f https://download.pytorch.org/whl/torch_stable.html + - name: Install openmim + run: pip install openmim + - name: Build and install + run: mim install -e . + - name: test commands of mim + run: mim search mmcls diff --git a/mmcls/datasets/transforms/processing.py b/mmcls/datasets/transforms/processing.py index dc1db1827..20b7b0b4b 100644 --- a/mmcls/datasets/transforms/processing.py +++ b/mmcls/datasets/transforms/processing.py @@ -504,7 +504,8 @@ class RandomErasing(BaseTransform): 'aspect_range should be positive.' assert aspect_range[0] <= aspect_range[1], \ 'In aspect_range (min, max), min should be smaller than max.' - assert mode in ['const', 'rand'] + assert mode in ['const', 'rand'], \ + 'Please select `mode` from ["const", "rand"].' if isinstance(fill_color, Number): fill_color = [fill_color] * 3 assert isinstance(fill_color, Sequence) and len(fill_color) == 3 \ diff --git a/tests/test_datasets/test_datasets.py b/tests/test_datasets/test_datasets.py index 18f1cb76c..f637fb958 100644 --- a/tests/test_datasets/test_datasets.py +++ b/tests/test_datasets/test_datasets.py @@ -2,6 +2,7 @@ import os import os.path as osp import pickle +import sys import tempfile from unittest import TestCase from unittest.mock import MagicMock, call, patch @@ -141,12 +142,12 @@ class TestCustomDataset(TestBaseDataset): self.assertEqual(dataset.CLASSES, ('a', 'b')) # auto infer classes self.assertGreaterEqual( dataset.get_data_info(0).items(), { - 'img_path': osp.join(ASSETS_ROOT, 'a/1.JPG'), + 'img_path': osp.join(ASSETS_ROOT, 'a', '1.JPG'), 'gt_label': 0 }.items()) self.assertGreaterEqual( dataset.get_data_info(2).items(), { - 'img_path': osp.join(ASSETS_ROOT, 'b/subb/3.jpg'), + 'img_path': osp.join(ASSETS_ROOT, 'b', 'subb', '3.jpg'), 'gt_label': 1 }.items()) @@ -225,7 +226,7 @@ class TestCustomDataset(TestBaseDataset): self.assertEqual(len(dataset), 1) self.assertGreaterEqual( dataset.get_data_info(0).items(), { - 'img_path': osp.join(ASSETS_ROOT, 'b/2.jpeg'), + 'img_path': osp.join(ASSETS_ROOT, 'b', '2.jpeg'), 'gt_label': 1 }.items()) @@ -631,12 +632,12 @@ class TestVOC(TestBaseDataset): # Test different backend cfg = { **self.DEFAULT_ARGS, 'lazy_init': True, - 'data_root': 's3:/openmmlab/voc' + 'data_root': 's3://openmmlab/voc' } + petrel_mock = MagicMock() + sys.modules['petrel_client'] = petrel_mock dataset = dataset_class(**cfg) - dataset._check_integrity = MagicMock(return_value=False) - with self.assertRaisesRegex(FileNotFoundError, 's3:/openmmlab/voc'): - dataset.full_init() + petrel_mock.client.Client.assert_called() def test_extra_repr(self): dataset_class = DATASETS.get(self.DATASET_TYPE) diff --git a/tests/test_engine/test_hooks/test_precise_bn_hook.py b/tests/test_engine/test_hooks/test_precise_bn_hook.py index e79df9d99..661714cc1 100644 --- a/tests/test_engine/test_hooks/test_precise_bn_hook.py +++ b/tests/test_engine/test_hooks/test_precise_bn_hook.py @@ -1,6 +1,6 @@ # Copyright (c) OpenMMLab. All rights reserved. import copy -import shutil +import logging import tempfile from unittest import TestCase from unittest.mock import MagicMock, patch @@ -8,6 +8,7 @@ from unittest.mock import MagicMock, patch import pytest import torch import torch.nn as nn +from mmengine.logging import MMLogger from mmengine.model import BaseDataPreprocessor, BaseModel from mmengine.runner import Runner from torch.utils.data import DataLoader, Dataset @@ -115,7 +116,7 @@ class TestPreciseBNHookHook(TestCase): ) self.epoch_train_cfg = dict(by_epoch=True, max_epochs=1) self.iter_train_cfg = dict(by_epoch=False, max_iters=5) - self.tmpdir = tempfile.mkdtemp() + self.tmpdir = tempfile.TemporaryDirectory() self.preciseBN_cfg = copy.deepcopy(self.DEFAULT_ARGS) test_dataset = ExampleDataset() @@ -125,7 +126,7 @@ class TestPreciseBNHookHook(TestCase): def test_construct(self): self.runner = Runner( model=self.model, - work_dir=self.tmpdir, + work_dir=self.tmpdir.name, train_dataloader=self.loader, train_cfg=self.epoch_train_cfg, log_level='WARNING', @@ -160,7 +161,7 @@ class TestPreciseBNHookHook(TestCase): self.preciseBN_cfg['priority'] = 'ABOVE_NORMAL' self.runner = Runner( model=self.model, - work_dir=self.tmpdir, + work_dir=self.tmpdir.name, train_dataloader=self.loader, train_cfg=self.epoch_train_cfg, log_level='WARNING', @@ -176,7 +177,7 @@ class TestPreciseBNHookHook(TestCase): self.preciseBN_cfg['priority'] = 'ABOVE_NORMAL' self.runner = Runner( model=self.model, - work_dir=self.tmpdir, + work_dir=self.tmpdir.name, train_dataloader=self.loader, train_cfg=self.epoch_train_cfg, log_level='WARNING', @@ -213,7 +214,7 @@ class TestPreciseBNHookHook(TestCase): self.loader = DataLoader(test_dataset, batch_size=2) self.runner = Runner( model=self.model, - work_dir=self.tmpdir, + work_dir=self.tmpdir.name, train_dataloader=self.loader, train_cfg=self.iter_train_cfg, log_level='WARNING', @@ -226,4 +227,8 @@ class TestPreciseBNHookHook(TestCase): self.runner.train() def tearDown(self) -> None: - shutil.rmtree(self.tmpdir) + # `FileHandler` should be closed in Windows, otherwise we cannot + # delete the temporary directory. + logging.shutdown() + MMLogger._instance_dict.clear() + self.tmpdir.cleanup() diff --git a/tests/test_models/test_backbones/test_conformer.py b/tests/test_models/test_backbones/test_conformer.py index 96a5a2cce..0b1958c58 100644 --- a/tests/test_models/test_backbones/test_conformer.py +++ b/tests/test_models/test_backbones/test_conformer.py @@ -25,6 +25,7 @@ def check_norm_state(modules, train_state): return True +@torch.no_grad() # To save memory def test_conformer_backbone(): cfg_ori = dict( diff --git a/tests/test_models/test_backbones/test_convmixer.py b/tests/test_models/test_backbones/test_convmixer.py index 7d2219e29..262966153 100644 --- a/tests/test_models/test_backbones/test_convmixer.py +++ b/tests/test_models/test_backbones/test_convmixer.py @@ -18,6 +18,7 @@ def test_assertion(): ConvMixer(out_indices=-100) +@torch.no_grad() # To save memory def test_convmixer(): # Test forward