mirrors
/
mmrazor
mirror of https://github.com/open-mmlab/mmrazor.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

[CI] Add circle ci () 

* copy .circleci from mmdet

* adapt mmrazor

* change the min docstring coverage

* fix typos

* update publish model script

* update circle ci config
pull/263/head
pppppM 2022-08-30 20:20:10 +08:00 committed by GitHub
parent c650b3eee5
commit 8a249fd98d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 291 additions and 42 deletions
tools/model_converters

34
.circleci/config.yml 100644
View File

@ -0,0 +1,34 @@
version: 2.1
# this allows you to use CircleCI's dynamic configuration feature
setup: true
# the path-filtering orb is required to continue a pipeline based on
# the path of an updated fileset
orbs:
path-filtering: circleci/path-filtering@0.1.2
workflows:
# the always-run workflow is always triggered, regardless of the pipeline parameters.
always-run:
jobs:
# the path-filtering/filter job determines which pipeline
# parameters to update.
- path-filtering/filter:
name: check-updated-files
# 3-column, whitespace-delimited mapping. One mapping per
# line:
# <regex path-to-test> <parameter-to-set> <value-of-pipeline-parameter>
mapping: |
mmrazor/.* lint_only false
requirements/.* lint_only false
tests/.* lint_only false
tools/.* lint_only false
configs/.* lint_only false
.circleci/.* lint_only false
base-revision: dev-1.x
# this is the path of the configuration we should trigger once
# path filtering and pipeline parameter value updates are
# complete. In this case, we are using the parent dynamic
# configuration itself.
config-path: .circleci/test.yml

11
.circleci/docker/Dockerfile 100644
View File

@ -0,0 +1,11 @@
ARG PYTORCH="1.8.1"
ARG CUDA="10.2"
ARG CUDNN="7"
FROM pytorch/pytorch:${PYTORCH}-cuda${CUDA}-cudnn${CUDNN}-devel
# To fix GPG key error when running apt-get update
RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/3bf863cc.pub
RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64/7fa2af80.pub
RUN apt-get update && apt-get install -y ninja-build libglib2.0-0 libsm6 libxrender-dev libxext6 libgl1-mesa-glx

203
.circleci/test.yml 100644
View File

@ -0,0 +1,203 @@
version: 2.1
# the default pipeline parameters, which will be updated according to
# the results of the path-filtering orb
parameters:
lint_only:
type: boolean
default: true
jobs:
lint:
docker:
- image: cimg/python:3.7.4
steps:
- checkout
- run:
name: Install pre-commit hook
command: |
pip install pre-commit
pre-commit install
- run:
name: Linting
command: pre-commit run --all-files
- run:
name: Check docstring coverage
command: |
pip install interrogate
interrogate -v --ignore-init-method --ignore-module --ignore-nested-functions --ignore-magic --ignore-regex "__repr__" --fail-under 80 mmrazor
build_cpu:
parameters:
# The python version must match available image tags in
# https://circleci.com/developer/images/image/cimg/python
python:
type: string
torch:
type: string
torchvision:
type: string
mmcv:
type: string
docker:
- image: cimg/python:<< parameters.python >>
resource_class: large
steps:
- checkout
- run:
name: Install Libraries
command: |
sudo apt-get update
sudo apt-get install -y ninja-build libglib2.0-0 libsm6 libxrender-dev libxext6 libgl1-mesa-glx libjpeg-dev zlib1g-dev libtinfo-dev libncurses5
- run:
name: Configure Python & pip
command: |
pip install --upgrade pip
pip install wheel
- run:
name: Install PyTorch
command: |
python -V
python -m pip install torch==<< parameters.torch >>+cpu torchvision==<< parameters.torchvision >>+cpu -f https://download.pytorch.org/whl/torch_stable.html
- when:
condition:
equal: [ "3.9.0", << parameters.python >> ]
steps:
- run: pip install "protobuf <= 3.20.1" && sudo apt-get update && sudo apt-get -y install libprotobuf-dev protobuf-compiler cmake
- run:
name: Install mmrazor dependencies
command: |
python -m pip install git+ssh://git@github.com/open-mmlab/mmengine.git@main
python -m pip install << parameters.mmcv >>
python -m pip install git+ssh://git@github.com/open-mmlab/mmclassification.git@dev-1.x
python -m pip install git+ssh://git@github.com/open-mmlab/mmdetection.git@dev-3.x
python -m pip install git+ssh://git@github.com/open-mmlab/mmsegmentation.git@dev-1.x
pip install -r requirements.txt
- run:
name: Build and install
command: |
pip install -e .
- run:
name: Run unittests
command: |
python -m coverage run --branch --source mmrazor -m pytest tests/
python -m coverage xml
python -m coverage report -m
build_cuda:
parameters:
torch:
type: string
cuda:
type: enum
enum: ["10.1", "10.2", "11.1"]
cudnn:
type: integer
default: 7
mmcv:
type: string
machine:
image: ubuntu-2004-cuda-11.4:202110-01
# docker_layer_caching: true
resource_class: gpu.nvidia.small
steps:
- checkout
- run:
# CLoning repos in VM since Docker doesn't have access to the private key
name: Clone Repos
command: |
git clone -b main --depth 1 ssh://git@github.com/open-mmlab/mmengine.git /home/circleci/mmengine
git clone -b dev-3.x --depth 1 ssh://git@github.com/open-mmlab/mmdetection.git /home/circleci/mmdetection
git clone -b dev-1.x --depth 1 ssh://git@github.com/open-mmlab/mmclassification.git /home/circleci/mmclassification
git clone -b dev-1.x --depth 1 ssh://git@github.com/open-mmlab/mmsegmentation.git /home/circleci/mmsegmentation
- run:
name: Build Docker image
command: |
docker build .circleci/docker -t mmrazor:gpu --build-arg PYTORCH=<< parameters.torch >> --build-arg CUDA=<< parameters.cuda >> --build-arg CUDNN=<< parameters.cudnn >>
docker run --gpus all -t -d -v /home/circleci/project:/mmrazor -v /home/circleci/mmengine:/mmengine -v /home/circleci/mmdetection:/mmdetection -v /home/circleci/mmclassification:/mmclassification -v /home/circleci/mmsegmentation:/mmsegmentation -w /mmrazor --name mmrazor mmrazor:gpu
- run:
name: Install mmrazor dependencies
# pip install mmcv-full -f https://download.openmmlab.com/mmcv/dist/cu101/torch${{matrix.torch_version}}/index.html
command: |
docker exec mmrazor pip install -e /mmengine
docker exec mmrazor pip install << parameters.mmcv >>
docker exec mmrazor pip install -e /mmdetection
docker exec mmrazor pip install -e /mmclassification
docker exec mmrazor pip install -e /mmsegmentation
pip install -r requirements.txt
python -c 'import mmcv; print(mmcv.__version__)'
- run:
name: Build and install
command: |
docker exec mmrazor pip install -e .
- run:
name: Run unittests
command: |
docker exec mmrazor python -m pytest tests/
workflows:
pr_stage_lint:
when: << pipeline.parameters.lint_only >>
jobs:
- lint:
name: lint
filters:
branches:
ignore:
- dev-1.x
pr_stage_test:
when:
not:
<< pipeline.parameters.lint_only >>
jobs:
- lint:
name: lint
filters:
branches:
ignore:
- dev-1.x
- build_cpu:
name: minimum_version_cpu
torch: 1.6.0
torchvision: 0.7.0
python: 3.6.9 # The lowest python 3.6.x version available on CircleCI images
mmcv: https://download.openmmlab.com/mmcv/dev-2.x/cpu/torch1.6.0/mmcv_full-2.0.0rc0-cp36-cp36m-manylinux1_x86_64.whl
requires:
- lint
- build_cpu:
name: maximum_version_cpu
torch: 1.9.0
torchvision: 0.10.0
python: 3.9.0
mmcv: https://download.openmmlab.com/mmcv/dev-2.x/cpu/torch1.9.0/mmcv_full-2.0.0rc0-cp39-cp39-manylinux1_x86_64.whl
requires:
- minimum_version_cpu
- hold:
type: approval
requires:
- maximum_version_cpu
- build_cuda:
name: mainstream_version_gpu
torch: 1.8.1
# Use double quotation mark to explicitly specify its type
# as string instead of number
cuda: "10.2"
mmcv: https://download.openmmlab.com/mmcv/dev-2.x/cu102/torch1.8.0/mmcv_full-2.0.0rc0-cp37-cp37m-manylinux1_x86_64.whl
requires:
- hold
merge_stage_test:
when:
not:
<< pipeline.parameters.lint_only >>
jobs:
- build_cuda:
name: minimum_version_gpu
torch: 1.6.0
# Use double quotation mark to explicitly specify its type
# as string instead of number
mmcv: https://download.openmmlab.com/mmcv/dev-2.x/cu101/torch1.6.0/mmcv_full-2.0.0rc0-cp37-cp37m-manylinux1_x86_64.whl
cuda: "10.1"
filters:
branches:
only:
- dev-1.x

85
tools/model_converters/publish_model.py
View File

@ -1,10 +1,9 @@
# Copyright (c) OpenMMLab. All rights reserved.
import argparse
import datetime
import shutil
from pathlib import Path
from typing import Optional
from typing import Union
import mmengine
import torch
from mmengine import digit_version
@ -12,19 +11,15 @@ from mmengine import digit_version
def parse_args():
parser = argparse.ArgumentParser(
description='Process a checkpoint to be published')
parser.add_argument('in_file', help='input checkpoint filename', type=str)
parser.add_argument(
'out_file', help='output checkpoint filename', default=None, type=str)
parser.add_argument(
'subnet_cfg_file',
help='input subnet config filename',
default=None,
type=str)
parser.add_argument('ckpt', help='input checkpoint filename', type=str)
parser.add_argument('--model-name', help='model(config) name', type=str)
parser.add_argument('--timestamp', help='training timestamp', type=str)
parser.add_argument('--out-dir', help='output dir', type=str)
args = parser.parse_args()
return args
def cal_file_sha256(file_path: str) -> str:
def cal_file_sha256(file_path: Union[str, Path]) -> str:
import hashlib
BLOCKSIZE = 65536
@ -39,51 +34,57 @@ def cal_file_sha256(file_path: str) -> str:
return sha256_hash.hexdigest()
def process_checkpoint(in_file: str,
out_file: Optional[str] = None,
subnet_cfg_file: Optional[str] = None) -> None:
checkpoint = torch.load(in_file, map_location='cpu')
def process_checkpoint(ckpt_path_str: str, model_name: str, timestamp: str,
out_dir_str: str) -> None:
ckpt_path = Path(ckpt_path_str)
work_dir = ckpt_path.parent
out_dir: Path = Path(out_dir_str)
out_dir.mkdir(parents=True, exist_ok=True)
tmp_ckpt_path = out_dir / 'tmp.pth'
checkpoint = torch.load(ckpt_path, map_location='cpu')
# remove optimizer for smaller file size
if 'optimizer' in checkpoint:
del checkpoint['optimizer']
# remove message_hub for smaller file size
if 'message_hub' in checkpoint:
del checkpoint['message_hub']
# remove param_schedulers for smaller file size
if 'param_schedulers' in checkpoint:
del checkpoint['param_schedulers']
if out_file is None:
out_file = in_file
# if it is necessary to remove some sensitive data in checkpoint['meta'],
# add the code here.
if digit_version(torch.__version__) >= digit_version('1.6'):
torch.save(checkpoint, out_file, _use_new_zipfile_serialization=False)
torch.save(
checkpoint, tmp_ckpt_path, _use_new_zipfile_serialization=False)
else:
torch.save(checkpoint, out_file)
torch.save(checkpoint, tmp_ckpt_path)
sha = cal_file_sha256(out_file)
if out_file.endswith('.pth'):
out_file_name = out_file[:-4]
else:
out_file_name = out_file
sha = cal_file_sha256(tmp_ckpt_path)
save_ckpt_path = f'{out_dir}/{model_name}_{timestamp}-{sha[:8]}.pth'
tmp_ckpt_path.rename(save_ckpt_path)
print(f'Successfully generated the publish-ckpt as {save_ckpt_path}.')
current_date = datetime.datetime.now().strftime('%Y%m%d')
final_file_prefix = out_file_name + f'_{current_date}-{sha[:8]}'
final_ckpt_file = f'{final_file_prefix}.pth'
Path(out_file).rename(final_ckpt_file)
log_path = work_dir / timestamp / f'{timestamp}.log'
save_log_path = f'{out_dir}/{model_name}_{timestamp}-{sha[:8]}.log'
shutil.copy(str(log_path), str(save_log_path))
print(f'Successfully generated the publish-log as {save_log_path}.')
print(f'Successfully generated the publish-ckpt as {final_ckpt_file}.')
if subnet_cfg_file is not None:
subnet_cfg = mmengine.fileio.load(subnet_cfg_file)
final_subnet_cfg_file = f'{final_file_prefix}_subnet_cfg.yaml'
mmengine.fileio.dump(subnet_cfg, final_subnet_cfg_file)
print(f'Successfully generated the publish-subnet-cfg as \
{final_subnet_cfg_file}.')
log_path = work_dir / timestamp / f'{timestamp}.log'
json_path = work_dir / timestamp / f'vis_data/{timestamp}.json'
save_json_path = f'{out_dir}/{model_name}_{timestamp}-{sha[:8]}.json'
shutil.copy(str(json_path), str(save_json_path))
print(f'Successfully generated the publish-log as {save_json_path}.')
def main():
args = parse_args()
out_dir = Path(args.out_file).parent
if not out_dir.exists():
raise ValueError(f'Directory {out_dir} does not exist, '
'please generate it manually.')
process_checkpoint(args.in_file, args.out_file, args.subnet_cfg_file)
process_checkpoint(args.ckpt, args.model_name, args.timestamp,
args.out_dir)
if __name__ == '__main__':