[Refactor] Refactor fileio (#2543)

## Motivation

Use the new fileio from mmengine
https://github.com/open-mmlab/mmengine/pull/533

## Modification

1. Use `mmengine.fileio` to repalce FileClient  in mmseg/datasets
2. Use `mmengine.fileio` to repalce FileClient in
mmseg/datasets/transforms
3. Use `mmengine.fileio` to repalce FileClient in mmseg/visualization

## BC-breaking (Optional)

we modify all the dataset configurations, so please use the latest config file.
pull/2532/head^2
谢昕辰 2023-02-01 17:53:22 +08:00 committed by GitHub
parent 698dee9525
commit 124b87ce90
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
30 changed files with 134 additions and 150 deletions

View File

@ -61,7 +61,7 @@ jobs:
command: |
pip install git+https://github.com/open-mmlab/mmengine.git@main
pip install -U openmim
mim install 'mmcv==2.0.0rc3'
mim install 'mmcv>=2.0.0rc4'
pip install git+https://github.com/open-mmlab/mmclassification@dev-1.x
mim install 'mmdet==3.0.0rc5'
pip install -r requirements/tests.txt -r requirements/optional.txt
@ -107,7 +107,7 @@ jobs:
command: |
docker exec mmseg pip install -e /mmengine
docker exec mmseg pip install -U openmim
docker exec mmseg mim install 'mmcv==2.0.0rc3'
docker exec mmseg mim install 'mmcv>=2.0.0rc4'
docker exec mmseg pip install -e /mmclassification
docker exec mmseg mim install 'mmdet==3.0.0rc5'
docker exec mmseg pip install -r requirements/tests.txt -r requirements/optional.txt

View File

@ -44,7 +44,7 @@ jobs:
python -V
pip install -U openmim
pip install git+https://github.com/open-mmlab/mmengine.git
mim install 'mmcv==2.0.0rc3'
mim install 'mmcv>=2.0.0rc4'
pip install git+https://github.com/open-mmlab/mmclassification.git@dev-1.x
mim install 'mmdet==3.0.0rc5'
- name: Install unittest dependencies
@ -100,7 +100,7 @@ jobs:
python -V
pip install -U openmim
pip install git+https://github.com/open-mmlab/mmengine.git
mim install 'mmcv==2.0.0rc3'
mim install 'mmcv>=2.0.0rc4'
pip install git+https://github.com/open-mmlab/mmclassification.git@dev-1.x
mim install 'mmdet==3.0.0rc5'
- name: Install unittest dependencies
@ -166,7 +166,7 @@ jobs:
python -V
pip install -U openmim
pip install git+https://github.com/open-mmlab/mmengine.git
mim install 'mmcv==2.0.0rc3'
mim install 'mmcv>=2.0.0rc4'
pip install git+https://github.com/open-mmlab/mmclassification.git@dev-1.x
mim install 'mmdet==3.0.0rc5'
- name: Install unittest dependencies
@ -209,7 +209,7 @@ jobs:
python -V
pip install -U openmim
pip install git+https://github.com/open-mmlab/mmengine.git
mim install 'mmcv==2.0.0rc3'
mim install 'mmcv>=2.0.0rc4'
pip install git+https://github.com/open-mmlab/mmclassification.git@dev-1.x
mim install 'mmdet==3.0.0rc5'
- name: Install unittest dependencies
@ -244,7 +244,7 @@ jobs:
python -V
pip install -U openmim
pip install git+https://github.com/open-mmlab/mmengine.git
mim install 'mmcv==2.0.0rc3'
mim install 'mmcv>=2.0.0rc4'
pip install git+https://github.com/open-mmlab/mmclassification.git@dev-1.x
mim install 'mmdet==3.0.0rc5'
- name: Install unittest dependencies

View File

@ -44,7 +44,7 @@ jobs:
run: |
pip install -U openmim
pip install git+https://github.com/open-mmlab/mmengine.git
mim install 'mmcv==2.0.0rc3'
mim install 'mmcv>=2.0.0rc4'
pip install git+https://github.com/open-mmlab/mmclassification.git@dev-1.x
mim install 'mmdet==3.0.0rc5'
- name: Install unittest dependencies
@ -100,7 +100,7 @@ jobs:
python -V
pip install -U openmim
pip install git+https://github.com/open-mmlab/mmengine.git
mim install 'mmcv==2.0.0rc3'
mim install 'mmcv>=2.0.0rc4'
pip install git+https://github.com/open-mmlab/mmclassification.git@dev-1.x
mim install 'mmdet==3.0.0rc5'
- name: Install unittest dependencies
@ -135,7 +135,7 @@ jobs:
python -V
pip install -U openmim
pip install git+https://github.com/open-mmlab/mmengine.git
mim install 'mmcv==2.0.0rc3'
mim install 'mmcv>=2.0.0rc4'
pip install git+https://github.com/open-mmlab/mmclassification.git@dev-1.x
mim install 'mmdet==3.0.0rc5'
- name: Install unittest dependencies

View File

@ -25,7 +25,7 @@ test_pipeline = [
]
img_ratios = [0.5, 0.75, 1.0, 1.25, 1.5, 1.75]
tta_pipeline = [
dict(type='LoadImageFromFile', file_client_args=dict(backend='disk')),
dict(type='LoadImageFromFile', backend_args=dict(backend='local')),
dict(
type='TestTimeAug',
transforms=[

View File

@ -25,7 +25,7 @@ test_pipeline = [
]
img_ratios = [0.5, 0.75, 1.0, 1.25, 1.5, 1.75]
tta_pipeline = [
dict(type='LoadImageFromFile', file_client_args=dict(backend='disk')),
dict(type='LoadImageFromFile', backend_args=dict(backend='local')),
dict(
type='TestTimeAug',
transforms=[

View File

@ -26,7 +26,7 @@ test_pipeline = [
]
img_ratios = [0.5, 0.75, 1.0, 1.25, 1.5, 1.75]
tta_pipeline = [
dict(type='LoadImageFromFile', file_client_args=dict(backend='disk')),
dict(type='LoadImageFromFile', backend_args=dict(backend='local')),
dict(
type='TestTimeAug',
transforms=[

View File

@ -25,7 +25,7 @@ test_pipeline = [
]
img_ratios = [0.5, 0.75, 1.0, 1.25, 1.5, 1.75]
tta_pipeline = [
dict(type='LoadImageFromFile', file_client_args=dict(backend='disk')),
dict(type='LoadImageFromFile', backend_args=dict(backend='local')),
dict(
type='TestTimeAug',
transforms=[

View File

@ -25,7 +25,7 @@ test_pipeline = [
]
img_ratios = [0.5, 0.75, 1.0, 1.25, 1.5, 1.75]
tta_pipeline = [
dict(type='LoadImageFromFile', file_client_args=dict(backend='disk')),
dict(type='LoadImageFromFile', backend_args=dict(backend='local')),
dict(
type='TestTimeAug',
transforms=[

View File

@ -25,7 +25,7 @@ test_pipeline = [
]
img_ratios = [0.5, 0.75, 1.0, 1.25, 1.5, 1.75]
tta_pipeline = [
dict(type='LoadImageFromFile', file_client_args=dict(backend='disk')),
dict(type='LoadImageFromFile', backend_args=dict(backend='local')),
dict(
type='TestTimeAug',
transforms=[

View File

@ -26,7 +26,7 @@ test_pipeline = [
]
img_ratios = [0.5, 0.75, 1.0, 1.25, 1.5, 1.75]
tta_pipeline = [
dict(type='LoadImageFromFile', file_client_args=dict(backend='disk')),
dict(type='LoadImageFromFile', backend_args=dict(backend='local')),
dict(
type='TestTimeAug',
transforms=[

View File

@ -26,7 +26,7 @@ test_pipeline = [
]
img_ratios = [0.5, 0.75, 1.0, 1.25, 1.5, 1.75]
tta_pipeline = [
dict(type='LoadImageFromFile', file_client_args=dict(backend='disk')),
dict(type='LoadImageFromFile', backend_args=dict(backend='local')),
dict(
type='TestTimeAug',
transforms=[

View File

@ -32,7 +32,7 @@ test_pipeline = [
]
img_ratios = [0.5, 0.75, 1.0, 1.25, 1.5, 1.75]
tta_pipeline = [
dict(type='LoadImageFromFile', file_client_args=dict(backend='disk')),
dict(type='LoadImageFromFile', backend_args=dict(backend='local')),
dict(
type='TestTimeAug',
transforms=[

View File

@ -25,7 +25,7 @@ test_pipeline = [
]
img_ratios = [0.5, 0.75, 1.0, 1.25, 1.5, 1.75]
tta_pipeline = [
dict(type='LoadImageFromFile', file_client_args=dict(backend='disk')),
dict(type='LoadImageFromFile', backend_args=dict(backend='local')),
dict(
type='TestTimeAug',
transforms=[

View File

@ -28,7 +28,7 @@ test_pipeline = [
]
img_ratios = [0.5, 0.75, 1.0, 1.25, 1.5, 1.75]
tta_pipeline = [
dict(type='LoadImageFromFile', file_client_args=dict(backend='disk')),
dict(type='LoadImageFromFile', backend_args=dict(backend='local')),
dict(
type='TestTimeAug',
transforms=[

View File

@ -25,7 +25,7 @@ test_pipeline = [
]
img_ratios = [0.5, 0.75, 1.0, 1.25, 1.5, 1.75]
tta_pipeline = [
dict(type='LoadImageFromFile', file_client_args=dict(backend='disk')),
dict(type='LoadImageFromFile', backend_args=dict(backend='local')),
dict(
type='TestTimeAug',
transforms=[

View File

@ -27,7 +27,7 @@ test_pipeline = [
]
img_ratios = [0.5, 0.75, 1.0, 1.25, 1.5, 1.75]
tta_pipeline = [
dict(type='LoadImageFromFile', file_client_args=dict(backend='disk')),
dict(type='LoadImageFromFile', backend_args=dict(backend='local')),
dict(
type='TestTimeAug',
transforms=[

View File

@ -25,7 +25,7 @@ test_pipeline = [
]
img_ratios = [0.5, 0.75, 1.0, 1.25, 1.5, 1.75]
tta_pipeline = [
dict(type='LoadImageFromFile', file_client_args=dict(backend='disk')),
dict(type='LoadImageFromFile', backend_args=dict(backend='local')),
dict(
type='TestTimeAug',
transforms=[

View File

@ -26,7 +26,7 @@ test_pipeline = [
]
img_ratios = [0.5, 0.75, 1.0, 1.25, 1.5, 1.75]
tta_pipeline = [
dict(type='LoadImageFromFile', file_client_args=dict(backend='disk')),
dict(type='LoadImageFromFile', backend_args=dict(backend='local')),
dict(
type='TestTimeAug',
transforms=[

View File

@ -25,7 +25,7 @@ test_pipeline = [
]
img_ratios = [0.5, 0.75, 1.0, 1.25, 1.5, 1.75]
tta_pipeline = [
dict(type='LoadImageFromFile', file_client_args=dict(backend='disk')),
dict(type='LoadImageFromFile', backend_args=dict(backend='local')),
dict(
type='TestTimeAug',
transforms=[

View File

@ -237,7 +237,7 @@ test_pipeline = [
]
img_ratios = [0.5, 0.75, 1.0, 1.25, 1.5, 1.75]
tta_pipeline = [
dict(type='LoadImageFromFile', file_client_args=dict(backend='disk')),
dict(type='LoadImageFromFile', backend_args=dict(backend='local')),
dict(
type='TestTimeAug',
transforms=[

View File

@ -4,37 +4,19 @@ We list some common troubles faced by many users and their corresponding solutio
## Installation
The compatible MMSegmentation and MMCV versions are as below. Please install the correct version of MMCV to avoid installation issues.
The compatible MMSegmentation, MMCV and MMEngine versions are as below. Please install the correct versions of them to avoid installation issues.
| MMSegmentation version | MMCV version | MMClassification (optional) version | MMDetection (optional) version |
| :--------------------: | :----------------------------: | :---------------------------------: | :----------------------------: |
| 1.x/dev-1.x branch | mmcv == 2.0.0rc3 | mmcls>=1.0.0rc0 | mmdet>=3.0.0rc4, \<=3.0.0rc5> |
| 1.0.0rc4 | mmcv == 2.0.0rc3 | mmcls>=1.0.0rc0 | mmdet>=3.0.0rc4, \<=3.0.0rc5> |
| 1.0.0rc3 | mmcv == 2.0.0rc3 | mmcls>=1.0.0rc0 | mmdet>=3.0.0rc4 \<=3.0.0rc5> |
| 1.0.0rc2 | mmcv == 2.0.0rc3 | mmcls>=1.0.0rc0 | mmdet>=3.0.0rc4 \<=3.0.0rc5> |
| 1.0.0rc1 | mmcv >= 2.0.0rc1, \<=2.0.0rc3> | mmcls>=1.0.0rc0 | Not required |
| 1.0.0rc0 | mmcv >= 2.0.0rc1, \<=2.0.0rc3> | mmcls>=1.0.0rc0 | Not required |
| master | mmcv-full>=1.4.4, \<=1.6.0 | mmcls>=0.20.1, \<=1.0.0 | Not required |
| 0.24.1 | mmcv-full>=1.4.4, \<=1.6.0 | mmcls>=0.20.1, \<=1.0.0 | Not required |
| 0.23.0 | mmcv-full>=1.4.4, \<=1.6.0 | mmcls>=0.20.1, \<=1.0.0 | Not required |
| 0.22.0 | mmcv-full>=1.4.4, \<=1.6.0 | mmcls>=0.20.1, \<=1.0.0 | Not required |
| 0.21.1 | mmcv-full>=1.4.4, \<=1.6.0 | Not required | Not required |
| 0.20.2 | mmcv-full>=1.3.13, \<=1.6.0 | Not required | Not required |
| 0.19.0 | mmcv-full>=1.3.13, \<1.3.17 | Not required | Not required |
| 0.18.0 | mmcv-full>=1.3.13, \<1.3.17 | Not required | Not required |
| 0.17.0 | mmcv-full>=1.3.7, \<1.3.17 | Not required | Not required |
| 0.16.0 | mmcv-full>=1.3.7, \<1.3.17 | Not required | Not required |
| 0.15.0 | mmcv-full>=1.3.7, \<1.3.17 | Not required | Not required |
| 0.14.1 | mmcv-full>=1.3.7, \<1.3.17 | Not required | Not required |
| 0.14.0 | mmcv-full>=1.3.1, \<1.3.2 | Not required | Not required |
| 0.13.0 | mmcv-full>=1.3.1, \<1.3.2 | Not required | Not required |
| 0.12.0 | mmcv-full>=1.1.4, \<1.3.2 | Not required | Not required |
| 0.11.0 | mmcv-full>=1.1.4, \<1.3.0 | Not required | Not required |
| 0.10.0 | mmcv-full>=1.1.4, \<1.3.0 | Not required | Not required |
| 0.9.0 | mmcv-full>=1.1.4, \<1.3.0 | Not required | Not required |
| 0.8.0 | mmcv-full>=1.1.4, \<1.2.0 | Not required | Not required |
| 0.7.0 | mmcv-full>=1.1.2, \<1.2.0 | Not required | Not required |
| 0.6.0 | mmcv-full>=1.1.2, \<1.2.0 | Not required | Not required |
| MMSegmentation version | MMCV version | MMEngine version | MMClassification (optional) version | MMDetection (optional) version |
| :--------------------: | :----------------------------: | :---------------: | :---------------------------------: | :----------------------------: |
| dev-1.x branch | mmcv >= 2.0.0rc4 | MMEngine >= 0.2.0 | mmcls>=1.0.0rc0 | mmdet>=3.0.0rc4, \<=3.0.0rc5> |
| 1.x branch | mmcv == 2.0.0rc3 | MMEngine >= 0.2.0 | mmcls>=1.0.0rc0 | mmdet>=3.0.0rc4, \<=3.0.0rc5> |
| 1.0.0rc4 | mmcv == 2.0.0rc3 | MMEngine >= 0.1.0 | mmcls>=1.0.0rc0 | mmdet>=3.0.0rc4, \<=3.0.0rc5> |
| 1.0.0rc3 | mmcv == 2.0.0rc3 | MMEngine >= 0.1.0 | mmcls>=1.0.0rc0 | mmdet>=3.0.0rc4 \<=3.0.0rc5> |
| 1.0.0rc2 | mmcv == 2.0.0rc3 | MMEngine >= 0.1.0 | mmcls>=1.0.0rc0 | mmdet>=3.0.0rc4 \<=3.0.0rc5> |
| 1.0.0rc1 | mmcv >= 2.0.0rc1, \<=2.0.0rc3> | MMEngine >= 0.1.0 | mmcls>=1.0.0rc0 | Not required |
| 1.0.0rc0 | mmcv >= 2.0.0rc1, \<=2.0.0rc3> | MMEngine >= 0.1.0 | mmcls>=1.0.0rc0 | Not required |
Notes: To install MMSegmentation 0.x and master branch, please refer to [the faq 0.x document](https://mmsegmentation.readthedocs.io/en/latest/faq.html#installation) to check compatible versions of MMCV.
## How to know the number of GPUs needed to train the model

View File

@ -7,9 +7,9 @@ from packaging.version import parse
from .version import __version__, version_info
MMCV_MIN = '2.0.0rc3'
MMCV_MAX = '2.0.0rc3'
MMENGINE_MIN = '0.1.0'
MMCV_MIN = '2.0.0rc4'
MMCV_MAX = '2.1.0'
MMENGINE_MIN = '0.2.0'
MMENGINE_MAX = '1.0.0'
@ -58,9 +58,9 @@ mmcv_max_version = digit_version(MMCV_MAX)
mmcv_version = digit_version(mmcv.__version__)
assert (mmcv_min_version <= mmcv_version <= mmcv_max_version), \
assert (mmcv_min_version <= mmcv_version < mmcv_max_version), \
f'MMCV=={mmcv.__version__} is used but incompatible. ' \
f'Please install mmcv==2.0.0rc3.'
f'Please install mmcv>=2.0.0rc4.'
mmengine_min_version = digit_version(MMENGINE_MIN)
mmengine_max_version = digit_version(MMENGINE_MAX)

View File

@ -4,6 +4,7 @@ import os.path as osp
from typing import Callable, Dict, List, Optional, Sequence, Union
import mmengine
import mmengine.fileio as fileio
import numpy as np
from mmengine.dataset import BaseDataset, Compose
@ -72,9 +73,10 @@ class BaseSegDataset(BaseDataset):
ignore_index (int): The label index to be ignored. Default: 255
reduce_zero_label (bool): Whether to mark label zero as ignored.
Default to False.
file_client_args (dict): Arguments to instantiate a FileClient.
See :class:`mmengine.fileio.FileClient` for details.
Defaults to ``dict(backend='disk')``.
backend_args (dict): Arguments to instantiate a file backend.
See https://mmengine.readthedocs.io/en/latest/api/fileio.htm
for details. Defaults to ``dict(backend='local')``
Notes: mmcv>=2.0.0rc4, mmengine>=0.2.0 required.
"""
METAINFO: dict = dict()
@ -95,16 +97,14 @@ class BaseSegDataset(BaseDataset):
max_refetch: int = 1000,
ignore_index: int = 255,
reduce_zero_label: bool = False,
file_client_args: dict = dict(backend='disk')
backend_args: dict = dict(backend='local')
) -> None:
self.img_suffix = img_suffix
self.seg_map_suffix = seg_map_suffix
self.ignore_index = ignore_index
self.reduce_zero_label = reduce_zero_label
self.file_client_args = file_client_args
self.file_client = mmengine.FileClient.infer_client(
self.file_client_args)
self.backend_args = backend_args.copy()
self.data_root = data_root
self.data_prefix = copy.copy(data_prefix)
@ -239,7 +239,7 @@ class BaseSegDataset(BaseDataset):
ann_dir = self.data_prefix.get('seg_map_path', None)
if osp.isfile(self.ann_file):
lines = mmengine.list_from_file(
self.ann_file, file_client_args=self.file_client_args)
self.ann_file, backend_args=self.backend_args)
for line in lines:
img_name = line.strip()
data_info = dict(
@ -252,11 +252,12 @@ class BaseSegDataset(BaseDataset):
data_info['seg_fields'] = []
data_list.append(data_info)
else:
for img in self.file_client.list_dir_or_file(
for img in fileio.list_dir_or_file(
dir_path=img_dir,
list_dir=False,
suffix=self.img_suffix,
recursive=True):
recursive=True,
backend_args=self.backend_args):
data_info = dict(img_path=osp.join(img_dir, img))
if ann_dir is not None:
seg_map = img.replace(self.img_suffix, self.seg_map_suffix)

View File

@ -1,4 +1,6 @@
# Copyright (c) OpenMMLab. All rights reserved.
import mmengine.fileio as fileio
from mmseg.registry import DATASETS
from .basesegdataset import BaseSegDataset
@ -33,4 +35,5 @@ class iSAIDDataset(BaseSegDataset):
seg_map_suffix=seg_map_suffix,
ignore_index=ignore_index,
**kwargs)
assert self.file_client.exists(self.data_prefix['img_path'])
assert fileio.exists(
self.data_prefix['img_path'], backend_args=self.backend_args)

View File

@ -3,7 +3,7 @@ import warnings
from typing import Dict
import mmcv
import mmengine
import mmengine.fileio as fileio
import numpy as np
from mmcv.transforms import BaseTransform
from mmcv.transforms import LoadAnnotations as MMCV_LoadAnnotations
@ -54,15 +54,16 @@ class LoadAnnotations(MMCV_LoadAnnotations):
argument for :func:``mmcv.imfrombytes``.
See :fun:``mmcv.imfrombytes`` for details.
Defaults to 'pillow'.
file_client_args (dict): Arguments to instantiate a FileClient.
See :class:``mmcv.fileio.FileClient`` for details.
Defaults to ``dict(backend='disk')``.
backend_args (dict): Arguments to instantiate a file backend.
See https://mmengine.readthedocs.io/en/latest/api/fileio.htm
for details. Defaults to ``dict(backend='local')``
Notes: mmcv>=2.0.0rc4, mmengine>=0.2.0 required.
"""
def __init__(
self,
reduce_zero_label=None,
file_client_args=dict(backend='disk'),
backend_args=dict(backend='local'),
imdecode_backend='pillow',
) -> None:
super().__init__(
@ -71,14 +72,13 @@ class LoadAnnotations(MMCV_LoadAnnotations):
with_seg=True,
with_keypoints=False,
imdecode_backend=imdecode_backend,
file_client_args=file_client_args)
backend_args=backend_args)
self.reduce_zero_label = reduce_zero_label
if self.reduce_zero_label is not None:
warnings.warn('`reduce_zero_label` will be deprecated, '
'if you would like to ignore the zero label, please '
'set `reduce_zero_label=True` when dataset '
'initialized')
self.file_client_args = file_client_args.copy()
self.imdecode_backend = imdecode_backend
def _load_seg_map(self, results: dict) -> None:
@ -91,7 +91,8 @@ class LoadAnnotations(MMCV_LoadAnnotations):
dict: The dict contains loaded semantic segmentation annotations.
"""
img_bytes = self.file_client.get(results['seg_map_path'])
img_bytes = fileio.get(
results['seg_map_path'], backend_args=self.backend_args)
gt_semantic_seg = mmcv.imfrombytes(
img_bytes, flag='unchanged',
backend=self.imdecode_backend).squeeze().astype(np.uint8)
@ -121,9 +122,9 @@ class LoadAnnotations(MMCV_LoadAnnotations):
def __repr__(self) -> str:
repr_str = self.__class__.__name__
repr_str += f'(reduce_zero_label={self.reduce_zero_label},'
repr_str += f"imdecode_backend='{self.imdecode_backend}')"
repr_str += f'file_client_args={self.file_client_args})'
repr_str += f'(reduce_zero_label={self.reduce_zero_label}, '
repr_str += f"imdecode_backend='{self.imdecode_backend}', "
repr_str += f'backend_args={self.backend_args})'
return repr_str
@ -202,9 +203,10 @@ class LoadBiomedicalImageFromFile(BaseTransform):
to_float32 (bool): Whether to convert the loaded image to a float32
numpy array. If set to False, the loaded image is an float64 array.
Defaults to True.
file_client_args (dict): Arguments to instantiate a FileClient.
See :class:`mmengine.fileio.FileClient` for details.
Defaults to ``dict(backend='disk')``.
backend_args (dict): Arguments to instantiate a file backend.
See https://mmengine.readthedocs.io/en/latest/api/fileio.htm
for details. Defaults to ``dict(backend='local')``
Notes: mmcv>=2.0.0rc4, mmengine>=0.2.0 required.
"""
def __init__(
@ -212,13 +214,12 @@ class LoadBiomedicalImageFromFile(BaseTransform):
decode_backend: str = 'nifti',
to_xyz: bool = False,
to_float32: bool = True,
file_client_args: dict = dict(backend='disk')
backend_args: dict = dict(backend='local')
) -> None:
self.decode_backend = decode_backend
self.to_xyz = to_xyz
self.to_float32 = to_float32
self.file_client_args = file_client_args.copy()
self.file_client = mmengine.FileClient(**self.file_client_args)
self.backend_args = backend_args.copy()
def transform(self, results: Dict) -> Dict:
"""Functions to load image.
@ -232,7 +233,7 @@ class LoadBiomedicalImageFromFile(BaseTransform):
filename = results['img_path']
data_bytes = self.file_client.get(filename)
data_bytes = fileio.get(filename, self.backend_args)
img = datafrombytes(data_bytes, backend=self.decode_backend)
if self.to_float32:
@ -257,7 +258,7 @@ class LoadBiomedicalImageFromFile(BaseTransform):
f"decode_backend='{self.decode_backend}', "
f'to_xyz={self.to_xyz}, '
f'to_float32={self.to_float32}, '
f'file_client_args={self.file_client_args})')
f'backend_args={self.backend_args})')
return repr_str
@ -294,9 +295,10 @@ class LoadBiomedicalAnnotation(BaseTransform):
to_float32 (bool): Whether to convert the loaded seg map to a float32
numpy array. If set to False, the loaded image is an float64 array.
Defaults to True.
file_client_args (dict): Arguments to instantiate a FileClient.
See :class:`mmengine.fileio.FileClient` for details.
Defaults to ``dict(backend='disk')``.
backend_args (dict): Arguments to instantiate a file backend.
See :class:`mmengine.fileio` for details.
Defaults to ``dict(backend='local')``.
Notes: mmcv>=2.0.0rc4, mmengine>=0.2.0 required.
"""
def __init__(
@ -304,14 +306,13 @@ class LoadBiomedicalAnnotation(BaseTransform):
decode_backend: str = 'nifti',
to_xyz: bool = False,
to_float32: bool = True,
file_client_args: dict = dict(backend='disk')
backend_args: dict = dict(backend='local')
) -> None:
super().__init__()
self.decode_backend = decode_backend
self.to_xyz = to_xyz
self.to_float32 = to_float32
self.file_client_args = file_client_args.copy()
self.file_client = mmengine.FileClient(**self.file_client_args)
self.backend_args = backend_args.copy()
def transform(self, results: Dict) -> Dict:
"""Functions to load image.
@ -322,7 +323,7 @@ class LoadBiomedicalAnnotation(BaseTransform):
Returns:
dict: The dict contains loaded image and meta information.
"""
data_bytes = self.file_client.get(results['seg_map_path'])
data_bytes = fileio.get(results['seg_map_path'], self.backend_args)
gt_seg_map = datafrombytes(data_bytes, backend=self.decode_backend)
if self.to_float32:
@ -342,7 +343,7 @@ class LoadBiomedicalAnnotation(BaseTransform):
f"decode_backend='{self.decode_backend}', "
f'to_xyz={self.to_xyz}, '
f'to_float32={self.to_float32}, '
f'file_client_args={self.file_client_args})')
f'backend_args={self.backend_args})')
return repr_str
@ -383,9 +384,10 @@ class LoadBiomedicalData(BaseTransform):
backend is 'nifti'. Defaults to 'nifti'.
to_xyz (bool): Whether transpose data from Z, Y, X to X, Y, Z.
Defaults to False.
file_client_args (dict): Arguments to instantiate a FileClient.
See :class:`mmengine.fileio.FileClient` for details.
Defaults to ``dict(backend='disk')``.
backend_args (dict): Arguments to instantiate a file backend.
See https://mmengine.readthedocs.io/en/latest/api/fileio.htm
for details. Defaults to ``dict(backend='local')``
Notes: mmcv>=2.0.0rc4, mmengine>=0.2.0 required.
"""
def __init__(
@ -393,13 +395,12 @@ class LoadBiomedicalData(BaseTransform):
with_seg=False,
decode_backend: str = 'numpy',
to_xyz: bool = False,
file_client_args: dict = dict(backend='disk')
) -> None:
backend_args: dict = dict(backend='local')
) -> None: # noqa
self.with_seg = with_seg
self.decode_backend = decode_backend
self.to_xyz = to_xyz
self.file_client_args = file_client_args.copy()
self.file_client = mmengine.FileClient(**self.file_client_args)
self.backend_args = backend_args.copy()
def transform(self, results: Dict) -> Dict:
"""Functions to load image.
@ -410,7 +411,7 @@ class LoadBiomedicalData(BaseTransform):
Returns:
dict: The dict contains loaded image and meta information.
"""
data_bytes = self.file_client.get(results['img_path'])
data_bytes = fileio.get(results['img_path'], self.backend_args)
data = datafrombytes(data_bytes, backend=self.decode_backend)
# img is 4D data (N, X, Y, Z), N is the number of protocol
img = data[:-1, :]
@ -440,5 +441,5 @@ class LoadBiomedicalData(BaseTransform):
f'with_seg={self.with_seg}, '
f"decode_backend='{self.decode_backend}', "
f'to_xyz={self.to_xyz}, '
f'file_client_args={self.file_client_args})')
f'backend_args={self.backend_args})')
return repr_str

View File

@ -1,6 +1,8 @@
# Copyright (c) OpenMMLab. All rights reserved.
import os.path as osp
import mmengine.fileio as fileio
from mmseg.registry import DATASETS
from .basesegdataset import BaseSegDataset
@ -34,5 +36,5 @@ class PascalVOCDataset(BaseSegDataset):
seg_map_suffix=seg_map_suffix,
ann_file=ann_file,
**kwargs)
assert self.file_client.exists(
self.data_prefix['img_path']) and osp.isfile(self.ann_file)
assert fileio.exists(self.data_prefix['img_path'],
self.backend_args) and osp.isfile(self.ann_file)

View File

@ -4,7 +4,7 @@ import warnings
from typing import Sequence
import mmcv
from mmengine.fileio import FileClient
import mmengine.fileio as fileio
from mmengine.hooks import Hook
from mmengine.runner import Runner
@ -30,9 +30,10 @@ class SegVisualizationHook(Hook):
interval (int): The interval of visualization. Defaults to 50.
show (bool): Whether to display the drawn image. Default to False.
wait_time (float): The interval of show (s). Defaults to 0.
file_client_args (dict): Arguments to instantiate a FileClient.
See :class:`mmengine.fileio.FileClient` for details.
Defaults to ``dict(backend='disk')``.
backend_args (dict): Arguments to instantiate a file backend.
See https://mmengine.readthedocs.io/en/latest/api/fileio.htm
for details. Defaults to ``dict(backend='local')``
Notes: mmcv>=2.0.0rc4, mmengine>=0.2.0 required.
"""
def __init__(self,
@ -40,7 +41,7 @@ class SegVisualizationHook(Hook):
interval: int = 50,
show: bool = False,
wait_time: float = 0.,
file_client_args: dict = dict(backend='disk')):
backend_args: dict = dict(backend='local')):
self._visualizer: SegLocalVisualizer = \
SegLocalVisualizer.get_current_instance()
self.interval = interval
@ -54,8 +55,7 @@ class SegVisualizationHook(Hook):
'needs to be excluded.')
self.wait_time = wait_time
self.file_client_args = file_client_args.copy()
self.file_client = None
self.backend_args = backend_args.copy()
self.draw = draw
if not self.draw:
warnings.warn('The draw is False, it means that the '
@ -81,13 +81,11 @@ class SegVisualizationHook(Hook):
if self.draw is False or mode == 'train':
return
if self.file_client is None:
self.file_client = FileClient(**self.file_client_args)
if self.every_n_inner_iters(batch_idx, self.interval):
for output in outputs:
img_path = output.img_path
img_bytes = self.file_client.get(img_path)
img_bytes = fileio.get(
img_path, backend_args=self.backend_args)
img = mmcv.imfrombytes(img_bytes, channel_order='rgb')
window_name = f'{mode}_{osp.basename(img_path)}'

View File

@ -1,4 +1,4 @@
mmcls>=1.0.0rc0
mmcv==2.0.0rc3
mmcv>=2.0.0rc4
mmdet==3.0.0rc5
mmengine>=0.1.0,<1.0.0
mmengine>=0.2.0,<1.0.0

View File

@ -2,7 +2,6 @@
import os
import os.path as osp
import tempfile
from unittest.mock import MagicMock
import pytest
@ -300,17 +299,15 @@ def test_lip():
def test_custom_classes_override_default(dataset, classes):
dataset_class = DATASETS.get(dataset)
if isinstance(dataset_class, PascalVOCDataset):
tmp_file = tempfile.NamedTemporaryFile()
ann_file = f'{tmp_file.name}.txt'
else:
ann_file = MagicMock()
original_classes = dataset_class.METAINFO.get('classes', None)
tmp_file = tempfile.NamedTemporaryFile()
ann_file = tmp_file.name
img_path = tempfile.mkdtemp()
# Test setting classes as a tuple
custom_dataset = dataset_class(
data_prefix=dict(img_path=MagicMock()),
data_prefix=dict(img_path=img_path),
ann_file=ann_file,
metainfo=dict(classes=classes),
test_mode=True,
@ -323,7 +320,7 @@ def test_custom_classes_override_default(dataset, classes):
# Test setting classes as a list
custom_dataset = dataset_class(
data_prefix=dict(img_path=MagicMock()),
data_prefix=dict(img_path=img_path),
ann_file=ann_file,
metainfo=dict(classes=list(classes)),
test_mode=True,
@ -337,7 +334,7 @@ def test_custom_classes_override_default(dataset, classes):
# Test overriding not a subset
custom_dataset = dataset_class(
ann_file=ann_file,
data_prefix=dict(img_path=MagicMock()),
data_prefix=dict(img_path=img_path),
metainfo=dict(classes=[classes[0]]),
test_mode=True,
lazy_init=True)
@ -352,13 +349,13 @@ def test_custom_classes_override_default(dataset, classes):
with pytest.raises(AssertionError):
custom_dataset = dataset_class(
ann_file=ann_file,
data_prefix=dict(img_path=MagicMock()),
data_prefix=dict(img_path=img_path),
metainfo=None,
test_mode=True,
lazy_init=True)
else:
custom_dataset = dataset_class(
data_prefix=dict(img_path=MagicMock()),
data_prefix=dict(img_path=img_path),
ann_file=ann_file,
metainfo=None,
test_mode=True,
@ -371,8 +368,8 @@ def test_custom_classes_override_default(dataset, classes):
def test_custom_dataset_random_palette_is_generated():
dataset = BaseSegDataset(
pipeline=[],
data_prefix=dict(img_path=MagicMock()),
ann_file=MagicMock(),
data_prefix=dict(img_path=tempfile.mkdtemp()),
ann_file=tempfile.mkdtemp(),
metainfo=dict(classes=('bus', 'car')),
lazy_init=True,
test_mode=True)
@ -384,8 +381,8 @@ def test_custom_dataset_random_palette_is_generated():
def test_custom_dataset_custom_palette():
dataset = BaseSegDataset(
data_prefix=dict(img_path=MagicMock()),
ann_file=MagicMock(),
data_prefix=dict(img_path=tempfile.mkdtemp()),
ann_file=tempfile.mkdtemp(),
metainfo=dict(
classes=('bus', 'car'), palette=[[100, 100, 100], [200, 200,
200]]),
@ -396,7 +393,7 @@ def test_custom_dataset_custom_palette():
# test custom class and palette don't match
with pytest.raises(ValueError):
dataset = BaseSegDataset(
data_prefix=dict(img_path=MagicMock()),
ann_file=MagicMock(),
data_prefix=dict(img_path=tempfile.mkdtemp()),
ann_file=tempfile.mkdtemp(),
metainfo=dict(classes=('bus', 'car'), palette=[[200, 200, 200]]),
lazy_init=True)

View File

@ -30,7 +30,7 @@ class TestLoading:
assert results['ori_shape'] == results['img'].shape[:2]
assert repr(transform) == transform.__class__.__name__ + \
"(ignore_empty=False, to_float32=False, color_type='color'," + \
" imdecode_backend='cv2', file_client_args={'backend': 'disk'})"
" imdecode_backend='cv2', backend_args=None)"
# to_float32
transform = LoadImageFromFile(to_float32=True)
@ -57,9 +57,9 @@ class TestLoading:
results = transform(copy.deepcopy(results))
assert results['gt_seg_map'].shape == (288, 512)
assert results['gt_seg_map'].dtype == np.uint8
assert repr(transform) == transform.__class__.__name__ + \
"(reduce_zero_label=True,imdecode_backend='pillow')" + \
"file_client_args={'backend': 'disk'})"
# assert repr(transform) == transform.__class__.__name__ + \
# "(reduce_zero_label=True, imdecode_backend='pillow', " + \
# "backend_args={'backend': 'local'})"
# reduce_zero_label
transform = LoadAnnotations(reduce_zero_label=True)
@ -225,7 +225,7 @@ class TestLoading:
'to_float32=False, '
"color_type='color', "
"imdecode_backend='cv2', "
"file_client_args={'backend': 'disk'})")
'backend_args=None)')
def test_load_biomedical_img(self):
results = dict(
@ -241,7 +241,7 @@ class TestLoading:
"decode_backend='nifti', "
'to_xyz=False, '
'to_float32=True, '
"file_client_args={'backend': 'disk'})")
"backend_args={'backend': 'local'})")
def test_load_biomedical_annotation(self):
results = dict(
@ -265,7 +265,7 @@ class TestLoading:
'with_seg=True, '
"decode_backend='numpy', "
'to_xyz=False, '
"file_client_args={'backend': 'disk'})")
"backend_args={'backend': 'local'})")
transform = LoadBiomedicalData(with_seg=False)
results = transform(copy.deepcopy(input_results))
@ -275,4 +275,4 @@ class TestLoading:
'with_seg=False, '
"decode_backend='numpy', "
'to_xyz=False, '
"file_client_args={'backend': 'disk'})")
"backend_args={'backend': 'local'})")