[Refactor] Support to use "split" to specify training set/validation set in the ImageNet dataset (#1535)
* [Feature]: Add caption * [Feature]: Update scienceqa * [CI] Add test mim CI. (#879) * refactor imagenet dataset * refactor imagenet dataset * refactor imagenet dataset * update imagenet21k * update configs * update mnist * update dataset_prepare.md * fix sun397 url and update user_guides/dataset_prepare.md * update dataset_prepare.md * fix sun397 dataset * fix sun397 * update chinese dataset_prepare.md * update dataset_prepare.md * [Refactor] update voc dataset * [Refactor] update voc dataset * refactor imagenet * refactor imagenet * use mmengine.fileio --------- Co-authored-by: liuyuan <3463423099@qq.com> Co-authored-by: Ma Zerun <mzr1996@163.com> Co-authored-by: Ezra-Yu <18586273+Ezra-Yu@users.noreply.github.com>pull/1609/head
zzc98
GitHub
parent
795607cfeb
commit
bc3c4a35ee
configs/_base_/datasets
docs
en/user_guides
zh_CN/user_guides
mmpretrain/datasets
tests/test_datasets
|
|
@ -16,38 +16,13 @@ train_pipeline = [
|
|||
dict(type='PackInputs'),
|
||||
]
|
||||
|
||||
test_pipeline = [
|
||||
dict(type='LoadImageFromFile'),
|
||||
dict(type='ResizeEdge', scale=256, edge='short'),
|
||||
dict(type='CenterCrop', crop_size=224),
|
||||
dict(type='PackInputs'),
|
||||
]
|
||||
|
||||
train_dataloader = dict(
|
||||
batch_size=128,
|
||||
num_workers=5,
|
||||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root='data/imagenet21k',
|
||||
ann_file='meta/train.txt',
|
||||
data_prefix='train',
|
||||
split='train',
|
||||
pipeline=train_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=True),
|
||||
)
|
||||
|
||||
val_dataloader = dict(
|
||||
batch_size=128,
|
||||
num_workers=5,
|
||||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root='data/imagenet21k',
|
||||
ann_file='meta/val.txt',
|
||||
data_prefix='val',
|
||||
pipeline=test_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=False),
|
||||
)
|
||||
val_evaluator = dict(type='Accuracy', topk=(1, 5))
|
||||
|
||||
# If you want standard test, please manually configure the test dataset
|
||||
test_dataloader = val_dataloader
|
||||
test_evaluator = val_evaluator
|
||||
|
|
|
|||
|
|
@ -44,8 +44,7 @@ train_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root='data/imagenet',
|
||||
ann_file='meta/train.txt',
|
||||
data_prefix='train',
|
||||
split='train',
|
||||
pipeline=train_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=True),
|
||||
)
|
||||
|
|
@ -56,8 +55,7 @@ val_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root='data/imagenet',
|
||||
ann_file='meta/val.txt',
|
||||
data_prefix='val',
|
||||
split='val',
|
||||
pipeline=test_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=False),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -58,8 +58,7 @@ train_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root='data/imagenet',
|
||||
ann_file='meta/train.txt',
|
||||
data_prefix='train',
|
||||
split='train',
|
||||
pipeline=train_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=True),
|
||||
)
|
||||
|
|
@ -70,8 +69,7 @@ val_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root='data/imagenet',
|
||||
ann_file='meta/val.txt',
|
||||
data_prefix='val',
|
||||
split='val',
|
||||
pipeline=test_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=False),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -58,8 +58,7 @@ train_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root='data/imagenet',
|
||||
ann_file='meta/train.txt',
|
||||
data_prefix='train',
|
||||
split='train',
|
||||
pipeline=train_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=True),
|
||||
)
|
||||
|
|
@ -70,8 +69,7 @@ val_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root='data/imagenet',
|
||||
ann_file='meta/val.txt',
|
||||
data_prefix='val',
|
||||
split='val',
|
||||
pipeline=test_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=False),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -59,8 +59,7 @@ train_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root='data/imagenet',
|
||||
ann_file='meta/train.txt',
|
||||
data_prefix='train',
|
||||
split='train',
|
||||
pipeline=train_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=True),
|
||||
persistent_workers=True,
|
||||
|
|
@ -72,8 +71,7 @@ val_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root='data/imagenet',
|
||||
ann_file='meta/val.txt',
|
||||
data_prefix='val',
|
||||
split='val',
|
||||
pipeline=test_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=False),
|
||||
persistent_workers=True,
|
||||
|
|
|
|||
|
|
@ -58,8 +58,7 @@ train_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root='data/imagenet',
|
||||
ann_file='meta/train.txt',
|
||||
data_prefix='train',
|
||||
split='train',
|
||||
pipeline=train_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=True),
|
||||
)
|
||||
|
|
@ -70,8 +69,7 @@ val_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root='data/imagenet',
|
||||
ann_file='meta/val.txt',
|
||||
data_prefix='val',
|
||||
split='val',
|
||||
pipeline=test_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=False),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -58,8 +58,7 @@ train_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root='data/imagenet',
|
||||
ann_file='meta/train.txt',
|
||||
data_prefix='train',
|
||||
split='train',
|
||||
pipeline=train_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=True),
|
||||
)
|
||||
|
|
@ -70,8 +69,7 @@ val_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root='data/imagenet',
|
||||
ann_file='meta/val.txt',
|
||||
data_prefix='val',
|
||||
split='val',
|
||||
pipeline=test_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=False),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -58,8 +58,7 @@ train_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root='data/imagenet',
|
||||
ann_file='meta/train.txt',
|
||||
data_prefix='train',
|
||||
split='train',
|
||||
pipeline=train_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=True),
|
||||
)
|
||||
|
|
@ -70,8 +69,7 @@ val_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root='data/imagenet',
|
||||
ann_file='meta/val.txt',
|
||||
data_prefix='val',
|
||||
split='val',
|
||||
pipeline=test_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=False),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -38,8 +38,7 @@ train_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root='data/imagenet',
|
||||
ann_file='meta/train.txt',
|
||||
data_prefix='train',
|
||||
split='train',
|
||||
pipeline=train_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=True),
|
||||
)
|
||||
|
|
@ -50,8 +49,7 @@ val_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root='data/imagenet',
|
||||
ann_file='meta/val.txt',
|
||||
data_prefix='val',
|
||||
split='val',
|
||||
pipeline=test_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=False),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -38,8 +38,7 @@ train_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root='data/imagenet',
|
||||
ann_file='meta/train.txt',
|
||||
data_prefix='train',
|
||||
split='train',
|
||||
pipeline=train_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=True),
|
||||
)
|
||||
|
|
@ -50,8 +49,7 @@ val_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root='data/imagenet',
|
||||
ann_file='meta/val.txt',
|
||||
data_prefix='val',
|
||||
split='val',
|
||||
pipeline=test_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=False),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -38,8 +38,7 @@ train_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root='data/imagenet',
|
||||
ann_file='meta/train.txt',
|
||||
data_prefix='train',
|
||||
split='train',
|
||||
pipeline=train_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=True),
|
||||
)
|
||||
|
|
@ -50,8 +49,7 @@ val_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root='data/imagenet',
|
||||
ann_file='meta/val.txt',
|
||||
data_prefix='val',
|
||||
split='val',
|
||||
pipeline=test_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=False),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -31,8 +31,7 @@ train_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root='data/imagenet',
|
||||
ann_file='meta/train.txt',
|
||||
data_prefix='train',
|
||||
split='train',
|
||||
pipeline=train_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=True),
|
||||
)
|
||||
|
|
@ -43,8 +42,7 @@ val_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root='data/imagenet',
|
||||
ann_file='meta/val.txt',
|
||||
data_prefix='val',
|
||||
split='val',
|
||||
pipeline=test_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=False),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -43,6 +43,5 @@ train_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root=data_root,
|
||||
ann_file='meta/train.txt',
|
||||
data_prefix=dict(img_path='train/'),
|
||||
split='train',
|
||||
pipeline=train_pipeline))
|
||||
|
|
|
|||
|
|
@ -58,8 +58,7 @@ train_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root='data/imagenet',
|
||||
ann_file='meta/train.txt',
|
||||
data_prefix='train',
|
||||
split='train',
|
||||
pipeline=train_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=True),
|
||||
)
|
||||
|
|
@ -70,8 +69,7 @@ val_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root='data/imagenet',
|
||||
ann_file='meta/val.txt',
|
||||
data_prefix='val',
|
||||
split='val',
|
||||
pipeline=test_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=False),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -57,9 +57,8 @@ train_dataloader = dict(
|
|||
num_workers=4,
|
||||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root=r'E:\imagenet',
|
||||
ann_file='meta/val.txt',
|
||||
data_prefix='ILSVRC2012_img_val',
|
||||
data_root='data/imagenet',
|
||||
split='train',
|
||||
pipeline=train_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=True),
|
||||
)
|
||||
|
|
@ -69,9 +68,8 @@ val_dataloader = dict(
|
|||
num_workers=4,
|
||||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root=r'E:\imagenet',
|
||||
ann_file='meta/val.txt',
|
||||
data_prefix='ILSVRC2012_img_val',
|
||||
data_root='data/imagenet',
|
||||
split='val',
|
||||
pipeline=test_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=False),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -50,8 +50,7 @@ train_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root='data/imagenet',
|
||||
ann_file='meta/train.txt',
|
||||
data_prefix='train',
|
||||
split='train',
|
||||
pipeline=train_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=True),
|
||||
)
|
||||
|
|
@ -62,8 +61,7 @@ val_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root='data/imagenet',
|
||||
ann_file='meta/val.txt',
|
||||
data_prefix='val',
|
||||
split='val',
|
||||
pipeline=test_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=False),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -50,8 +50,7 @@ train_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root='data/imagenet',
|
||||
ann_file='meta/train.txt',
|
||||
data_prefix='train',
|
||||
split='train',
|
||||
pipeline=train_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=True),
|
||||
)
|
||||
|
|
@ -62,8 +61,7 @@ val_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root='data/imagenet',
|
||||
ann_file='meta/val.txt',
|
||||
data_prefix='val',
|
||||
split='val',
|
||||
pipeline=test_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=False),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -29,6 +29,5 @@ train_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root=data_root,
|
||||
ann_file='meta/train.txt',
|
||||
data_prefix=dict(img_path='train/'),
|
||||
split='train',
|
||||
pipeline=train_pipeline))
|
||||
|
|
|
|||
|
|
@ -58,8 +58,7 @@ train_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root=data_root,
|
||||
ann_file='meta/train.txt',
|
||||
data_prefix='train',
|
||||
split='train',
|
||||
pipeline=train_pipeline),
|
||||
)
|
||||
|
||||
|
|
@ -72,8 +71,7 @@ val_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root=data_root,
|
||||
ann_file='meta/val.txt',
|
||||
data_prefix='val',
|
||||
split='val',
|
||||
pipeline=test_pipeline),
|
||||
)
|
||||
val_evaluator = dict(type='Accuracy', topk=(1, 5))
|
||||
|
|
|
|||
|
|
@ -29,8 +29,7 @@ train_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root='data/imagenet',
|
||||
ann_file='meta/train.txt',
|
||||
data_prefix='train',
|
||||
split='train',
|
||||
pipeline=train_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=True),
|
||||
)
|
||||
|
|
@ -41,8 +40,7 @@ val_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root='data/imagenet',
|
||||
ann_file='meta/val.txt',
|
||||
data_prefix='val',
|
||||
split='val',
|
||||
pipeline=test_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=False),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -85,6 +85,5 @@ train_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root=data_root,
|
||||
ann_file='meta/train.txt',
|
||||
data_prefix=dict(img_path='train/'),
|
||||
split='train',
|
||||
pipeline=train_pipeline))
|
||||
|
|
|
|||
|
|
@ -54,6 +54,5 @@ train_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root=data_root,
|
||||
ann_file='meta/train.txt',
|
||||
data_prefix=dict(img_path='train/'),
|
||||
split='train',
|
||||
pipeline=train_pipeline))
|
||||
|
|
|
|||
|
|
@ -38,8 +38,7 @@ train_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root='data/imagenet',
|
||||
ann_file='meta/train.txt',
|
||||
data_prefix='train',
|
||||
split='train',
|
||||
pipeline=train_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=True),
|
||||
)
|
||||
|
|
@ -50,8 +49,7 @@ val_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root='data/imagenet',
|
||||
ann_file='meta/val.txt',
|
||||
data_prefix='val',
|
||||
split='val',
|
||||
pipeline=test_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=False),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -29,8 +29,7 @@ train_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root='data/imagenet',
|
||||
ann_file='meta/train.txt',
|
||||
data_prefix='train',
|
||||
split='train',
|
||||
pipeline=train_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=True),
|
||||
)
|
||||
|
|
@ -41,8 +40,7 @@ val_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root='data/imagenet',
|
||||
ann_file='meta/val.txt',
|
||||
data_prefix='val',
|
||||
split='val',
|
||||
pipeline=test_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=False),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -48,6 +48,5 @@ train_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root=data_root,
|
||||
ann_file='meta/train.txt',
|
||||
data_prefix=dict(img_path='train/'),
|
||||
split='train',
|
||||
pipeline=train_pipeline))
|
||||
|
|
|
|||
|
|
@ -28,6 +28,5 @@ train_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root=data_root,
|
||||
ann_file='meta/train.txt',
|
||||
data_prefix=dict(img_path='train/'),
|
||||
split='train',
|
||||
pipeline=train_pipeline))
|
||||
|
|
|
|||
|
|
@ -86,6 +86,5 @@ train_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root=data_root,
|
||||
ann_file='meta/train.txt',
|
||||
data_prefix=dict(img_path='train/'),
|
||||
split='train',
|
||||
pipeline=train_pipeline))
|
||||
|
|
|
|||
|
|
@ -29,8 +29,7 @@ train_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root='data/imagenet',
|
||||
ann_file='meta/train.txt',
|
||||
data_prefix='train',
|
||||
split='train',
|
||||
pipeline=train_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=True),
|
||||
)
|
||||
|
|
@ -41,8 +40,7 @@ val_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root='data/imagenet',
|
||||
ann_file='meta/val.txt',
|
||||
data_prefix='val',
|
||||
split='val',
|
||||
pipeline=test_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=False),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -37,8 +37,7 @@ train_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root='data/imagenet',
|
||||
ann_file='meta/train.txt',
|
||||
data_prefix='train',
|
||||
split='train',
|
||||
pipeline=train_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=True),
|
||||
)
|
||||
|
|
@ -49,8 +48,7 @@ val_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root='data/imagenet',
|
||||
ann_file='meta/val.txt',
|
||||
data_prefix='val',
|
||||
split='val',
|
||||
pipeline=test_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=False),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -55,18 +55,19 @@ data = dict(
|
|||
workers_per_gpu=8,
|
||||
train=dict(
|
||||
type=dataset_type,
|
||||
data_prefix='data/imagenet/train',
|
||||
data_root='data/imagenet',
|
||||
split='train',
|
||||
pipeline=train_pipeline),
|
||||
val=dict(
|
||||
type=dataset_type,
|
||||
data_prefix='data/imagenet/val',
|
||||
ann_file='data/imagenet/meta/val.txt',
|
||||
data_root='data/imagenet',
|
||||
split='val',
|
||||
pipeline=test_pipeline),
|
||||
test=dict(
|
||||
# replace `data/val` with `data/test` for standard test
|
||||
type=dataset_type,
|
||||
data_prefix='data/imagenet/val',
|
||||
ann_file='data/imagenet/meta/val.txt',
|
||||
data_root='data/imagenet',
|
||||
split='val',
|
||||
pipeline=test_pipeline))
|
||||
|
||||
evaluation = dict(interval=10, metric='accuracy')
|
||||
|
|
|
|||
|
|
@ -55,18 +55,19 @@ data = dict(
|
|||
workers_per_gpu=8,
|
||||
train=dict(
|
||||
type=dataset_type,
|
||||
data_prefix='data/imagenet/train',
|
||||
data_root='data/imagenet',
|
||||
split='train',
|
||||
pipeline=train_pipeline),
|
||||
val=dict(
|
||||
type=dataset_type,
|
||||
data_prefix='data/imagenet/val',
|
||||
ann_file='data/imagenet/meta/val.txt',
|
||||
data_root='data/imagenet',
|
||||
split='val',
|
||||
pipeline=test_pipeline),
|
||||
test=dict(
|
||||
# replace `data/val` with `data/test` for standard test
|
||||
type=dataset_type,
|
||||
data_prefix='data/imagenet/val',
|
||||
ann_file='data/imagenet/meta/val.txt',
|
||||
data_root='data/imagenet',
|
||||
split='val',
|
||||
pipeline=test_pipeline))
|
||||
|
||||
evaluation = dict(interval=10, metric='accuracy')
|
||||
|
|
|
|||
|
|
@ -56,18 +56,19 @@ data = dict(
|
|||
workers_per_gpu=8,
|
||||
train=dict(
|
||||
type=dataset_type,
|
||||
data_prefix='data/imagenet/train',
|
||||
data_root='data/imagenet',
|
||||
split='train',
|
||||
pipeline=train_pipeline),
|
||||
val=dict(
|
||||
type=dataset_type,
|
||||
data_prefix='data/imagenet/val',
|
||||
ann_file='data/imagenet/meta/val.txt',
|
||||
data_root='data/imagenet',
|
||||
split='val',
|
||||
pipeline=test_pipeline),
|
||||
test=dict(
|
||||
# replace `data/val` with `data/test` for standard test
|
||||
type=dataset_type,
|
||||
data_prefix='data/imagenet/val',
|
||||
ann_file='data/imagenet/meta/val.txt',
|
||||
data_root='data/imagenet',
|
||||
split='val',
|
||||
pipeline=test_pipeline))
|
||||
|
||||
evaluation = dict(interval=10, metric='accuracy')
|
||||
|
|
|
|||
|
|
@ -58,8 +58,7 @@ train_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root='data/imagenet',
|
||||
ann_file='meta/train.txt',
|
||||
data_prefix='train',
|
||||
split='train',
|
||||
pipeline=train_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=True),
|
||||
)
|
||||
|
|
@ -70,8 +69,7 @@ val_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root='data/imagenet',
|
||||
ann_file='meta/val.txt',
|
||||
data_prefix='val',
|
||||
split='val',
|
||||
pipeline=test_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=False),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -58,8 +58,7 @@ train_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root='data/imagenet',
|
||||
ann_file='meta/train.txt',
|
||||
data_prefix='train',
|
||||
split='train',
|
||||
pipeline=train_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=True),
|
||||
)
|
||||
|
|
@ -70,8 +69,7 @@ val_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root='data/imagenet',
|
||||
ann_file='meta/val.txt',
|
||||
data_prefix='val',
|
||||
split='val',
|
||||
pipeline=test_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=False),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -38,8 +38,7 @@ train_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root='data/imagenet',
|
||||
ann_file='meta/train.txt',
|
||||
data_prefix='train',
|
||||
split='train',
|
||||
pipeline=train_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=True),
|
||||
)
|
||||
|
|
@ -50,8 +49,7 @@ val_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root='data/imagenet',
|
||||
ann_file='meta/val.txt',
|
||||
data_prefix='val',
|
||||
split='val',
|
||||
pipeline=test_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=False),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -58,8 +58,7 @@ train_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root='data/imagenet',
|
||||
ann_file='meta/train.txt',
|
||||
data_prefix='train',
|
||||
split='train',
|
||||
pipeline=train_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=True),
|
||||
)
|
||||
|
|
@ -70,8 +69,7 @@ val_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root='data/imagenet',
|
||||
ann_file='meta/val.txt',
|
||||
data_prefix='val',
|
||||
split='val',
|
||||
pipeline=test_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=False),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -30,8 +30,7 @@ train_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root='data/imagenet',
|
||||
ann_file='meta/train.txt',
|
||||
data_prefix='train',
|
||||
split='train',
|
||||
pipeline=train_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=True),
|
||||
)
|
||||
|
|
@ -42,8 +41,7 @@ val_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root='data/imagenet',
|
||||
ann_file='meta/val.txt',
|
||||
data_prefix='val',
|
||||
split='val',
|
||||
pipeline=test_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=False),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -29,8 +29,7 @@ train_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root='data/imagenet',
|
||||
ann_file='meta/train.txt',
|
||||
data_prefix='train',
|
||||
split='train',
|
||||
pipeline=train_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=True),
|
||||
)
|
||||
|
|
@ -41,8 +40,7 @@ val_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root='data/imagenet',
|
||||
ann_file='meta/val.txt',
|
||||
data_prefix='val',
|
||||
split='val',
|
||||
pipeline=test_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=False),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -46,8 +46,7 @@ train_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root='data/imagenet',
|
||||
ann_file='meta/train.txt',
|
||||
data_prefix='train',
|
||||
split='train',
|
||||
pipeline=train_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=True),
|
||||
)
|
||||
|
|
@ -58,8 +57,7 @@ val_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root='data/imagenet',
|
||||
ann_file='meta/val.txt',
|
||||
data_prefix='val',
|
||||
split='val',
|
||||
pipeline=test_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=False),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -58,8 +58,7 @@ train_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root='data/imagenet',
|
||||
ann_file='meta/train.txt',
|
||||
data_prefix='train',
|
||||
split='train',
|
||||
pipeline=train_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=True),
|
||||
)
|
||||
|
|
@ -70,8 +69,7 @@ val_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root='data/imagenet',
|
||||
ann_file='meta/val.txt',
|
||||
data_prefix='val',
|
||||
split='val',
|
||||
pipeline=test_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=False),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -58,8 +58,7 @@ train_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root='data/imagenet',
|
||||
ann_file='meta/train.txt',
|
||||
data_prefix='train',
|
||||
split='train',
|
||||
pipeline=train_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=True),
|
||||
)
|
||||
|
|
@ -70,8 +69,7 @@ val_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root='data/imagenet',
|
||||
ann_file='meta/val.txt',
|
||||
data_prefix='val',
|
||||
split='val',
|
||||
pipeline=test_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=False),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -32,8 +32,7 @@ train_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root='data/imagenet',
|
||||
ann_file='meta/train.txt',
|
||||
data_prefix='train',
|
||||
split='train',
|
||||
pipeline=train_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=True),
|
||||
)
|
||||
|
|
@ -44,8 +43,7 @@ val_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root='data/imagenet',
|
||||
ann_file='meta/val.txt',
|
||||
data_prefix='val',
|
||||
split='val',
|
||||
pipeline=test_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=False),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -58,8 +58,7 @@ train_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root='data/imagenet',
|
||||
ann_file='meta/train.txt',
|
||||
data_prefix='train',
|
||||
split='train',
|
||||
pipeline=train_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=True),
|
||||
)
|
||||
|
|
@ -70,8 +69,7 @@ val_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root='data/imagenet',
|
||||
ann_file='meta/val.txt',
|
||||
data_prefix='val',
|
||||
split='val',
|
||||
pipeline=test_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=False),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -37,8 +37,7 @@ train_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root='data/imagenet',
|
||||
ann_file='meta/train.txt',
|
||||
data_prefix='train',
|
||||
split='train',
|
||||
pipeline=train_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=True),
|
||||
)
|
||||
|
|
@ -49,8 +48,7 @@ val_dataloader = dict(
|
|||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root='data/imagenet',
|
||||
ann_file='meta/val.txt',
|
||||
data_prefix='val',
|
||||
split='val',
|
||||
pipeline=test_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=False),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -22,7 +22,12 @@ test_pipeline = [
|
|||
dict(type='LoadImageFromFile'),
|
||||
dict(type='ResizeEdge', scale=256, edge='short'),
|
||||
dict(type='CenterCrop', crop_size=224),
|
||||
dict(type='PackInputs'),
|
||||
dict(
|
||||
type='PackInputs',
|
||||
# `gt_label_difficult` is needed for VOC evaluation
|
||||
meta_keys=('sample_idx', 'img_path', 'ori_shape', 'img_shape',
|
||||
'scale_factor', 'flip', 'flip_direction',
|
||||
'gt_label_difficult')),
|
||||
]
|
||||
|
||||
train_dataloader = dict(
|
||||
|
|
@ -30,8 +35,8 @@ train_dataloader = dict(
|
|||
num_workers=5,
|
||||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root='data/VOCdevkit/VOC2007',
|
||||
image_set_path='ImageSets/Layout/val.txt',
|
||||
data_root='data/VOC2007',
|
||||
split='trainval',
|
||||
pipeline=train_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=True),
|
||||
)
|
||||
|
|
@ -41,22 +46,13 @@ val_dataloader = dict(
|
|||
num_workers=5,
|
||||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root='data/VOCdevkit/VOC2007',
|
||||
image_set_path='ImageSets/Layout/val.txt',
|
||||
data_root='data/VOC2007',
|
||||
split='test',
|
||||
pipeline=test_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=False),
|
||||
)
|
||||
|
||||
test_dataloader = dict(
|
||||
batch_size=16,
|
||||
num_workers=5,
|
||||
dataset=dict(
|
||||
type=dataset_type,
|
||||
data_root='data/VOCdevkit/VOC2007',
|
||||
image_set_path='ImageSets/Layout/val.txt',
|
||||
pipeline=test_pipeline),
|
||||
sampler=dict(type='DefaultSampler', shuffle=False),
|
||||
)
|
||||
test_dataloader = val_dataloader
|
||||
|
||||
# calculate precision_recall_f1 and mAP
|
||||
val_evaluator = [
|
||||
|
|
@ -65,6 +61,5 @@ val_evaluator = [
|
|||
dict(type='VOCAveragePrecision')
|
||||
]
|
||||
|
||||
# If you want standard test, please manually configure the test dataset
|
||||
test_dataloader = val_dataloader
|
||||
test_evaluator = val_evaluator
|
||||
|
|
|
|||
|
|
@ -145,16 +145,14 @@ ImageNet has multiple versions, but the most commonly used one is [ILSVRC 2012](
|
|||
- ILSVRC2012_img_train.tar (~138GB)
|
||||
- ILSVRC2012_img_val.tar (~6.3GB)
|
||||
3. Untar the downloaded files
|
||||
4. Re-organize the image files according the format convention of [CustomDataset](#CustomDataset)
|
||||
|
||||
```{note}
|
||||
In MMPretrain, we use the text annotation file format ImageNet in all provided config files. Therefore, to use
|
||||
the subfolder format, you please set `ann_file=''` in these config files.
|
||||
```
|
||||
### The Directory Structrue of the ImageNet dataset
|
||||
|
||||
### Subfolder Format
|
||||
We support two ways of organizing the ImageNet dataset: Subfolder Format and Text Annotation File Format.
|
||||
|
||||
Re-organize the dataset as below:
|
||||
#### Subfolder Format
|
||||
|
||||
We have provided a sample, which you can download and extract from this [link](https://download.openmmlab.com/mmpretrain/datasets/imagenet_1k.zip). The directory structure of the dataset should be as below:
|
||||
|
||||
```text
|
||||
data/imagenet/
|
||||
|
|
@ -177,37 +175,7 @@ data/imagenet/
|
|||
│ ├── ...
|
||||
```
|
||||
|
||||
And then, you can use the [`ImageNet`](mmpretrain.datasets.ImageNet) dataset with the below configurations:
|
||||
|
||||
```python
|
||||
train_dataloader = dict(
|
||||
...
|
||||
# Training dataset configurations
|
||||
dataset=dict(
|
||||
type='ImageNet',
|
||||
data_root='data/imagenet',
|
||||
data_prefix='train',
|
||||
ann_file='',
|
||||
pipeline=...,
|
||||
)
|
||||
)
|
||||
|
||||
val_dataloader = dict(
|
||||
...
|
||||
# Validation dataset configurations
|
||||
dataset=dict(
|
||||
type='ImageNet',
|
||||
data_root='data/imagenet',
|
||||
data_prefix='val',
|
||||
ann_file='',
|
||||
pipeline=...,
|
||||
)
|
||||
)
|
||||
|
||||
test_dataloader = val_dataloader
|
||||
```
|
||||
|
||||
### Text Annotation File Format
|
||||
#### Text Annotation File Format
|
||||
|
||||
You can download and untar the meta data from this [link](https://download.openmmlab.com/mmclassification/datasets/imagenet/meta/caffe_ilsvrc12.tar.gz). And re-organize the dataset as below:
|
||||
|
||||
|
|
@ -235,7 +203,9 @@ data/imagenet/
|
|||
│ ├── ...
|
||||
```
|
||||
|
||||
And then, you can use the [`ImageNet`](mmpretrain.datasets.ImageNet) dataset with the below configurations:
|
||||
### Configuration
|
||||
|
||||
Once your dataset is organized in the way described above, you can use the [`ImageNet`](mmpretrain.datasets.ImageNet) dataset with the below configurations:
|
||||
|
||||
```python
|
||||
train_dataloader = dict(
|
||||
|
|
@ -243,9 +213,8 @@ train_dataloader = dict(
|
|||
# Training dataset configurations
|
||||
dataset=dict(
|
||||
type='ImageNet',
|
||||
data_root='imagenet_folder',
|
||||
data_prefix='train/',
|
||||
ann_file='meta/train.txt',
|
||||
data_root='data/imagenet',
|
||||
split='train',
|
||||
pipeline=...,
|
||||
)
|
||||
)
|
||||
|
|
@ -255,9 +224,8 @@ val_dataloader = dict(
|
|||
# Validation dataset configurations
|
||||
dataset=dict(
|
||||
type='ImageNet',
|
||||
data_root='imagenet_folder',
|
||||
data_prefix='val/',
|
||||
ann_file='meta/val.txt',
|
||||
data_root='data/imagenet',
|
||||
split='val',
|
||||
pipeline=...,
|
||||
)
|
||||
)
|
||||
|
|
@ -265,69 +233,27 @@ val_dataloader = dict(
|
|||
test_dataloader = val_dataloader
|
||||
```
|
||||
|
||||
## CIFAR
|
||||
## Supported Image Classification Datasets
|
||||
|
||||
We support downloading the [`CIFAR10`](mmpretrain.datasets.CIFAR10) and [`CIFAR100`](mmpretrain.datasets.CIFAR100) datasets automatically, and you just need to specify the
|
||||
download folder in the `data_root` field. And please specify `test_mode=False` / `test_mode=True`
|
||||
to use training datasets or test datasets.
|
||||
| Datasets | split | HomePage |
|
||||
| ---------------------------------------------------------------------------------- | :---------------------------------- | ----------------------------------------------------------------------------------- |
|
||||
| [`Calthch101`](mmpretrain.datasets.Caltech101)(data_root[, split, pipeline, ...]) | ["train", "test"] | [Caltech 101](https://data.caltech.edu/records/mzrjq-6wc02) Dataset. |
|
||||
| [`CIFAR10`](mmpretrain.datasets.CIFAR10)(data_root[, split, pipeline, ...]) | ["train", "test"] | [CIFAR10](https://www.cs.toronto.edu/~kriz/cifar.html) Dataset. |
|
||||
| [`CIFAR100`](mmpretrain.datasets.CIFAR100)(data_root[, split, pipeline, ...]) | ["train", "test"] | [CIFAR100](https://www.cs.toronto.edu/~kriz/cifar.html) Dataset. |
|
||||
| [`CUB`](mmpretrain.datasets.CUB)(data_root[, split, pipeline, ...]) | ["train", "test"] | [CUB-200-2011](http://www.vision.caltech.edu/datasets/cub_200_2011/) Dataset. |
|
||||
| [`DTD`](mmpretrain.datasets.DTD)(data_root[, split, pipeline, ...]) | ["train", "val", "tranval", "test"] | [Describable Texture Dataset (DTD)](https://www.robots.ox.ac.uk/~vgg/data/dtd/) Dataset. |
|
||||
| [`FashionMNIST`](mmpretrain.datasets.FashionMNIST) (data_root[, split, pipeline, ...]) | ["train", "test"] | [Fashion-MNIST](https://github.com/zalandoresearch/fashion-mnist) Dataset. |
|
||||
| [`FGVCAircraft`](mmpretrain.datasets.FGVCAircraft)(data_root[, split, pipeline, ...]) | ["train", "val", "tranval", "test"] | [FGVC Aircraft](https://www.robots.ox.ac.uk/~vgg/data/fgvc-aircraft/) Dataset. |
|
||||
| [`Flowers102`](mmpretrain.datasets.Flowers102)(data_root[, split, pipeline, ...]) | ["train", "val", "tranval", "test"] | [Oxford 102 Flower](https://www.robots.ox.ac.uk/~vgg/data/flowers/102/) Dataset. |
|
||||
| [`Food101`](mmpretrain.datasets.Food101)(data_root[, split, pipeline, ...]) | ["train", "test"] | [Food101](https://data.vision.ee.ethz.ch/cvl/datasets_extra/food-101/) Dataset. |
|
||||
| [`MNIST`](mmpretrain.datasets.MNIST) (data_root[, split, pipeline, ...]) | ["train", "test"] | [MNIST](http://yann.lecun.com/exdb/mnist/) Dataset. |
|
||||
| [`OxfordIIITPet`](mmpretrain.datasets.OxfordIIITPet)(data_root[, split, pipeline, ...]) | ["tranval", test"] | [Oxford-IIIT Pets](https://www.robots.ox.ac.uk/~vgg/data/pets/) Dataset. |
|
||||
| [`Places205`](mmpretrain.datasets.Places205)(data_root[, pipeline, ...]) | - | [Places205](http://places.csail.mit.edu/downloadData.html) Dataset. |
|
||||
| [`StanfordCars`](mmpretrain.datasets.StanfordCars)(data_root[, split, pipeline, ...]) | ["train", "test"] | [Stanford Cars](https://ai.stanford.edu/~jkrause/cars/car_dataset.html) Dataset. |
|
||||
| [`SUN397`](mmpretrain.datasets.SUN397)(data_root[, split, pipeline, ...]) | ["train", "test"] | [SUN397](https://vision.princeton.edu/projects/2010/SUN/) Dataset. |
|
||||
| [`VOC`](mmpretrain.datasets.VOC)(data_root[, image_set_path, pipeline, ...]) | ["train", "val", "tranval", "test"] | [Pascal VOC](http://host.robots.ox.ac.uk/pascal/VOC/) Dataset. |
|
||||
|
||||
```python
|
||||
train_dataloader = dict(
|
||||
...
|
||||
# Training dataset configurations
|
||||
dataset=dict(
|
||||
type='CIFAR10',
|
||||
data_root='data/cifar10',
|
||||
test_mode=False,
|
||||
pipeline=...,
|
||||
)
|
||||
)
|
||||
|
||||
val_dataloader = dict(
|
||||
...
|
||||
# Validation dataset configurations
|
||||
dataset=dict(
|
||||
type='CIFAR10',
|
||||
data_root='data/cifar10',
|
||||
test_mode=True,
|
||||
pipeline=...,
|
||||
)
|
||||
)
|
||||
|
||||
test_dataloader = val_dataloader
|
||||
```
|
||||
|
||||
## MNIST
|
||||
|
||||
We support downloading the [MNIST](mmpretrain.datasets.MNIST) and [Fashion-MNIST](mmpretrain.datasets.FashionMNIST) datasets automatically, and you just need to specify the
|
||||
download folder in the `data_root` field. And please specify `test_mode=False` / `test_mode=True`
|
||||
to use training datasets or test datasets.
|
||||
|
||||
```python
|
||||
train_dataloader = dict(
|
||||
...
|
||||
# Training dataset configurations
|
||||
dataset=dict(
|
||||
type='MNIST',
|
||||
data_root='data/mnist',
|
||||
test_mode=False,
|
||||
pipeline=...,
|
||||
)
|
||||
)
|
||||
|
||||
val_dataloader = dict(
|
||||
...
|
||||
# Validation dataset configurations
|
||||
dataset=dict(
|
||||
type='MNIST',
|
||||
data_root='data/mnist',
|
||||
test_mode=True,
|
||||
pipeline=...,
|
||||
)
|
||||
)
|
||||
|
||||
test_dataloader = val_dataloader
|
||||
```
|
||||
Some dataset homepage links may be unavailable, and you can download datasets through [OpenDataLab](https://opendatalab.com/), such as [Stanford Cars](https://opendatalab.com/Stanford_Cars/download).
|
||||
|
||||
## OpenMMLab 2.0 Standard Dataset
|
||||
|
||||
|
|
|
|||
|
|
@ -143,15 +143,14 @@ ImageNet 有多个版本，但最常用的一个是 [ILSVRC 2012](http://www.ima
|
|||
- ILSVRC2012_img_train.tar (~138GB)
|
||||
- ILSVRC2012_img_val.tar (~6.3GB)
|
||||
3. 解压已下载的图片。
|
||||
4. 根据 [CustomDataset](#CustomDataset) 的格式约定重新组织图像文件
|
||||
|
||||
```{note}
|
||||
在 MMPretrain 中，所有配置文件默认使用文本标注文件的数据集。因此，如果希望使用子文件夹格式，请在配置文件中设置 `ann_file=''`。
|
||||
```
|
||||
### ImageNet数据集目录结构
|
||||
|
||||
### 子文件夹格式
|
||||
我们支持两种方式组织ImageNet数据集，子目录格式和文本注释文件格式。
|
||||
|
||||
重新组织数据集如下：
|
||||
#### 子文件夹格式
|
||||
|
||||
我们提供了一个样例，您可以从这个[链接](https://download.openmmlab.com/mmpretrain/datasets/imagenet_1k.zip)下载和解压。数据集的目录结构应如下所示：
|
||||
|
||||
```text
|
||||
data/imagenet/
|
||||
|
|
@ -174,37 +173,7 @@ data/imagenet/
|
|||
│ ├── ...
|
||||
```
|
||||
|
||||
然后，您可以使用具有以下配置的 [`ImageNet`](mmpretrain.datasets.ImageNet) 数据集：
|
||||
|
||||
```python
|
||||
train_dataloader = dict(
|
||||
...
|
||||
# 训练数据集配置
|
||||
dataset=dict(
|
||||
type='ImageNet',
|
||||
data_root='data/imagenet',
|
||||
data_prefix='train',
|
||||
ann_file='',
|
||||
pipeline=...,
|
||||
)
|
||||
)
|
||||
|
||||
val_dataloader = dict(
|
||||
...
|
||||
# 验证数据集配置
|
||||
dataset=dict(
|
||||
type='ImageNet',
|
||||
data_root='data/imagenet',
|
||||
data_prefix='val',
|
||||
ann_file='',
|
||||
pipeline=...,
|
||||
)
|
||||
)
|
||||
|
||||
test_dataloader = val_dataloader
|
||||
```
|
||||
|
||||
### 文本标注文件格式
|
||||
#### 文本标注文件格式
|
||||
|
||||
您可以从[此链接](https://download.openmmlab.com/mmclassification/datasets/imagenet/meta/caffe_ilsvrc12.tar.gz)下载并解压元数据，然后组织文件夹如下：
|
||||
|
||||
|
|
@ -232,7 +201,9 @@ data/imagenet/
|
|||
│ ├── ...
|
||||
```
|
||||
|
||||
然后，您可以使用具有以下配置的 [`ImageNet`](mmpretrain.datasets.ImageNet) 数据集：
|
||||
### 配置
|
||||
|
||||
当您的数据集以上述方式组织时，您可以使用具有以下配置的 [`ImageNet`](mmpretrain.datasets.ImageNet) 数据集：
|
||||
|
||||
```python
|
||||
train_dataloader = dict(
|
||||
|
|
@ -240,9 +211,8 @@ train_dataloader = dict(
|
|||
# 训练数据集配置
|
||||
dataset=dict(
|
||||
type='ImageNet',
|
||||
data_root='imagenet_folder',
|
||||
data_prefix='train/',
|
||||
ann_file='meta/train.txt',
|
||||
data_root='data/imagenet/',
|
||||
split='train',
|
||||
pipeline=...,
|
||||
)
|
||||
)
|
||||
|
|
@ -252,9 +222,8 @@ val_dataloader = dict(
|
|||
# 验证数据集配置
|
||||
dataset=dict(
|
||||
type='ImageNet',
|
||||
data_root='imagenet_folder',
|
||||
data_prefix='val/',
|
||||
ann_file='meta/val.txt',
|
||||
data_root='data/imagenet/',
|
||||
split='val',
|
||||
pipeline=...,
|
||||
)
|
||||
)
|
||||
|
|
@ -262,61 +231,27 @@ val_dataloader = dict(
|
|||
test_dataloader = val_dataloader
|
||||
```
|
||||
|
||||
## CIFAR
|
||||
## 支持的图像分类数据集
|
||||
|
||||
我们支持自动下载 [`CIFAR10`](mmpretrain.datasets.CIFAR10) 和 [`CIFAR100`](mmpretrain.datasets.CIFAR100) 数据集，您只需在 `data_root` 字段中指定下载文件夹即可。 并且通过指定 `test_mode=False` / `test_mode=True` 来使用训练数据集或测试数据集。
|
||||
| 数据集 | split | 主页 |
|
||||
| ----------------------------------------------------------------------------------- | ----------------------------------- | ---------------------------------------------------------------------------------- |
|
||||
| [`Calthch101`](mmpretrain.datasets.Caltech101)(data_root[, split, pipeline, ...]) | ["train", "test"] | [Caltech 101](https://data.caltech.edu/records/mzrjq-6wc02) 数据集 |
|
||||
| [`CIFAR10`](mmpretrain.datasets.CIFAR10)(data_root[, split, pipeline, ...]) | ["train", "test"] | [CIFAR10](https://www.cs.toronto.edu/~kriz/cifar.html) 数据集 |
|
||||
| [`CIFAR100`](mmpretrain.datasets.CIFAR100)(data_root[, split, pipeline, ...]) | ["train", "test"] | [CIFAR100](https://www.cs.toronto.edu/~kriz/cifar.html) 数据集 |
|
||||
| [`CUB`](mmpretrain.datasets.CUB)(data_root[, split, pipeline, ...]) | ["train", "test"] | [CUB-200-2011](http://www.vision.caltech.edu/datasets/cub_200_2011/) 数据集 |
|
||||
| [`DTD`](mmpretrain.datasets.DTD)(data_root[, split, pipeline, ...]) | ["train", "val", "tranval", "test"] | [Describable Texture Dataset (DTD)](https://www.robots.ox.ac.uk/~vgg/data/dtd/) 数据集 |
|
||||
| [`FashionMNIST`](mmpretrain.datasets.FashionMNIST) (data_root[, split, pipeline, ...]) | ["train", "test"] | [Fashion-MNIST](https://github.com/zalandoresearch/fashion-mnist) 数据集 |
|
||||
| [`FGVCAircraft`](mmpretrain.datasets.FGVCAircraft)(data_root[, split, pipeline, ...]) | ["train", "val", "tranval", "test"] | [FGVC Aircraft](https://www.robots.ox.ac.uk/~vgg/data/fgvc-aircraft/) 数据集 |
|
||||
| [`Flowers102`](mmpretrain.datasets.Flowers102)(data_root[, split, pipeline, ...]) | ["train", "val", "tranval", "test"] | [Oxford 102 Flower](https://www.robots.ox.ac.uk/~vgg/data/flowers/102/) 数据集 |
|
||||
| [`Food101`](mmpretrain.datasets.Food101)(data_root[, split, pipeline, ...]) | ["train", "test"] | [Food101](https://data.vision.ee.ethz.ch/cvl/datasets_extra/food-101/) 数据集 |
|
||||
| [`MNIST`](mmpretrain.datasets.MNIST) (data_root[, split, pipeline, ...]) | ["train", "test"] | [MNIST](http://yann.lecun.com/exdb/mnist/) 数据集 |
|
||||
| [`OxfordIIITPet`](mmpretrain.datasets.OxfordIIITPet)(data_root[, split, pipeline, ...]) | ["tranval", test"] | [Oxford-IIIT Pets](https://www.robots.ox.ac.uk/~vgg/data/pets/) 数据集 |
|
||||
| [`Places205`](mmpretrain.datasets.Places205)(data_root[, pipeline, ...]) | - | [Places205](http://places.csail.mit.edu/downloadData.html) 数据集 |
|
||||
| [`StanfordCars`](mmpretrain.datasets.StanfordCars)(data_root[, split, pipeline, ...]) | ["train", "test"] | [StanfordCars](https://ai.stanford.edu/~jkrause/cars/car_dataset.html) 数据集 |
|
||||
| [`SUN397`](mmpretrain.datasets.SUN397)(data_root[, split, pipeline, ...]) | ["train", "test"] | [SUN397](https://vision.princeton.edu/projects/2010/SUN/) 数据集 |
|
||||
| [`VOC`](mmpretrain.datasets.VOC)(data_root[, image_set_path, pipeline, ...]) | ["train", "val", "tranval", "test"] | [Pascal VOC](http://host.robots.ox.ac.uk/pascal/VOC/) 数据集 |
|
||||
|
||||
```python
|
||||
train_dataloader = dict(
|
||||
...
|
||||
# 训练数据集配置
|
||||
dataset=dict(
|
||||
type='CIFAR10',
|
||||
data_root='data/cifar10',
|
||||
test_mode=False,
|
||||
pipeline=...,
|
||||
)
|
||||
)
|
||||
val_dataloader = dict(
|
||||
...
|
||||
# 验证数据集配置
|
||||
dataset=dict(
|
||||
type='CIFAR10',
|
||||
data_root='data/cifar10',
|
||||
test_mode=True,
|
||||
pipeline=...,
|
||||
)
|
||||
)
|
||||
test_dataloader = val_dataloader
|
||||
```
|
||||
|
||||
## MNIST
|
||||
|
||||
我们支持自动下载 [MNIST](mmpretrain.datasets.MNIST) 和 [Fashion-MNIST](mmpretrain.datasets.FashionMNIST) 数据集，您只需指定 `data_root` 字段中的下载路径即可。 并且通过指定 `test_mode=False` / `test_mode=True` 来使用训练数据集或测试数据集。
|
||||
|
||||
```python
|
||||
train_dataloader = dict(
|
||||
...
|
||||
# 训练数据集配置
|
||||
dataset=dict(
|
||||
type='MNIST',
|
||||
data_root='data/mnist',
|
||||
test_mode=False,
|
||||
pipeline=...,
|
||||
)
|
||||
)
|
||||
val_dataloader = dict(
|
||||
...
|
||||
# 验证数据集配置
|
||||
dataset=dict(
|
||||
type='MNIST',
|
||||
data_root='data/mnist',
|
||||
test_mode=True,
|
||||
pipeline=...,
|
||||
)
|
||||
)
|
||||
test_dataloader = val_dataloader
|
||||
```
|
||||
有些数据集主页链接可能已经失效，您可以通过[OpenDataLab](https://opendatalab.com/)下载数据集，例如 [Stanford Cars](https://opendatalab.com/Stanford_Cars/download)数据集。
|
||||
|
||||
## OpenMMLab 2.0 标准数据集
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
# Copyright (c) OpenMMLab. All rights reserved.
|
||||
from typing import Optional, Union
|
||||
from typing import List, Optional, Union
|
||||
|
||||
from mmengine import fileio
|
||||
from mmengine.logging import MMLogger
|
||||
|
||||
from mmpretrain.registry import DATASETS
|
||||
|
|
@ -12,18 +13,79 @@ from .custom import CustomDataset
|
|||
class ImageNet(CustomDataset):
|
||||
"""`ImageNet <http://www.image-net.org>`_ Dataset.
|
||||
|
||||
The dataset supports two kinds of annotation format. More details can be
|
||||
found in :class:`CustomDataset`.
|
||||
The dataset supports two kinds of directory format,
|
||||
|
||||
::
|
||||
|
||||
imagenet
|
||||
├── train
|
||||
│ ├──class_x
|
||||
| | ├── x1.jpg
|
||||
| | ├── x2.jpg
|
||||
| | └── ...
|
||||
│ ├── class_y
|
||||
| | ├── y1.jpg
|
||||
| | ├── y2.jpg
|
||||
| | └── ...
|
||||
| └── ...
|
||||
├── val
|
||||
│ ├──class_x
|
||||
| | └── ...
|
||||
│ ├── class_y
|
||||
| | └── ...
|
||||
| └── ...
|
||||
└── test
|
||||
├── test1.jpg
|
||||
├── test2.jpg
|
||||
└── ...
|
||||
|
||||
or ::
|
||||
|
||||
imagenet
|
||||
├── train
|
||||
│ ├── x1.jpg
|
||||
│ ├── y1.jpg
|
||||
│ └── ...
|
||||
├── val
|
||||
│ ├── x3.jpg
|
||||
│ ├── y3.jpg
|
||||
│ └── ...
|
||||
├── test
|
||||
│ ├── test1.jpg
|
||||
│ ├── test2.jpg
|
||||
│ └── ...
|
||||
└── meta
|
||||
├── train.txt
|
||||
└── val.txt
|
||||
|
||||
|
||||
Args:
|
||||
data_root (str): The root directory for ``data_prefix`` and
|
||||
``ann_file``. Defaults to ''.
|
||||
split (str): The dataset split, supports "train", "val" and "test".
|
||||
Default to ''.
|
||||
data_prefix (str | dict): Prefix for training data. Defaults to ''.
|
||||
ann_file (str): Annotation file path. Defaults to ''.
|
||||
metainfo (dict, optional): Meta information for dataset, such as class
|
||||
information. Defaults to None.
|
||||
**kwargs: Other keyword arguments in :class:`CustomDataset` and
|
||||
:class:`BaseDataset`.
|
||||
|
||||
|
||||
Examples:
|
||||
>>> from mmpretrain.datasets import ImageNet
|
||||
>>> train_dataset = ImageNet(data_root='data/imagenet', split='train')
|
||||
>>> train_dataset
|
||||
Dataset ImageNet
|
||||
Number of samples: 1281167
|
||||
Number of categories: 1000
|
||||
Root of dataset: data/imagenet
|
||||
>>> test_dataset = ImageNet(data_root='data/imagenet', split='val')
|
||||
>>> test_dataset
|
||||
Dataset ImageNet
|
||||
Number of samples: 50000
|
||||
Number of categories: 1000
|
||||
Root of dataset: data/imagenet
|
||||
""" # noqa: E501
|
||||
|
||||
IMG_EXTENSIONS = ('.jpg', '.jpeg', '.png', '.ppm', '.bmp', '.pgm', '.tif')
|
||||
|
|
@ -31,11 +93,32 @@ class ImageNet(CustomDataset):
|
|||
|
||||
def __init__(self,
|
||||
data_root: str = '',
|
||||
split: str = '',
|
||||
data_prefix: Union[str, dict] = '',
|
||||
ann_file: str = '',
|
||||
metainfo: Optional[dict] = None,
|
||||
**kwargs):
|
||||
kwargs = {'extensions': self.IMG_EXTENSIONS, **kwargs}
|
||||
|
||||
if split:
|
||||
splits = ['train', 'val', 'test']
|
||||
assert split in splits, \
|
||||
f"The split must be one of {splits}, but get '{split}'"
|
||||
|
||||
if split == 'test':
|
||||
logger = MMLogger.get_current_instance()
|
||||
logger.info(
|
||||
'Since the ImageNet1k test set does not provide label'
|
||||
'annotations, `with_label` is set to False')
|
||||
kwargs['with_label'] = False
|
||||
|
||||
data_prefix = split if data_prefix == '' else data_prefix
|
||||
|
||||
if ann_file == '':
|
||||
_ann_path = fileio.join_path(data_root, 'meta', f'{split}.txt')
|
||||
if fileio.exists(_ann_path):
|
||||
ann_file = _ann_path
|
||||
|
||||
super().__init__(
|
||||
data_root=data_root,
|
||||
data_prefix=data_prefix,
|
||||
|
|
@ -43,14 +126,39 @@ class ImageNet(CustomDataset):
|
|||
metainfo=metainfo,
|
||||
**kwargs)
|
||||
|
||||
def extra_repr(self) -> List[str]:
|
||||
"""The extra repr information of the dataset."""
|
||||
body = [
|
||||
f'Root of dataset: \t{self.data_root}',
|
||||
]
|
||||
return body
|
||||
|
||||
|
||||
@DATASETS.register_module()
|
||||
class ImageNet21k(CustomDataset):
|
||||
"""ImageNet21k Dataset.
|
||||
|
||||
Since the dataset ImageNet21k is extremely big, cantains 21k+ classes
|
||||
Since the dataset ImageNet21k is extremely big, contains 21k+ classes
|
||||
and 1.4B files. We won't provide the default categories list. Please
|
||||
specify it from the ``classes`` argument.
|
||||
The dataset directory structure is as follows,
|
||||
|
||||
ImageNet21k dataset directory ::
|
||||
|
||||
imagenet21k
|
||||
├── train
|
||||
│ ├──class_x
|
||||
| | ├── x1.jpg
|
||||
| | ├── x2.jpg
|
||||
| | └── ...
|
||||
│ ├── class_y
|
||||
| | ├── y1.jpg
|
||||
| | ├── y2.jpg
|
||||
| | └── ...
|
||||
| └── ...
|
||||
└── meta
|
||||
└── train.txt
|
||||
|
||||
|
||||
Args:
|
||||
data_root (str): The root directory for ``data_prefix`` and
|
||||
|
|
@ -63,12 +171,22 @@ class ImageNet21k(CustomDataset):
|
|||
Defaults to False.
|
||||
**kwargs: Other keyword arguments in :class:`CustomDataset` and
|
||||
:class:`BaseDataset`.
|
||||
"""
|
||||
|
||||
Examples:
|
||||
>>> from mmpretrain.datasets import ImageNet21k
|
||||
>>> train_dataset = ImageNet21k(data_root='data/imagenet21k', split='train')
|
||||
>>> train_dataset
|
||||
Dataset ImageNet21k
|
||||
Number of samples: 14197088
|
||||
Annotation file: data/imagenet21k/meta/train.txt
|
||||
Prefix of images: data/imagenet21k/train
|
||||
""" # noqa: E501
|
||||
|
||||
IMG_EXTENSIONS = ('.jpg', '.jpeg', '.png', '.ppm', '.bmp', '.pgm', '.tif')
|
||||
|
||||
def __init__(self,
|
||||
data_root: str = '',
|
||||
split: str = '',
|
||||
data_prefix: Union[str, dict] = '',
|
||||
ann_file: str = '',
|
||||
metainfo: Optional[dict] = None,
|
||||
|
|
@ -79,6 +197,21 @@ class ImageNet21k(CustomDataset):
|
|||
'The `multi_label` option is not supported by now.')
|
||||
self.multi_label = multi_label
|
||||
|
||||
if split:
|
||||
splits = ['train']
|
||||
assert split in splits, \
|
||||
f"The split must be one of {splits}, but get '{split}'.\
|
||||
If you want to specify your own validation set or test set,\
|
||||
please set split to None."
|
||||
|
||||
self.split = split
|
||||
data_prefix = split if data_prefix == '' else data_prefix
|
||||
|
||||
if not ann_file:
|
||||
_ann_path = fileio.join_path(data_root, 'meta', f'{split}.txt')
|
||||
if fileio.exists(_ann_path):
|
||||
ann_file = _ann_path
|
||||
|
||||
logger = MMLogger.get_current_instance()
|
||||
|
||||
if not ann_file:
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import mmengine.dist as dist
|
|||
import numpy as np
|
||||
import torch
|
||||
from mmengine.fileio import LocalBackend, exists, get_file_backend, join_path
|
||||
from mmengine.logging import MMLogger
|
||||
|
||||
from mmpretrain.registry import DATASETS
|
||||
from .base_dataset import BaseDataset
|
||||
|
|
@ -23,13 +24,11 @@ class MNIST(BaseDataset):
|
|||
https://github.com/pytorch/vision/blob/master/torchvision/datasets/mnist.py
|
||||
|
||||
Args:
|
||||
data_prefix (str): Prefix for data.
|
||||
test_mode (bool): ``test_mode=True`` means in test phase.
|
||||
It determines to use the training set or test set.
|
||||
data_root (str): The root directory of the MNIST Dataset.
|
||||
split (str, optional): The dataset split, supports "train" and "test".
|
||||
Default to "train".
|
||||
metainfo (dict, optional): Meta information for dataset, such as
|
||||
categories information. Defaults to None.
|
||||
data_root (str): The root directory for ``data_prefix``.
|
||||
Defaults to ''.
|
||||
download (bool): Whether to download the dataset if not exists.
|
||||
Defaults to True.
|
||||
**kwargs: Other keyword arguments in :class:`BaseDataset`.
|
||||
|
|
@ -49,12 +48,29 @@ class MNIST(BaseDataset):
|
|||
METAINFO = {'classes': MNIST_CATEGORITES}
|
||||
|
||||
def __init__(self,
|
||||
data_prefix: str,
|
||||
test_mode: bool,
|
||||
metainfo: Optional[dict] = None,
|
||||
data_root: str = '',
|
||||
split: str = 'train',
|
||||
metainfo: Optional[dict] = None,
|
||||
download: bool = True,
|
||||
data_prefix: str = '',
|
||||
test_mode: bool = False,
|
||||
**kwargs):
|
||||
|
||||
splits = ['train', 'test']
|
||||
assert split in splits, \
|
||||
f"The split must be one of {splits}, but get '{split}'"
|
||||
self.split = split
|
||||
|
||||
# To handle the BC-breaking
|
||||
if split == 'train' and test_mode:
|
||||
logger = MMLogger.get_current_instance()
|
||||
logger.warning('split="train" but test_mode=True. '
|
||||
'The training set will be used.')
|
||||
|
||||
if not data_root and not data_prefix:
|
||||
raise RuntimeError('Please set ``data_root`` to'
|
||||
'specify the dataset path')
|
||||
|
||||
self.download = download
|
||||
super().__init__(
|
||||
# The MNIST dataset doesn't need specify annotation file
|
||||
|
|
@ -138,13 +154,11 @@ class FashionMNIST(MNIST):
|
|||
Dataset.
|
||||
|
||||
Args:
|
||||
data_prefix (str): Prefix for data.
|
||||
test_mode (bool): ``test_mode=True`` means in test phase.
|
||||
It determines to use the training set or test set.
|
||||
data_root (str): The root directory of the MNIST Dataset.
|
||||
split (str, optional): The dataset split, supports "train" and "test".
|
||||
Default to "train".
|
||||
metainfo (dict, optional): Meta information for dataset, such as
|
||||
categories information. Defaults to None.
|
||||
data_root (str): The root directory for ``data_prefix``.
|
||||
Defaults to ''.
|
||||
download (bool): Whether to download the dataset if not exists.
|
||||
Defaults to True.
|
||||
**kwargs: Other keyword arguments in :class:`BaseDataset`.
|
||||
|
|
|
|||
|
|
@ -7,130 +7,12 @@ from mmpretrain.registry import DATASETS
|
|||
from .base_dataset import BaseDataset
|
||||
from .categories import SUN397_CATEGORIES
|
||||
|
||||
# Note that some images are not a jpg file although the name ends
|
||||
# with jpg and therefore cannot be read properly. So we provide
|
||||
# a list to skip these files.
|
||||
INVALID = [
|
||||
'/a/assembly_line/sun_ajckcfldgdrdjogj.jpg',
|
||||
'/a/auto_factory/sun_apfsprenzdnzbhmt.jpg',
|
||||
'/b/baggage_claim/sun_avittiqqaiibgcau.jpg',
|
||||
'/b/batters_box/sun_alqlfpgtbgggezyr.jpg',
|
||||
'/b/bow_window/indoor/sun_ahsholsagvlrsboa.jpg',
|
||||
'/b/bow_window/indoor/sun_aioomcoujmmcxkkx.jpg',
|
||||
'/b/bow_window/outdoor/sun_atgtjdpqikjmllth.jpg',
|
||||
'/c/carrousel/sun_atsgphqympojgxnc.jpg',
|
||||
'/c/carrousel/sun_auzitjuirwolazns.jpg',
|
||||
'/c/church/outdoor/sun_boagasgfltequmal.jpg',
|
||||
'/c/church/outdoor/sun_brhmnwzzbkphcvfo.jpg',
|
||||
'/c/church/outdoor/sun_byjkqzybxpjnuofa.jpg',
|
||||
'/c/corridor/sun_aznefxvocwpgimko.jpg',
|
||||
'/d/dentists_office/sun_aaefsoauqlcsihou.jpg',
|
||||
'/d/diner/indoor/sun_apswilaujhntrybg.jpg',
|
||||
'/e/elevator/door/sun_aaudobqlphijkjdv.jpg',
|
||||
'/f/fastfood_restaurant/sun_axeniwtesffxqedr.jpg',
|
||||
'/f/fire_station/sun_bjyapttwilyyuxqm.jpg',
|
||||
'/f/fountain/sun_axgmpbdyvqhtkhee.jpg',
|
||||
'/h/hospital_room/sun_ahokhhxjiclpxqqa.jpg',
|
||||
'/o/oast_house/sun_bqsrrygxyrutgjve.jpg',
|
||||
'/r/restaurant_patio/sun_aurwypviprwycame.jpg',
|
||||
'/s/ski_resort/sun_bplmntyzoiobcqhp.jpg',
|
||||
'/w/wine_cellar/bottle_storage/sun_afmzwxkzmxkbamqi.jpg',
|
||||
'/w/wine_cellar/bottle_storage/sun_ahyymswdjejrbhyb.jpg',
|
||||
'/w/wine_cellar/bottle_storage/sun_avnttpxamufejbfe.jpg',
|
||||
'/a/archive/sun_awgsrbljlsvhqjij.jpg',
|
||||
'/a/art_school/sun_aabogqsjulyvmcse.jpg',
|
||||
'/a/art_school/sun_apnzojafyvkariue.jpg',
|
||||
'/b/ball_pit/sun_atjhwqngtoeuwhso.jpg',
|
||||
'/b/bow_window/indoor/sun_asxvsqbexmmtqmht.jpg',
|
||||
'/b/bow_window/indoor/sun_abeugxecxrwzmffp.jpg',
|
||||
'/b/bow_window/outdoor/sun_auwcqhrtzkgihvlv.jpg',
|
||||
'/b/bow_window/outdoor/sun_apnvdyecnjjmcuhi.jpg',
|
||||
'/c/childs_room/sun_alggivksjwwiklmt.jpg',
|
||||
'/c/control_tower/outdoor/sun_avbcxakrvpomqdgr.jpg',
|
||||
'/d/diner/indoor/sun_ajmzozstvsxisvgx.jpg',
|
||||
'/e/elevator/door/sun_aaqsyluqbluugqgy.jpg',
|
||||
'/f/fastfood_restaurant/sun_aevchxlxoruhxgrb.jpg',
|
||||
'/f/firing_range/indoor/sun_affrzvahwjorpalo.jpg',
|
||||
'/f/formal_garden/sun_bjvrlaeatjufekft.jpg',
|
||||
'/g/garage/indoor/sun_akbocuwclkxqlofx.jpg',
|
||||
'/g/greenhouse/indoor/sun_addirvgtxfbndlwf.jpg',
|
||||
'/k/kindergarden_classroom/sun_ajtpaahilrqzarri.jpg',
|
||||
'/l/laundromat/sun_afrrjykuhhlwiwun.jpg',
|
||||
'/m/music_studio/sun_bsntklkmwqgnjrjj.jpg',
|
||||
'/t/track/outdoor/sun_aophkoiosslinihb.jpg',
|
||||
'/a/archive/sun_aegmzltkiwyevpwa.jpg',
|
||||
'/a/auto_factory/sun_aybymzvbxgvcrwgn.jpg',
|
||||
'/b/baggage_claim/sun_atpmiqmnxjpgqsxi.jpg',
|
||||
'/b/baggage_claim/sun_ajffcdpsvgqfzoxx.jpg',
|
||||
'/b/bamboo_forest/sun_ausmxphosyahoyjo.jpg',
|
||||
'/b/batters_box/sun_aaeheulsicxtxnbu.jpg',
|
||||
'/c/carrousel/sun_arjrjcxemhttubqz.jpg',
|
||||
'/c/chicken_coop/outdoor/sun_abcegmmdbizqkpgh.jpg',
|
||||
'/c/control_tower/outdoor/sun_axhjfpkxdvqdfkyr.jpg',
|
||||
'/d/diner/indoor/sun_apaotiublwqeowck.jpg',
|
||||
'/f/fastfood_restaurant/sun_anexashcgmxdbmxq.jpg',
|
||||
'/l/landing_deck/sun_aizahnjfkuurjibw.jpg',
|
||||
'/n/nuclear_power_plant/outdoor/sun_aoblfvgyleweqanr.jpg',
|
||||
'/w/waiting_room/sun_aicytusmthfvqcwc.jpg',
|
||||
'/b/bow_window/indoor/sun_asmvdfnjlulewkpr.jpg',
|
||||
'/b/bus_interior/sun_adhktvidwzmodeou.jpg',
|
||||
'/c/catacomb/sun_algnawesgjzzmcqd.jpg',
|
||||
'/c/church/outdoor/sun_baihxlseimcsdhdx.jpg',
|
||||
'/d/diner/indoor/sun_agoyalzcawgxodbm.jpg',
|
||||
'/e/elevator_shaft/sun_awaitimkinrjaybl.jpg',
|
||||
'/f/fastfood_restaurant/sun_aplvzfbmtqtbsvbx.jpg',
|
||||
'/g/greenhouse/indoor/sun_bkccvyfpwetwjuhk.jpg',
|
||||
'/c/car_interior/backseat/sun_adexwfoqdyhowxpu.jpg',
|
||||
'/c/church/outdoor/sun_blmmweiumednscuf.jpg',
|
||||
'/f/fire_station/sun_bibntbsuunbsdrum.jpg',
|
||||
'/g/game_room/sun_aopfaqlllpvzhrak.jpg',
|
||||
'/u/underwater/coral_reef/sun_biiueajvszaxqopo.jpg',
|
||||
'/a/airplane_cabin/sun_arqyikigkyfpegug.jpg',
|
||||
'/b/badminton_court/indoor/sun_amppvxecgtjpfold.jpg',
|
||||
'/c/carrousel/sun_anxtrtieimkpmhvk.jpg',
|
||||
'/c/computer_room/sun_aebgvpgtwoqbfyvl.jpg',
|
||||
'/f/fire_escape/sun_atbraxuwwlvdoolv.jpg',
|
||||
'/k/kasbah/sun_abxkkoielpavsouu.jpg',
|
||||
'/t/tower/sun_bccqnzcvqkiwicjt.jpg',
|
||||
'/a/archive/sun_afngadshxudodkct.jpg',
|
||||
'/b/bow_window/indoor/sun_awnrlipyxpgxxgxz.jpg',
|
||||
'/c/control_tower/outdoor/sun_arohngcbtsvbthho.jpg',
|
||||
'/f/fire_station/sun_brbskkfgghbfvgkk.jpg',
|
||||
'/r/restaurant_patio/sun_amjfbqzfgxarrpec.jpg',
|
||||
'/v/vineyard/sun_bdxhnbgbnolddswz.jpg',
|
||||
'/b/baggage_claim/sun_axrtsmillrglugia.jpg',
|
||||
'/d/diner/indoor/sun_alaqevbwpjaqqdqz.jpg',
|
||||
'/l/landing_deck/sun_acodgoamhgnnbmvr.jpg',
|
||||
'/c/carrousel/sun_adsafgyrinnekycc.jpg',
|
||||
'/c/church/outdoor/sun_bzqhuwshtdgakkay.jpg',
|
||||
'/c/closet/sun_absahzamlrylkxyn.jpg',
|
||||
'/f/fire_escape/sun_acdthenaosuqcoqn.jpg',
|
||||
'/b/butchers_shop/sun_asrdgbefoszenfex.jpg',
|
||||
'/c/church/outdoor/sun_bzfyucfrdigaqneg.jpg',
|
||||
'/c/church/outdoor/sun_byzxhknqrejdajxi.jpg',
|
||||
'/c/cockpit/sun_ajkulpqauavrmxae.jpg',
|
||||
'/l/living_room/sun_aefoqbeatyufobtx.jpg',
|
||||
'/s/supermarket/sun_attvxbzocurnddbz.jpg',
|
||||
'/c/closet/sun_aqnutmwfkypmrnfy.jpg',
|
||||
'/f/fire_station/sun_bttrtzktpbymxkmf.jpg',
|
||||
'/s/shopping_mall/indoor/sun_avwzjsijaxnwuzjx.jpg',
|
||||
'/w/windmill/sun_blvczkyqbmabzeej.jpg',
|
||||
'/c/chicken_coop/outdoor/sun_amaonsnnkskxwmrj.jpg',
|
||||
'/s/swimming_pool/outdoor/sun_bslaihiqlhfewtzn.jpg',
|
||||
'/u/underwater/coral_reef/sun_bhcrnmvbgnkvcvkr.jpg',
|
||||
'/d/dining_room/sun_azlxdhiajwrhaivq.jpg',
|
||||
'/c/church/outdoor/sun_bnunxbznqnvgeykx.jpg',
|
||||
'/c/corridor/sun_aspwpqqlcwzfanvl.jpg',
|
||||
'/r/restaurant_patio/sun_awcbpizjbudjvrhs.jpg',
|
||||
'/b/ball_pit/sun_avdnmemjrgrbkwjm.jpg',
|
||||
]
|
||||
|
||||
|
||||
@DATASETS.register_module()
|
||||
class SUN397(BaseDataset):
|
||||
"""The SUN397 Dataset.
|
||||
|
||||
Support the `SUN397 Dataset <https://data.vision.ee.ethz.ch/cvl/datasets_extra/food-101/>`_ Dataset.
|
||||
Support the `SUN397 Dataset <https://vision.princeton.edu/projects/2010/SUN/>`_ Dataset.
|
||||
After downloading and decompression, the dataset directory structure is as follows.
|
||||
|
||||
SUN397 dataset directory: ::
|
||||
|
|
@ -166,15 +48,26 @@ class SUN397(BaseDataset):
|
|||
>>> train_dataset = SUN397(data_root='data/SUN397', split='train')
|
||||
>>> train_dataset
|
||||
Dataset SUN397
|
||||
Number of samples: 19824
|
||||
Number of samples: 19850
|
||||
Number of categories: 397
|
||||
Root of dataset: data/SUN397
|
||||
>>> test_dataset = SUN397(data_root='data/SUN397', split='test')
|
||||
>>> test_dataset
|
||||
Dataset SUN397
|
||||
Number of samples: 19829
|
||||
Number of samples: 19850
|
||||
Number of categories: 397
|
||||
Root of dataset: data/SUN397
|
||||
|
||||
**Note that some images are not a jpg file although the name ends with ".jpg".
|
||||
The backend of SUN397 should be "pillow" as below to read these images properly,**
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
pipeline = [
|
||||
dict(type='LoadImageFromFile', imdecode_backend='pillow'),
|
||||
dict(type='RandomResizedCrop', scale=224),
|
||||
dict(type='PackInputs')
|
||||
]
|
||||
""" # noqa: E501
|
||||
|
||||
METAINFO = {'classes': SUN397_CATEGORIES}
|
||||
|
|
@ -206,8 +99,6 @@ class SUN397(BaseDataset):
|
|||
pairs = list_from_file(self.ann_file)
|
||||
data_list = []
|
||||
for pair in pairs:
|
||||
if pair in INVALID:
|
||||
continue
|
||||
img_path = self.backend.join_path(self.img_prefix, pair[1:])
|
||||
items = pair.split('/')
|
||||
class_name = '_'.join(items[2:-1])
|
||||
|
|
@ -217,6 +108,15 @@ class SUN397(BaseDataset):
|
|||
|
||||
return data_list
|
||||
|
||||
def __getitem__(self, idx: int) -> dict:
|
||||
try:
|
||||
return super().__getitem__(idx)
|
||||
except AttributeError:
|
||||
raise RuntimeError(
|
||||
'Some images in the SUN397 dataset are not a jpg file '
|
||||
'although the name ends with ".jpg". The backend of SUN397 '
|
||||
'should be "pillow" to read these images properly.')
|
||||
|
||||
def extra_repr(self) -> List[str]:
|
||||
"""The extra repr information of the dataset."""
|
||||
body = [
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import xml.etree.ElementTree as ET
|
|||
from typing import List, Optional, Union
|
||||
|
||||
from mmengine import get_file_backend, list_from_file
|
||||
from mmengine.logging import MMLogger
|
||||
|
||||
from mmpretrain.registry import DATASETS
|
||||
from .base_dataset import expanduser
|
||||
|
|
@ -18,16 +19,22 @@ class VOC(MultiLabelDataset):
|
|||
|
||||
VOC dataset directory: ::
|
||||
|
||||
VOC2007 (data_root)/
|
||||
├── JPEGImages (data_prefix['img_path'])
|
||||
VOC2007
|
||||
├── JPEGImages
|
||||
│ ├── xxx.jpg
|
||||
│ ├── xxy.jpg
|
||||
│ └── ...
|
||||
├── Annotations (data_prefix['ann_path'])
|
||||
├── Annotations
|
||||
│ ├── xxx.xml
|
||||
│ ├── xxy.xml
|
||||
│ └── ...
|
||||
└── ImageSets (directory contains various imageset file)
|
||||
└── ImageSets
|
||||
└── Main
|
||||
├── train.txt
|
||||
├── val.txt
|
||||
├── trainval.txt
|
||||
├── test.txt
|
||||
└── ...
|
||||
|
||||
Extra difficult label is in VOC annotations, we will use
|
||||
`gt_label_difficult` to record the difficult labels in each sample
|
||||
|
|
@ -37,42 +44,85 @@ class VOC(MultiLabelDataset):
|
|||
|
||||
Args:
|
||||
data_root (str): The root directory for VOC dataset.
|
||||
image_set_path (str): The path of image set, The file which
|
||||
split (str, optional): The dataset split, supports "train",
|
||||
"val", "trainval", and "test". Default to "trainval".
|
||||
image_set_path (str, optional): The path of image set, The file which
|
||||
lists image ids of the sub dataset, and this path is relative
|
||||
to ``data_root``.
|
||||
to ``data_root``. Default to ''.
|
||||
data_prefix (dict): Prefix for data and annotation, keyword
|
||||
'img_path' and 'ann_path' can be set. Defaults to be
|
||||
``dict(img_path='JPEGImages', ann_path='Annotations')``.
|
||||
test_mode (bool): ``test_mode=True`` means in test phase.
|
||||
It determines to use the training set or test set.
|
||||
metainfo (dict, optional): Meta information for dataset, such as
|
||||
categories information. Defaults to None.
|
||||
**kwargs: Other keyword arguments in :class:`BaseDataset`.
|
||||
|
||||
Examples:
|
||||
>>> from mmpretrain.datasets import VOC
|
||||
>>> train_dataset = VOC(data_root='data/VOC2007', split='trainval')
|
||||
>>> train_dataset
|
||||
Dataset VOC
|
||||
Number of samples: 5011
|
||||
Number of categories: 20
|
||||
Prefix of dataset: data/VOC2007
|
||||
Path of image set: data/VOC2007/ImageSets/Main/trainval.txt
|
||||
Prefix of images: data/VOC2007/JPEGImages
|
||||
Prefix of annotations: data/VOC2007/Annotations
|
||||
>>> test_dataset = VOC(data_root='data/VOC2007', split='test')
|
||||
>>> test_dataset
|
||||
Dataset VOC
|
||||
Number of samples: 4952
|
||||
Number of categories: 20
|
||||
Prefix of dataset: data/VOC2007
|
||||
Path of image set: data/VOC2007/ImageSets/Main/test.txt
|
||||
Prefix of images: data/VOC2007/JPEGImages
|
||||
Prefix of annotations: data/VOC2007/Annotations
|
||||
""" # noqa: E501
|
||||
|
||||
METAINFO = {'classes': VOC2007_CATEGORIES}
|
||||
|
||||
def __init__(self,
|
||||
data_root: str,
|
||||
image_set_path: str,
|
||||
split: str = 'trainval',
|
||||
image_set_path: str = '',
|
||||
data_prefix: Union[str, dict] = dict(
|
||||
img_path='JPEGImages', ann_path='Annotations'),
|
||||
test_mode: bool = False,
|
||||
metainfo: Optional[dict] = None,
|
||||
**kwargs):
|
||||
|
||||
self.backend = get_file_backend(data_root, enable_singleton=True)
|
||||
|
||||
if split:
|
||||
splits = ['train', 'val', 'trainval', 'test']
|
||||
assert split in splits, \
|
||||
f"The split must be one of {splits}, but get '{split}'"
|
||||
self.split = split
|
||||
|
||||
if not data_prefix:
|
||||
data_prefix = dict(
|
||||
img_path='JPEGImages', ann_path='Annotations')
|
||||
if not image_set_path:
|
||||
image_set_path = self.backend.join_path(
|
||||
'ImageSets', 'Main', f'{split}.txt')
|
||||
|
||||
# To handle the BC-breaking
|
||||
if (split == 'train' or split == 'trainval') and test_mode:
|
||||
logger = MMLogger.get_current_instance()
|
||||
logger.warning(f'split="{split}" but test_mode=True. '
|
||||
f'The {split} set will be used.')
|
||||
|
||||
if isinstance(data_prefix, str):
|
||||
data_prefix = dict(img_path=expanduser(data_prefix))
|
||||
assert isinstance(data_prefix, dict) and 'img_path' in data_prefix, \
|
||||
'`data_prefix` must be a dict with key img_path'
|
||||
|
||||
if test_mode is False:
|
||||
if (split and split not in ['val', 'test']) or not test_mode:
|
||||
assert 'ann_path' in data_prefix and data_prefix[
|
||||
'ann_path'] is not None, \
|
||||
'"ann_path" must be set in `data_prefix` if `test_mode` is' \
|
||||
' False.'
|
||||
'"ann_path" must be set in `data_prefix`' \
|
||||
'when validation or test set is used.'
|
||||
|
||||
self.data_root = data_root
|
||||
self.backend = get_file_backend(data_root, enable_singleton=True)
|
||||
self.image_set_path = self.backend.join_path(data_root, image_set_path)
|
||||
|
||||
super().__init__(
|
||||
|
|
|
|||
|
|
@ -290,41 +290,169 @@ class TestCustomDataset(TestBaseDataset):
|
|||
class TestImageNet(TestCustomDataset):
|
||||
DATASET_TYPE = 'ImageNet'
|
||||
|
||||
DEFAULT_ARGS = dict(data_root=ASSETS_ROOT, ann_file='ann.txt')
|
||||
@classmethod
|
||||
def setUpClass(cls) -> None:
|
||||
super().setUpClass()
|
||||
|
||||
tmpdir = tempfile.TemporaryDirectory()
|
||||
cls.tmpdir = tmpdir
|
||||
cls.root = tmpdir.name
|
||||
cls.meta_folder = 'meta'
|
||||
cls.train_file = 'train.txt'
|
||||
cls.val_file = 'val.txt'
|
||||
cls.test_file = 'test.txt'
|
||||
cls.categories = ['cat', 'dog']
|
||||
|
||||
os.mkdir(osp.join(cls.root, cls.meta_folder))
|
||||
|
||||
cls.DEFAULT_ARGS = dict(data_root=cls.root, split='train')
|
||||
|
||||
with open(osp.join(cls.root, cls.meta_folder, cls.train_file),
|
||||
'w') as f:
|
||||
f.write('\n'.join([
|
||||
'1.jpg 0',
|
||||
'2.jpg 1',
|
||||
'3.jpg 1',
|
||||
]))
|
||||
|
||||
with open(osp.join(cls.root, cls.meta_folder, cls.val_file), 'w') as f:
|
||||
f.write('\n'.join([
|
||||
'11.jpg 0',
|
||||
'22.jpg 1',
|
||||
]))
|
||||
|
||||
with open(osp.join(cls.root, cls.meta_folder, cls.test_file),
|
||||
'w') as f:
|
||||
f.write('\n'.join([
|
||||
'aa.jpg',
|
||||
'bb.jpg',
|
||||
]))
|
||||
|
||||
def test_initialize(self):
|
||||
super().test_initialize()
|
||||
|
||||
dataset_class = DATASETS.get(self.DATASET_TYPE)
|
||||
|
||||
# Test invalid split
|
||||
with self.assertRaisesRegex(AssertionError, 'The split must be'):
|
||||
cfg = {**self.DEFAULT_ARGS}
|
||||
cfg['split'] = 'unknown'
|
||||
dataset_class(**cfg)
|
||||
|
||||
# Test valid splits
|
||||
splits = ['train', 'val']
|
||||
for split in splits:
|
||||
cfg = {**self.DEFAULT_ARGS}
|
||||
cfg['split'] = split
|
||||
cfg['classes'] = self.categories
|
||||
dataset = dataset_class(**cfg)
|
||||
self.assertEqual(dataset.data_root, self.root)
|
||||
|
||||
# Test split="test"
|
||||
cfg = {**self.DEFAULT_ARGS}
|
||||
cfg['split'] = 'test'
|
||||
logger = MMLogger.get_current_instance()
|
||||
with self.assertLogs(logger, 'INFO') as log:
|
||||
dataset = dataset_class(**cfg)
|
||||
self.assertFalse(dataset.with_label)
|
||||
self.assertIn('Since the ImageNet1k test set', log.output[0])
|
||||
|
||||
def test_load_data_list(self):
|
||||
dataset_class = DATASETS.get(self.DATASET_TYPE)
|
||||
|
||||
# test classes number
|
||||
cfg = {
|
||||
**self.DEFAULT_ARGS,
|
||||
'data_prefix': ASSETS_ROOT,
|
||||
'ann_file': '',
|
||||
}
|
||||
with self.assertRaisesRegex(
|
||||
AssertionError, r"\(2\) doesn't match .* classes \(1000\)"):
|
||||
dataset_class(**cfg)
|
||||
# Test default behavior
|
||||
dataset = dataset_class(**self.DEFAULT_ARGS)
|
||||
self.assertEqual(len(dataset), 3)
|
||||
|
||||
data_info = dataset[0]
|
||||
self.assertEqual(data_info['img_path'],
|
||||
osp.join(self.root, 'train', '1.jpg'))
|
||||
self.assertEqual(data_info['gt_label'], 0)
|
||||
|
||||
# Test split="val"
|
||||
cfg = {**self.DEFAULT_ARGS, 'split': 'val'}
|
||||
dataset = dataset_class(**cfg)
|
||||
self.assertEqual(len(dataset), 2)
|
||||
|
||||
data_info = dataset[0]
|
||||
self.assertEqual(data_info['img_path'],
|
||||
osp.join(self.root, 'val', '11.jpg'))
|
||||
self.assertEqual(data_info['gt_label'], 0)
|
||||
|
||||
# Test split="test"
|
||||
cfg = {**self.DEFAULT_ARGS, 'split': 'test'}
|
||||
dataset = dataset_class(**cfg)
|
||||
self.assertEqual(len(dataset), 2)
|
||||
|
||||
data_info = dataset[0]
|
||||
self.assertEqual(data_info['img_path'],
|
||||
osp.join(self.root, 'test', 'aa.jpg'))
|
||||
|
||||
# test override classes
|
||||
cfg = {
|
||||
**self.DEFAULT_ARGS,
|
||||
'data_prefix': ASSETS_ROOT,
|
||||
'classes': ['cat', 'dog'],
|
||||
'ann_file': '',
|
||||
}
|
||||
dataset = dataset_class(**cfg)
|
||||
self.assertEqual(len(dataset), 3)
|
||||
self.assertEqual(dataset.CLASSES, ('cat', 'dog'))
|
||||
|
||||
def test_extra_repr(self):
|
||||
dataset_class = DATASETS.get(self.DATASET_TYPE)
|
||||
cfg = {**self.DEFAULT_ARGS}
|
||||
dataset = dataset_class(**cfg)
|
||||
|
||||
self.assertIn(f'Root of dataset: \t{dataset.data_root}', repr(dataset))
|
||||
|
||||
|
||||
class TestImageNet21k(TestCustomDataset):
|
||||
DATASET_TYPE = 'ImageNet21k'
|
||||
|
||||
DEFAULT_ARGS = dict(
|
||||
data_root=ASSETS_ROOT, classes=['cat', 'dog'], ann_file='ann.txt')
|
||||
@classmethod
|
||||
def setUpClass(cls) -> None:
|
||||
super().setUpClass()
|
||||
|
||||
tmpdir = tempfile.TemporaryDirectory()
|
||||
cls.tmpdir = tmpdir
|
||||
cls.root = tmpdir.name
|
||||
cls.meta_folder = 'meta'
|
||||
cls.train_file = 'train.txt'
|
||||
|
||||
os.mkdir(osp.join(cls.root, cls.meta_folder))
|
||||
|
||||
with open(osp.join(cls.root, cls.meta_folder, cls.train_file),
|
||||
'w') as f:
|
||||
f.write('\n'.join([
|
||||
'cat/a.jpg 0',
|
||||
'cat/b.jpg 0',
|
||||
'dog/a.jpg 1',
|
||||
'dog/b.jpg 1',
|
||||
]))
|
||||
|
||||
cls.DEFAULT_ARGS = dict(
|
||||
data_root=cls.root,
|
||||
classes=['cat', 'dog'],
|
||||
ann_file='meta/train.txt')
|
||||
|
||||
def test_initialize(self):
|
||||
super().test_initialize()
|
||||
|
||||
dataset_class = DATASETS.get(self.DATASET_TYPE)
|
||||
|
||||
# Test invalid split
|
||||
with self.assertRaisesRegex(AssertionError, 'The split must be'):
|
||||
cfg = {**self.DEFAULT_ARGS}
|
||||
cfg['split'] = 'unknown'
|
||||
dataset_class(**cfg)
|
||||
|
||||
# Test valid splits
|
||||
cfg = {**self.DEFAULT_ARGS}
|
||||
cfg['split'] = 'train'
|
||||
dataset = dataset_class(**cfg)
|
||||
self.assertEqual(dataset.split, 'train')
|
||||
self.assertEqual(dataset.data_root, self.root)
|
||||
|
||||
def test_load_data_list(self):
|
||||
super().test_initialize()
|
||||
dataset_class = DATASETS.get(self.DATASET_TYPE)
|
||||
|
||||
# The multi_label option is not implemented not.
|
||||
|
|
@ -333,11 +461,14 @@ class TestImageNet21k(TestCustomDataset):
|
|||
dataset_class(**cfg)
|
||||
|
||||
# Warn about ann_file
|
||||
cfg = {**self.DEFAULT_ARGS, 'ann_file': ''}
|
||||
cfg = {**self.DEFAULT_ARGS, 'ann_file': '', 'lazy_init': True}
|
||||
ann_path = osp.join(self.root, self.meta_folder, self.train_file)
|
||||
os.rename(ann_path, ann_path + 'copy')
|
||||
logger = MMLogger.get_current_instance()
|
||||
with self.assertLogs(logger, 'WARN') as log:
|
||||
with self.assertLogs(logger, 'INFO') as log:
|
||||
dataset_class(**cfg)
|
||||
self.assertIn('specify the `ann_file`', log.output[0])
|
||||
os.rename(ann_path + 'copy', ann_path)
|
||||
|
||||
# Warn about classes
|
||||
cfg = {**self.DEFAULT_ARGS, 'classes': None}
|
||||
|
|
@ -345,6 +476,11 @@ class TestImageNet21k(TestCustomDataset):
|
|||
dataset_class(**cfg)
|
||||
self.assertIn('specify the `classes`', log.output[0])
|
||||
|
||||
# Test split='train'
|
||||
cfg = {**self.DEFAULT_ARGS, 'split': 'train', 'classes': None}
|
||||
dataset = dataset_class(**self.DEFAULT_ARGS)
|
||||
self.assertEqual(len(dataset), 4)
|
||||
|
||||
|
||||
class TestPlaces205(TestCustomDataset):
|
||||
DATASET_TYPE = 'Places205'
|
||||
|
|
@ -599,17 +735,12 @@ class TestVOC(TestBaseDataset):
|
|||
cls.tmpdir = tmpdir
|
||||
data_root = tmpdir.name
|
||||
|
||||
cls.DEFAULT_ARGS = dict(
|
||||
data_root=data_root,
|
||||
image_set_path='ImageSets/train.txt',
|
||||
data_prefix=dict(img_path='JPEGImages', ann_path='Annotations'),
|
||||
pipeline=[],
|
||||
test_mode=False)
|
||||
cls.DEFAULT_ARGS = dict(data_root=data_root, split='trainval')
|
||||
|
||||
cls.image_folder = osp.join(data_root, 'JPEGImages')
|
||||
cls.ann_folder = osp.join(data_root, 'Annotations')
|
||||
cls.image_set_folder = osp.join(data_root, 'ImageSets')
|
||||
os.mkdir(cls.image_set_folder)
|
||||
cls.image_set_folder = osp.join(data_root, 'ImageSets', 'Main')
|
||||
os.makedirs(cls.image_set_folder)
|
||||
os.mkdir(cls.image_folder)
|
||||
os.mkdir(cls.ann_folder)
|
||||
|
||||
|
|
@ -621,7 +752,7 @@ class TestVOC(TestBaseDataset):
|
|||
train_list = [i for i in range(0, 4)]
|
||||
test_list = [i for i in range(4, 6)]
|
||||
|
||||
with open(osp.join(cls.image_set_folder, 'train.txt'), 'w') as f:
|
||||
with open(osp.join(cls.image_set_folder, 'trainval.txt'), 'w') as f:
|
||||
for train_item in train_list:
|
||||
f.write(str(train_item) + '\n')
|
||||
with open(osp.join(cls.image_set_folder, 'test.txt'), 'w') as f:
|
||||
|
|
@ -677,6 +808,31 @@ class TestVOC(TestBaseDataset):
|
|||
with self.assertRaisesRegex(ValueError, "type <class 'dict'>"):
|
||||
dataset_class(**cfg)
|
||||
|
||||
# Test invalid split
|
||||
with self.assertRaisesRegex(AssertionError, 'The split must be'):
|
||||
cfg = {**self.DEFAULT_ARGS}
|
||||
cfg['split'] = 'unknown'
|
||||
dataset_class(**cfg)
|
||||
|
||||
# Test valid splits
|
||||
splits = ['trainval', 'test']
|
||||
for split in splits:
|
||||
cfg = {**self.DEFAULT_ARGS}
|
||||
cfg['split'] = split
|
||||
dataset = dataset_class(**cfg)
|
||||
self.assertEqual(dataset.split, split)
|
||||
|
||||
# Test split='trainval' and test_mode = True
|
||||
logger = MMLogger.get_current_instance()
|
||||
with self.assertLogs(logger, 'WARN') as log:
|
||||
cfg = {**self.DEFAULT_ARGS}
|
||||
cfg['split'] = 'trainval'
|
||||
cfg['test_mode'] = True
|
||||
dataset = dataset_class(**cfg)
|
||||
self.assertEqual(dataset.split, 'trainval')
|
||||
self.assertEqual(dataset.test_mode, True)
|
||||
self.assertIn('The trainval set will be used', log.output[0])
|
||||
|
||||
def test_get_cat_ids(self):
|
||||
dataset_class = DATASETS.get(self.DATASET_TYPE)
|
||||
cfg = {'classes': self.fake_classes, **self.DEFAULT_ARGS}
|
||||
|
|
@ -710,9 +866,8 @@ class TestVOC(TestBaseDataset):
|
|||
self.assertEqual(data_info['img_path'], f'{fake_img_path}.jpg')
|
||||
self.assertEqual(set(data_info['gt_label']), set(self.fake_labels[0]))
|
||||
|
||||
# Test with test_mode=True
|
||||
cfg['image_set_path'] = 'ImageSets/test.txt'
|
||||
cfg['test_mode'] = True
|
||||
# Test with split='test'
|
||||
cfg['split'] = 'test'
|
||||
dataset = dataset_class(**cfg)
|
||||
self.assertEqual(len(dataset), 2)
|
||||
|
||||
|
|
@ -722,7 +877,8 @@ class TestVOC(TestBaseDataset):
|
|||
self.assertEqual(set(data_info['gt_label']), set(self.fake_labels[4]))
|
||||
|
||||
# Test with test_mode=True and ann_path = None
|
||||
cfg['image_set_path'] = 'ImageSets/test.txt'
|
||||
cfg['split'] = ''
|
||||
cfg['image_set_path'] = 'ImageSets/Main/test.txt'
|
||||
cfg['test_mode'] = True
|
||||
cfg['data_prefix'] = 'JPEGImages'
|
||||
dataset = dataset_class(**cfg)
|
||||
|
|
@ -771,9 +927,9 @@ class TestMNIST(TestBaseDataset):
|
|||
|
||||
tmpdir = tempfile.TemporaryDirectory()
|
||||
cls.tmpdir = tmpdir
|
||||
cls.root = tmpdir.name
|
||||
data_prefix = tmpdir.name
|
||||
cls.DEFAULT_ARGS = dict(
|
||||
data_prefix=data_prefix, pipeline=[], test_mode=False)
|
||||
cls.DEFAULT_ARGS = dict(data_root=cls.root, split='train')
|
||||
|
||||
dataset_class = DATASETS.get(cls.DATASET_TYPE)
|
||||
|
||||
|
|
@ -805,6 +961,39 @@ class TestMNIST(TestBaseDataset):
|
|||
with open(file, 'wb') as f:
|
||||
f.write(data)
|
||||
|
||||
def test_initialize(self):
|
||||
dataset_class = DATASETS.get(self.DATASET_TYPE)
|
||||
|
||||
# Test with invalid split
|
||||
with self.assertRaisesRegex(AssertionError, 'The split must be'):
|
||||
cfg = {**self.DEFAULT_ARGS}
|
||||
cfg['split'] = 'unknown'
|
||||
dataset_class(**cfg)
|
||||
|
||||
# Test with valid split
|
||||
splits = ['train', 'test']
|
||||
test_modes = [False, True]
|
||||
|
||||
for split in splits:
|
||||
for test_mode in test_modes:
|
||||
cfg = {**self.DEFAULT_ARGS}
|
||||
cfg['split'] = split
|
||||
cfg['test_mode'] = test_mode
|
||||
|
||||
if split == 'train' and test_mode:
|
||||
logger = MMLogger.get_current_instance()
|
||||
with self.assertLogs(logger, 'WARN') as log:
|
||||
dataset = dataset_class(**cfg)
|
||||
self.assertEqual(dataset.split, split)
|
||||
self.assertEqual(dataset.test_mode, test_mode)
|
||||
self.assertEqual(dataset.data_root, self.root)
|
||||
self.assertIn('training set will be used', log.output[0])
|
||||
else:
|
||||
dataset = dataset_class(**cfg)
|
||||
self.assertEqual(dataset.split, split)
|
||||
self.assertEqual(dataset.test_mode, test_mode)
|
||||
self.assertEqual(dataset.data_root, self.root)
|
||||
|
||||
def test_load_data_list(self):
|
||||
dataset_class = DATASETS.get(self.DATASET_TYPE)
|
||||
|
||||
|
|
@ -817,8 +1006,8 @@ class TestMNIST(TestBaseDataset):
|
|||
np.testing.assert_equal(data_info['img'], self.fake_img)
|
||||
np.testing.assert_equal(data_info['gt_label'], self.fake_label)
|
||||
|
||||
# Test with test_mode=True
|
||||
cfg = {**self.DEFAULT_ARGS, 'test_mode': True}
|
||||
# Test with split='test'
|
||||
cfg = {**self.DEFAULT_ARGS, 'split': 'test'}
|
||||
dataset = dataset_class(**cfg)
|
||||
self.assertEqual(len(dataset), 1)
|
||||
|
||||
|
|
@ -829,7 +1018,7 @@ class TestMNIST(TestBaseDataset):
|
|||
# Test automatically download
|
||||
with patch('mmpretrain.datasets.mnist.download_and_extract_archive'
|
||||
) as mock:
|
||||
cfg = {**self.DEFAULT_ARGS, 'lazy_init': True, 'test_mode': True}
|
||||
cfg = {**self.DEFAULT_ARGS, 'lazy_init': True, 'split': 'test'}
|
||||
dataset = dataset_class(**cfg)
|
||||
dataset.train_list = [['invalid_train_file', None]]
|
||||
dataset.test_list = [['invalid_test_file', None]]
|
||||
|
|
@ -852,7 +1041,7 @@ class TestMNIST(TestBaseDataset):
|
|||
with self.assertRaisesRegex(RuntimeError, '`download=True`'):
|
||||
cfg = {
|
||||
**self.DEFAULT_ARGS, 'lazy_init': True,
|
||||
'test_mode': True,
|
||||
'split': 'test',
|
||||
'download': False
|
||||
}
|
||||
dataset = dataset_class(**cfg)
|
||||
|
|
@ -1946,7 +2135,7 @@ class TestSUN397(TestBaseDataset):
|
|||
f.write('\n'.join([
|
||||
'/a/abbey/sun_aqswjsnjlrfzzhiz.jpg',
|
||||
'/a/airplane_cabin/sun_blczihbhbntqccux.jpg',
|
||||
'/a/assembly_line/sun_ajckcfldgdrdjogj.jpg', # invalid
|
||||
'/a/assembly_line/sun_ajckcfldgdrdjogj.jpg',
|
||||
]))
|
||||
|
||||
with open(osp.join(cls.root, cls.meta_folder, cls.test_file),
|
||||
|
|
@ -1954,8 +2143,8 @@ class TestSUN397(TestBaseDataset):
|
|||
f.write('\n'.join([
|
||||
'/a/abbey/sun_ajkqrqitspwywirx.jpg',
|
||||
'/a/airplane_cabin/sun_aqylhacwdsqfjuuu.jpg',
|
||||
'/a/auto_factory/sun_apfsprenzdnzbhmt.jpg', # invalid
|
||||
'/b/baggage_claim/sun_avittiqqaiibgcau.jpg', # invalid
|
||||
'/a/auto_factory/sun_apfsprenzdnzbhmt.jpg',
|
||||
'/b/baggage_claim/sun_avittiqqaiibgcau.jpg',
|
||||
]))
|
||||
|
||||
def test_initialize(self):
|
||||
|
|
@ -1981,7 +2170,7 @@ class TestSUN397(TestBaseDataset):
|
|||
|
||||
# Test default behavior
|
||||
dataset = dataset_class(**self.DEFAULT_ARGS)
|
||||
self.assertEqual(len(dataset), 3 - 1)
|
||||
self.assertEqual(len(dataset), 3)
|
||||
data_info = dataset[0]
|
||||
self.assertEqual(
|
||||
data_info['img_path'],
|
||||
|
|
@ -1992,13 +2181,13 @@ class TestSUN397(TestBaseDataset):
|
|||
# Test with split='test'
|
||||
cfg = {**self.DEFAULT_ARGS, 'split': 'test'}
|
||||
dataset = dataset_class(**cfg)
|
||||
self.assertEqual(len(dataset), 4 - 2)
|
||||
self.assertEqual(len(dataset), 4)
|
||||
data_info = dataset[-1]
|
||||
self.assertEqual(
|
||||
data_info['img_path'],
|
||||
osp.join(self.root, self.data_prefix,
|
||||
'a/airplane_cabin/sun_aqylhacwdsqfjuuu.jpg'))
|
||||
self.assertEqual(data_info['gt_label'], 1)
|
||||
'b/baggage_claim/sun_avittiqqaiibgcau.jpg'))
|
||||
self.assertEqual(data_info['gt_label'], 26)
|
||||
|
||||
def test_extra_repr(self):
|
||||
dataset_class = DATASETS.get(self.DATASET_TYPE)
|
||||
|
|
|
|||
Loading…
Reference in New Issue