145 lines
5.2 KiB
Python
Raw Normal View History

# Copyright (c) OpenMMLab. All rights reserved.
import copy
from typing import Callable, List, Union
from mmcv.transforms import BaseTransform, Compose
from mmpretrain.registry import TRANSFORMS
# Define type of transform or transform config
Transform = Union[dict, Callable[[dict], dict]]
@TRANSFORMS.register_module()
class MultiView(BaseTransform):
"""A transform wrapper for multiple views of an image.
Args:
transforms (list[dict | callable], optional): Sequence of transform
object or config dict to be wrapped.
mapping (dict): A dict that defines the input key mapping.
The keys corresponds to the inner key (i.e., kwargs of the
``transform`` method), and should be string type. The values
corresponds to the outer keys (i.e., the keys of the
data/results), and should have a type of string, list or dict.
None means not applying input mapping. Default: None.
allow_nonexist_keys (bool): If False, the outer keys in the mapping
must exist in the input data, or an exception will be raised.
Default: False.
Examples:
>>> # Example 1: MultiViews 1 pipeline with 2 views
>>> pipeline = [
>>> dict(type='MultiView',
>>> num_views=2,
>>> transforms=[
>>> [
>>> dict(type='Resize', scale=224))],
>>> ])
>>> ]
>>> # Example 2: MultiViews 2 pipelines, the first with 2 views,
>>> # the second with 6 views
>>> pipeline = [
>>> dict(type='MultiView',
>>> num_views=[2, 6],
>>> transforms=[
>>> [
>>> dict(type='Resize', scale=224)],
>>> [
>>> dict(type='Resize', scale=224),
>>> dict(type='RandomSolarize')],
>>> ])
>>> ]
"""
def __init__(self, transforms: List[List[Transform]],
num_views: Union[int, List[int]]) -> None:
if isinstance(num_views, int):
num_views = [num_views]
assert isinstance(num_views, List)
assert len(num_views) == len(transforms)
self.num_views = num_views
self.pipelines = []
for trans in transforms:
pipeline = Compose(trans)
self.pipelines.append(pipeline)
self.transforms = []
for i in range(len(num_views)):
self.transforms.extend([self.pipelines[i]] * num_views[i])
def transform(self, results: dict) -> dict:
"""Apply transformation to inputs.
Args:
results (dict): Result dict from previous pipelines.
Returns:
dict: Transformed results.
"""
multi_views_outputs = dict(img=[])
for trans in self.transforms:
inputs = copy.deepcopy(results)
outputs = trans(inputs)
multi_views_outputs['img'].append(outputs['img'])
results.update(multi_views_outputs)
return results
def __repr__(self) -> str:
repr_str = self.__class__.__name__ + '('
for i, p in enumerate(self.pipelines):
repr_str += f'\nPipeline {i + 1} with {self.num_views[i]} views:\n'
repr_str += str(p)
repr_str += ')'
return repr_str
[Feature] Support multiple multi-modal algorithms and inferencers. (#1561) * [Feat] Migrate blip caption to mmpretrain. (#50) * Migrate blip caption to mmpretrain * minor fix * support train * [Feature] Support OFA caption task. (#51) * [Feature] Support OFA caption task. * Remove duplicated files. * [Feature] Support OFA vqa task. (#58) * [Feature] Support OFA vqa task. * Fix lint. * [Feat] Add BLIP retrieval to mmpretrain. (#55) * init * minor fix for train * fix according to comments * refactor * Update Blip retrieval. (#62) * [Feature] Support OFA visual grounding task. (#59) * [Feature] Support OFA visual grounding task. * minor add TODO --------- Co-authored-by: yingfhu <yingfhu@gmail.com> * [Feat] Add flamingos coco caption and vqa. (#60) * first init * init flamingo coco * add vqa * minor fix * remove unnecessary modules * Update config * Use `ApplyToList`. --------- Co-authored-by: mzr1996 <mzr1996@163.com> * [Feature]: BLIP2 coco retrieval (#53) * [Feature]: Add blip2 retriever * [Feature]: Add blip2 all modules * [Feature]: Refine model * [Feature]: x1 * [Feature]: Runnable coco ret * [Feature]: Runnable version * [Feature]: Fix lint * [Fix]: Fix lint * [Feature]: Use 364 img size * [Feature]: Refactor blip2 * [Fix]: Fix lint * refactor files * minor fix * minor fix --------- Co-authored-by: yingfhu <yingfhu@gmail.com> * Remove * fix blip caption inputs (#68) * [Feat] Add BLIP NLVR support. (#67) * first init * init flamingo coco * add vqa * add nlvr * refactor nlvr * minor fix * minor fix * Update dataset --------- Co-authored-by: mzr1996 <mzr1996@163.com> * [Feature]: BLIP2 Caption (#70) * [Feature]: Add language model * [Feature]: blip2 caption forward * [Feature]: Reproduce the results * [Feature]: Refactor caption * refine config --------- Co-authored-by: yingfhu <yingfhu@gmail.com> * [Feat] Migrate BLIP VQA to mmpretrain (#69) * reformat * change * change * change * change * change * change * change * change * change * change * change * change * change * change * change * change * change * change * change * refactor code --------- Co-authored-by: yingfhu <yingfhu@gmail.com> * Update RefCOCO dataset * [Fix] fix lint * [Feature] Implement inference APIs for multi-modal tasks. (#65) * [Feature] Implement inference APIs for multi-modal tasks. * [Project] Add gradio demo. * [Improve] Update requirements * Update flamingo * Update blip * Add NLVR inferencer * Update flamingo * Update hugging face model register * Update ofa vqa * Update BLIP-vqa (#71) * Update blip-vqa docstring (#72) * Refine flamingo docstring (#73) * [Feature]: BLIP2 VQA (#61) * [Feature]: VQA forward * [Feature]: Reproduce accuracy * [Fix]: Fix lint * [Fix]: Add blank line * minor fix --------- Co-authored-by: yingfhu <yingfhu@gmail.com> * [Feature]: BLIP2 docstring (#74) * [Feature]: Add caption docstring * [Feature]: Add docstring to blip2 vqa * [Feature]: Add docstring to retrieval * Update BLIP-2 metafile and README (#75) * [Feature]: Add readme and docstring * Update blip2 results --------- Co-authored-by: mzr1996 <mzr1996@163.com> * [Feature] BLIP Visual Grounding on MMPretrain Branch (#66) * blip grounding merge with mmpretrain * remove commit * blip grounding test and inference api * refcoco dataset * refcoco dataset refine config * rebasing * gitignore * rebasing * minor edit * minor edit * Update blip-vqa docstring (#72) * rebasing * Revert "minor edit" This reverts commit 639cec757c215e654625ed0979319e60f0be9044. * blip grounding final * precommit * refine config * refine config * Update blip visual grounding --------- Co-authored-by: Yiqin Wang 王逸钦 <wyq1217@outlook.com> Co-authored-by: mzr1996 <mzr1996@163.com> * Update visual grounding metric * Update OFA docstring, README and metafiles. (#76) * [Docs] Update installation docs and gradio demo docs. (#77) * Update OFA name * Update Visual Grounding Visualizer * Integrate accelerate support * Fix imports. * Fix timm backbone * Update imports * Update README * Update circle ci * Update flamingo config * Add gradio demo README * [Feature]: Add scienceqa (#1571) * [Feature]: Add scienceqa * [Feature]: Change param name * Update docs * Update video --------- Co-authored-by: Hubert <42952108+yingfhu@users.noreply.github.com> Co-authored-by: yingfhu <yingfhu@gmail.com> Co-authored-by: Yuan Liu <30762564+YuanLiuuuuuu@users.noreply.github.com> Co-authored-by: Yiqin Wang 王逸钦 <wyq1217@outlook.com> Co-authored-by: Rongjie Li <limo97@163.com>
2023-05-19 16:50:04 +08:00
@TRANSFORMS.register_module()
class ApplyToList(BaseTransform):
"""A transform wrapper to apply the wrapped transforms to a list of items.
For example, to load and resize a list of images.
Args:
transforms (list[dict | callable]): Sequence of transform config dict
to be wrapped.
scatter_key (str): The key to scatter data dict. If the field is a
list, scatter the list to multiple data dicts to do transformation.
collate_keys (List[str]): The keys to collate from multiple data dicts.
The fields in ``collate_keys`` will be composed into a list after
transformation, and the other fields will be adopted from the
first data dict.
"""
def __init__(self, transforms, scatter_key, collate_keys):
super().__init__()
self.transforms = Compose([TRANSFORMS.build(t) for t in transforms])
self.scatter_key = scatter_key
self.collate_keys = set(collate_keys)
self.collate_keys.add(self.scatter_key)
def transform(self, results: dict):
scatter_field = results.get(self.scatter_key)
if isinstance(scatter_field, list):
scattered_results = []
for item in scatter_field:
single_results = copy.deepcopy(results)
single_results[self.scatter_key] = item
scattered_results.append(self.transforms(single_results))
final_output = scattered_results[0]
# merge output list to single output
for key in scattered_results[0].keys():
if key in self.collate_keys:
final_output[key] = [
single[key] for single in scattered_results
]
return final_output
else:
return self.transforms(results)