169 lines
6.4 KiB
Python
Raw Normal View History

[Feature] Support multiple multi-modal algorithms and inferencers. (#1561) * [Feat] Migrate blip caption to mmpretrain. (#50) * Migrate blip caption to mmpretrain * minor fix * support train * [Feature] Support OFA caption task. (#51) * [Feature] Support OFA caption task. * Remove duplicated files. * [Feature] Support OFA vqa task. (#58) * [Feature] Support OFA vqa task. * Fix lint. * [Feat] Add BLIP retrieval to mmpretrain. (#55) * init * minor fix for train * fix according to comments * refactor * Update Blip retrieval. (#62) * [Feature] Support OFA visual grounding task. (#59) * [Feature] Support OFA visual grounding task. * minor add TODO --------- Co-authored-by: yingfhu <yingfhu@gmail.com> * [Feat] Add flamingos coco caption and vqa. (#60) * first init * init flamingo coco * add vqa * minor fix * remove unnecessary modules * Update config * Use `ApplyToList`. --------- Co-authored-by: mzr1996 <mzr1996@163.com> * [Feature]: BLIP2 coco retrieval (#53) * [Feature]: Add blip2 retriever * [Feature]: Add blip2 all modules * [Feature]: Refine model * [Feature]: x1 * [Feature]: Runnable coco ret * [Feature]: Runnable version * [Feature]: Fix lint * [Fix]: Fix lint * [Feature]: Use 364 img size * [Feature]: Refactor blip2 * [Fix]: Fix lint * refactor files * minor fix * minor fix --------- Co-authored-by: yingfhu <yingfhu@gmail.com> * Remove * fix blip caption inputs (#68) * [Feat] Add BLIP NLVR support. (#67) * first init * init flamingo coco * add vqa * add nlvr * refactor nlvr * minor fix * minor fix * Update dataset --------- Co-authored-by: mzr1996 <mzr1996@163.com> * [Feature]: BLIP2 Caption (#70) * [Feature]: Add language model * [Feature]: blip2 caption forward * [Feature]: Reproduce the results * [Feature]: Refactor caption * refine config --------- Co-authored-by: yingfhu <yingfhu@gmail.com> * [Feat] Migrate BLIP VQA to mmpretrain (#69) * reformat * change * change * change * change * change * change * change * change * change * change * change * change * change * change * change * change * change * change * change * refactor code --------- Co-authored-by: yingfhu <yingfhu@gmail.com> * Update RefCOCO dataset * [Fix] fix lint * [Feature] Implement inference APIs for multi-modal tasks. (#65) * [Feature] Implement inference APIs for multi-modal tasks. * [Project] Add gradio demo. * [Improve] Update requirements * Update flamingo * Update blip * Add NLVR inferencer * Update flamingo * Update hugging face model register * Update ofa vqa * Update BLIP-vqa (#71) * Update blip-vqa docstring (#72) * Refine flamingo docstring (#73) * [Feature]: BLIP2 VQA (#61) * [Feature]: VQA forward * [Feature]: Reproduce accuracy * [Fix]: Fix lint * [Fix]: Add blank line * minor fix --------- Co-authored-by: yingfhu <yingfhu@gmail.com> * [Feature]: BLIP2 docstring (#74) * [Feature]: Add caption docstring * [Feature]: Add docstring to blip2 vqa * [Feature]: Add docstring to retrieval * Update BLIP-2 metafile and README (#75) * [Feature]: Add readme and docstring * Update blip2 results --------- Co-authored-by: mzr1996 <mzr1996@163.com> * [Feature] BLIP Visual Grounding on MMPretrain Branch (#66) * blip grounding merge with mmpretrain * remove commit * blip grounding test and inference api * refcoco dataset * refcoco dataset refine config * rebasing * gitignore * rebasing * minor edit * minor edit * Update blip-vqa docstring (#72) * rebasing * Revert "minor edit" This reverts commit 639cec757c215e654625ed0979319e60f0be9044. * blip grounding final * precommit * refine config * refine config * Update blip visual grounding --------- Co-authored-by: Yiqin Wang 王逸钦 <wyq1217@outlook.com> Co-authored-by: mzr1996 <mzr1996@163.com> * Update visual grounding metric * Update OFA docstring, README and metafiles. (#76) * [Docs] Update installation docs and gradio demo docs. (#77) * Update OFA name * Update Visual Grounding Visualizer * Integrate accelerate support * Fix imports. * Fix timm backbone * Update imports * Update README * Update circle ci * Update flamingo config * Add gradio demo README * [Feature]: Add scienceqa (#1571) * [Feature]: Add scienceqa * [Feature]: Change param name * Update docs * Update video --------- Co-authored-by: Hubert <42952108+yingfhu@users.noreply.github.com> Co-authored-by: yingfhu <yingfhu@gmail.com> Co-authored-by: Yuan Liu <30762564+YuanLiuuuuuu@users.noreply.github.com> Co-authored-by: Yiqin Wang 王逸钦 <wyq1217@outlook.com> Co-authored-by: Rongjie Li <limo97@163.com>
2023-05-19 16:50:04 +08:00
# Copyright (c) OpenMMLab. All rights reserved.
import torch
from mmengine.model import is_model_wrapper
from mmengine.runner import TestLoop, ValLoop, autocast
from mmpretrain.registry import LOOPS
@LOOPS.register_module()
class RetrievalValLoop(ValLoop):
"""Loop for multimodal retrieval val.
Args:
runner (Runner): A reference of runner.
dataloader (Dataloader or dict): A dataloader object or a dict to
build a dataloader.
evaluator (Evaluator or dict or list): Used for computing metrics.
fp16 (bool): Whether to enable fp16 valing. Defaults to
False.
"""
def run(self) -> dict:
"""Launch val."""
self.runner.call_hook('before_val')
self.runner.call_hook('before_val_epoch')
self.runner.model.eval()
feats_local = []
data_samples_local = []
for idx, data_batch in enumerate(self.dataloader):
with torch.no_grad():
self.runner.call_hook(
'before_val_iter', batch_idx=idx, data_batch=data_batch)
# predictions should be sequence of BaseDataElement
with autocast(enabled=self.fp16):
if is_model_wrapper(self.runner.model):
data_preprocessor = self.runner.model.module.data_preprocessor # noqa: E501
else:
data_preprocessor = self.runner.model.data_preprocessor
# get features for retrieval instead of data samples
data_batch = data_preprocessor(data_batch, False)
feats = self.runner.model._run_forward(
data_batch, mode='tensor')
feats_local.append(feats)
data_samples_local.extend(data_batch['data_samples'])
self.runner.call_hook(
'after_val_iter',
batch_idx=idx,
data_batch=data_batch,
outputs=feats)
# concatenate different features
feats_local = {
k: torch.cat([dic[k] for dic in feats_local])
for k in feats_local[0]
}
# get predictions
if is_model_wrapper(self.runner.model):
predict_all_fn = self.runner.model.module.predict_all
else:
predict_all_fn = self.runner.model.predict_all
img_size = self.dataloader.dataset.img_size
text_size = self.dataloader.dataset.text_size
with torch.no_grad():
i2t_data_samples, t2i_data_samples = predict_all_fn(
feats_local,
data_samples_local,
num_images=img_size,
num_texts=text_size,
)
# process in evaluator and compute metrics
self.evaluator.process(i2t_data_samples, None)
i2t_metrics = self.evaluator.evaluate(img_size)
i2t_metrics = {f'i2t/{k}': v for k, v in i2t_metrics.items()}
self.evaluator.process(t2i_data_samples, None)
t2i_metrics = self.evaluator.evaluate(text_size)
t2i_metrics = {f't2i/{k}': v for k, v in t2i_metrics.items()}
metrics = {**i2t_metrics, **t2i_metrics}
self.runner.call_hook('after_val_epoch', metrics=metrics)
self.runner.call_hook('after_val')
return metrics
@LOOPS.register_module()
class RetrievalTestLoop(TestLoop):
"""Loop for multimodal retrieval test.
Args:
runner (Runner): A reference of runner.
dataloader (Dataloader or dict): A dataloader object or a dict to
build a dataloader.
evaluator (Evaluator or dict or list): Used for computing metrics.
fp16 (bool): Whether to enable fp16 testing. Defaults to
False.
"""
def run(self) -> dict:
"""Launch test."""
self.runner.call_hook('before_test')
self.runner.call_hook('before_test_epoch')
self.runner.model.eval()
feats_local = []
data_samples_local = []
for idx, data_batch in enumerate(self.dataloader):
with torch.no_grad():
self.runner.call_hook(
'before_test_iter', batch_idx=idx, data_batch=data_batch)
# predictions should be sequence of BaseDataElement
with autocast(enabled=self.fp16):
if is_model_wrapper(self.runner.model):
data_preprocessor = self.runner.model.module.data_preprocessor # noqa: E501
else:
data_preprocessor = self.runner.model.data_preprocessor
# get features for retrieval instead of data samples
data_batch = data_preprocessor(data_batch, False)
feats = self.runner.model._run_forward(
data_batch, mode='tensor')
feats_local.append(feats)
data_samples_local.extend(data_batch['data_samples'])
self.runner.call_hook(
'after_test_iter',
batch_idx=idx,
data_batch=data_batch,
outputs=feats)
# concatenate different features
feats_local = {
k: torch.cat([dic[k] for dic in feats_local])
for k in feats_local[0]
}
# get predictions
if is_model_wrapper(self.runner.model):
predict_all_fn = self.runner.model.module.predict_all
else:
predict_all_fn = self.runner.model.predict_all
img_size = self.dataloader.dataset.img_size
text_size = self.dataloader.dataset.text_size
with torch.no_grad():
i2t_data_samples, t2i_data_samples = predict_all_fn(
feats_local,
data_samples_local,
num_images=img_size,
num_texts=text_size,
)
# process in evaluator and compute metrics
self.evaluator.process(i2t_data_samples, None)
i2t_metrics = self.evaluator.evaluate(img_size)
i2t_metrics = {f'i2t/{k}': v for k, v in i2t_metrics.items()}
self.evaluator.process(t2i_data_samples, None)
t2i_metrics = self.evaluator.evaluate(text_size)
t2i_metrics = {f't2i/{k}': v for k, v in t2i_metrics.items()}
metrics = {**i2t_metrics, **t2i_metrics}
self.runner.call_hook('after_test_epoch', metrics=metrics)
self.runner.call_hook('after_test')
return metrics