251 lines
7.9 KiB
Python
251 lines
7.9 KiB
Python
import os.path as osp
|
|
from argparse import ArgumentParser
|
|
|
|
import mmcv
|
|
import numpy as np
|
|
|
|
|
|
def print_coco_results(results):
|
|
|
|
def _print(result, ap=1, iouThr=None, areaRng='all', maxDets=100):
|
|
titleStr = 'Average Precision' if ap == 1 else 'Average Recall'
|
|
typeStr = '(AP)' if ap == 1 else '(AR)'
|
|
iouStr = '0.50:0.95' \
|
|
if iouThr is None else f'{iouThr:0.2f}'
|
|
iStr = f' {titleStr:<18} {typeStr} @[ IoU={iouStr:<9} | '
|
|
iStr += f'area={areaRng:>6s} | maxDets={maxDets:>3d} ] = {result:0.3f}'
|
|
print(iStr)
|
|
|
|
stats = np.zeros((12, ))
|
|
stats[0] = _print(results[0], 1)
|
|
stats[1] = _print(results[1], 1, iouThr=.5)
|
|
stats[2] = _print(results[2], 1, iouThr=.75)
|
|
stats[3] = _print(results[3], 1, areaRng='small')
|
|
stats[4] = _print(results[4], 1, areaRng='medium')
|
|
stats[5] = _print(results[5], 1, areaRng='large')
|
|
stats[6] = _print(results[6], 0, maxDets=1)
|
|
stats[7] = _print(results[7], 0, maxDets=10)
|
|
stats[8] = _print(results[8], 0)
|
|
stats[9] = _print(results[9], 0, areaRng='small')
|
|
stats[10] = _print(results[10], 0, areaRng='medium')
|
|
stats[11] = _print(results[11], 0, areaRng='large')
|
|
|
|
|
|
def get_coco_style_results(filename,
|
|
task='bbox',
|
|
metric=None,
|
|
prints='mPC',
|
|
aggregate='benchmark'):
|
|
|
|
assert aggregate in ['benchmark', 'all']
|
|
|
|
if prints == 'all':
|
|
prints = ['P', 'mPC', 'rPC']
|
|
elif isinstance(prints, str):
|
|
prints = [prints]
|
|
for p in prints:
|
|
assert p in ['P', 'mPC', 'rPC']
|
|
|
|
if metric is None:
|
|
metrics = [
|
|
'AP', 'AP50', 'AP75', 'APs', 'APm', 'APl', 'AR1', 'AR10', 'AR100',
|
|
'ARs', 'ARm', 'ARl'
|
|
]
|
|
elif isinstance(metric, list):
|
|
metrics = metric
|
|
else:
|
|
metrics = [metric]
|
|
|
|
for metric_name in metrics:
|
|
assert metric_name in [
|
|
'AP', 'AP50', 'AP75', 'APs', 'APm', 'APl', 'AR1', 'AR10', 'AR100',
|
|
'ARs', 'ARm', 'ARl'
|
|
]
|
|
|
|
eval_output = mmcv.load(filename)
|
|
|
|
num_distortions = len(list(eval_output.keys()))
|
|
results = np.zeros((num_distortions, 6, len(metrics)), dtype='float32')
|
|
|
|
for corr_i, distortion in enumerate(eval_output):
|
|
for severity in eval_output[distortion]:
|
|
for metric_j, metric_name in enumerate(metrics):
|
|
mAP = eval_output[distortion][severity][task][metric_name]
|
|
results[corr_i, severity, metric_j] = mAP
|
|
|
|
P = results[0, 0, :]
|
|
if aggregate == 'benchmark':
|
|
mPC = np.mean(results[:15, 1:, :], axis=(0, 1))
|
|
else:
|
|
mPC = np.mean(results[:, 1:, :], axis=(0, 1))
|
|
rPC = mPC / P
|
|
|
|
print(f'\nmodel: {osp.basename(filename)}')
|
|
if metric is None:
|
|
if 'P' in prints:
|
|
print(f'Performance on Clean Data [P] ({task})')
|
|
print_coco_results(P)
|
|
if 'mPC' in prints:
|
|
print(f'Mean Performance under Corruption [mPC] ({task})')
|
|
print_coco_results(mPC)
|
|
if 'rPC' in prints:
|
|
print(f'Realtive Performance under Corruption [rPC] ({task})')
|
|
print_coco_results(rPC)
|
|
else:
|
|
if 'P' in prints:
|
|
print(f'Performance on Clean Data [P] ({task})')
|
|
for metric_i, metric_name in enumerate(metrics):
|
|
print(f'{metric_name:5} = {P[metric_i]:0.3f}')
|
|
if 'mPC' in prints:
|
|
print(f'Mean Performance under Corruption [mPC] ({task})')
|
|
for metric_i, metric_name in enumerate(metrics):
|
|
print(f'{metric_name:5} = {mPC[metric_i]:0.3f}')
|
|
if 'rPC' in prints:
|
|
print(f'Relative Performance under Corruption [rPC] ({task})')
|
|
for metric_i, metric_name in enumerate(metrics):
|
|
print(f'{metric_name:5} => {rPC[metric_i] * 100:0.1f} %')
|
|
|
|
return results
|
|
|
|
|
|
def get_voc_style_results(filename, prints='mPC', aggregate='benchmark'):
|
|
|
|
assert aggregate in ['benchmark', 'all']
|
|
|
|
if prints == 'all':
|
|
prints = ['P', 'mPC', 'rPC']
|
|
elif isinstance(prints, str):
|
|
prints = [prints]
|
|
for p in prints:
|
|
assert p in ['P', 'mPC', 'rPC']
|
|
|
|
eval_output = mmcv.load(filename)
|
|
|
|
num_distortions = len(list(eval_output.keys()))
|
|
results = np.zeros((num_distortions, 6, 20), dtype='float32')
|
|
|
|
for i, distortion in enumerate(eval_output):
|
|
for severity in eval_output[distortion]:
|
|
mAP = [
|
|
eval_output[distortion][severity][j]['ap']
|
|
for j in range(len(eval_output[distortion][severity]))
|
|
]
|
|
results[i, severity, :] = mAP
|
|
|
|
P = results[0, 0, :]
|
|
if aggregate == 'benchmark':
|
|
mPC = np.mean(results[:15, 1:, :], axis=(0, 1))
|
|
else:
|
|
mPC = np.mean(results[:, 1:, :], axis=(0, 1))
|
|
rPC = mPC / P
|
|
|
|
print(f'\nmodel: {osp.basename(filename)}')
|
|
if 'P' in prints:
|
|
print(f'Performance on Clean Data [P] in AP50 = {np.mean(P):0.3f}')
|
|
if 'mPC' in prints:
|
|
print('Mean Performance under Corruption [mPC] in AP50 = '
|
|
f'{np.mean(mPC):0.3f}')
|
|
if 'rPC' in prints:
|
|
print('Realtive Performance under Corruption [rPC] in % = '
|
|
f'{np.mean(rPC) * 100:0.1f}')
|
|
|
|
return np.mean(results, axis=2, keepdims=True)
|
|
|
|
|
|
def get_results(filename,
|
|
dataset='coco',
|
|
task='bbox',
|
|
metric=None,
|
|
prints='mPC',
|
|
aggregate='benchmark'):
|
|
assert dataset in ['coco', 'voc', 'cityscapes']
|
|
|
|
if dataset in ['coco', 'cityscapes']:
|
|
results = get_coco_style_results(
|
|
filename,
|
|
task=task,
|
|
metric=metric,
|
|
prints=prints,
|
|
aggregate=aggregate)
|
|
elif dataset == 'voc':
|
|
if task != 'bbox':
|
|
print('Only bbox analysis is supported for Pascal VOC')
|
|
print('Will report bbox results\n')
|
|
if metric not in [None, ['AP'], ['AP50']]:
|
|
print('Only the AP50 metric is supported for Pascal VOC')
|
|
print('Will report AP50 metric\n')
|
|
results = get_voc_style_results(
|
|
filename, prints=prints, aggregate=aggregate)
|
|
|
|
return results
|
|
|
|
|
|
def get_distortions_from_file(filename):
|
|
|
|
eval_output = mmcv.load(filename)
|
|
|
|
return get_distortions_from_results(eval_output)
|
|
|
|
|
|
def get_distortions_from_results(eval_output):
|
|
distortions = []
|
|
for i, distortion in enumerate(eval_output):
|
|
distortions.append(distortion.replace('_', ' '))
|
|
return distortions
|
|
|
|
|
|
def main():
|
|
parser = ArgumentParser(description='Corruption Result Analysis')
|
|
parser.add_argument('filename', help='result file path')
|
|
parser.add_argument(
|
|
'--dataset',
|
|
type=str,
|
|
choices=['coco', 'voc', 'cityscapes'],
|
|
default='coco',
|
|
help='dataset type')
|
|
parser.add_argument(
|
|
'--task',
|
|
type=str,
|
|
nargs='+',
|
|
choices=['bbox', 'segm'],
|
|
default=['bbox'],
|
|
help='task to report')
|
|
parser.add_argument(
|
|
'--metric',
|
|
nargs='+',
|
|
choices=[
|
|
None, 'AP', 'AP50', 'AP75', 'APs', 'APm', 'APl', 'AR1', 'AR10',
|
|
'AR100', 'ARs', 'ARm', 'ARl'
|
|
],
|
|
default=None,
|
|
help='metric to report')
|
|
parser.add_argument(
|
|
'--prints',
|
|
type=str,
|
|
nargs='+',
|
|
choices=['P', 'mPC', 'rPC'],
|
|
default='mPC',
|
|
help='corruption benchmark metric to print')
|
|
parser.add_argument(
|
|
'--aggregate',
|
|
type=str,
|
|
choices=['all', 'benchmark'],
|
|
default='benchmark',
|
|
help='aggregate all results or only those \
|
|
for benchmark corruptions')
|
|
|
|
args = parser.parse_args()
|
|
|
|
for task in args.task:
|
|
get_results(
|
|
args.filename,
|
|
dataset=args.dataset,
|
|
task=task,
|
|
metric=args.metric,
|
|
prints=args.prints,
|
|
aggregate=args.aggregate)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|