mirrors
/
fast-reid
mirror of https://github.com/JDAI-CV/fast-reid.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

fix evaluator

pull/608/head
zuchen.wang 2021-11-04 16:49:31 +08:00
parent 1d8451a23a
commit ee009f0a0d
4 changed files with 141 additions and 112 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
fastreid/evaluation/evaluator.py
View File

@ -24,7 +24,7 @@ class DatasetEvaluator:
Preparation for a new round of evaluation.
Should be called before starting a round of evaluation.
"""
pass
self._predictions = []
def preprocess_inputs(self, inputs):
pass

72
fastreid/evaluation/pair_distance_evaluator.py
View File

@ -1,15 +1,29 @@
# coding: utf-8
import copy
import itertools
import logging
from collections import OrderedDict
import numpy as np
import torch
from sklearn import metrics as skmetrics
from .pair_evaluator import PairEvaluator
from fastreid.utils import comm
from .evaluator import DatasetEvaluator
logger = logging.getLogger(__name__)
class PairDistanceEvaluator(PairEvaluator):
class PairDistanceEvaluator(DatasetEvaluator):
def __init__(self, cfg, output_dir=None):
self.cfg = cfg
self._output_dir = output_dir
self._cpu_device = torch.device('cpu')
self._predictions = []
if self.cfg.eval_only:
self._threshold_list = [x / 1000 for x in range(1, 100)] + [x / 10 for x in range(1, 7)]
else:
self._threshold_list = [x / 100 for x in range(1, 100)] + [x / 10 for x in range(1, 7)]
def process(self, inputs, outputs):
query_feat = outputs['query_feature']
@ -23,3 +37,55 @@ class PairDistanceEvaluator(PairEvaluator):
'labels': inputs["targets"].to(self._cpu_device).numpy()
}
self._predictions.append(prediction)
def evaluate(self):
if comm.get_world_size() > 1:
comm.synchronize()
predictions = comm.gather(self._predictions, dst=0)
predictions = list(itertools.chain(*predictions))
if not comm.is_main_process():
return {}
else:
predictions = self._predictions
all_distances = []
all_labels = []
for prediction in predictions:
all_distances.append(prediction['distances'])
all_labels.append(prediction['labels'])
all_distances = np.concatenate(all_distances)
all_labels = np.concatenate(all_labels)
# 计算这3个总体值，还有给定阈值下的precision, recall, f1
cls_acc = skmetrics.accuracy_score(all_labels, all_distances <= 0.5)
ap = skmetrics.average_precision_score(1 - all_labels, all_distances)
auc = skmetrics.roc_auc_score(1 - all_labels, all_distances) # auc under roc
precisions = []
recalls = []
f1s = []
accs = []
for thresh in self._threshold_list:
precision = skmetrics.precision_score(all_labels, all_distances <= thresh, zero_division=0)
recall = skmetrics.recall_score(all_labels, all_distances <= thresh, zero_division=0)
f1 = 2 * precision * recall / (precision + recall + 1e-12)
acc = skmetrics.accuracy_score(all_labels, all_distances <= thresh)
precisions.append(precision)
recalls.append(recall)
f1s.append(f1)
accs.append(acc)
self._results = OrderedDict()
self._results['Acc@0.5'] = acc
self._results['Ap'] = ap
self._results['Auc'] = auc
self._results['Thresholds'] = self._threshold_list
self._results['Precisions'] = precisions
self._results['Recalls'] = recalls
self._results['F1_Scores'] = f1s
self._results['Accs'] = accs
return copy.deepcopy(self._results)

99
fastreid/evaluation/pair_evaluator.py
View File

@ -1,99 +0,0 @@
# coding: utf-8
import copy
import itertools
import logging
from collections import OrderedDict
import numpy as np
import torch
from fastreid.utils import comm
from sklearn import metrics as skmetrics
from fastreid.modeling.losses.utils import normalize
from .evaluator import DatasetEvaluator
logger = logging.getLogger(__name__)
class PairEvaluator(DatasetEvaluator):
def __init__(self, cfg, output_dir=None):
self.cfg = cfg
self._output_dir = output_dir
self._cpu_device = torch.device('cpu')
self._predictions = []
if self.cfg.eval_only:
self._threshold_list = [x / 10 for x in range(5, 10)] + [x / 1000 for x in range(901, 1000)]
else:
self._threshold_list = [x / 10 for x in range(5, 9)] + [x / 100 for x in range(90, 100)]
def reset(self):
self._predictions = []
def process(self, inputs, outputs):
embedding = outputs['features'].to(self._cpu_device)
embedding = embedding.view(embedding.size(0) * 2, -1)
embedding = normalize(embedding, axis=-1)
embed1 = embedding[0:len(embedding):2, :]
embed2 = embedding[1:len(embedding):2, :]
distances = torch.mul(embed1, embed2).sum(-1).numpy()
# print(distances)
prediction = {
'distances': distances,
'labels': inputs["targets"].to(self._cpu_device).numpy()
}
self._predictions.append(prediction)
def evaluate(self):
if comm.get_world_size() > 1:
comm.synchronize()
predictions = comm.gather(self._predictions, dst=0)
predictions = list(itertools.chain(*predictions))
if not comm.is_main_process():
return {}
else:
predictions = self._predictions
all_distances = []
all_labels = []
for prediction in predictions:
all_distances.append(prediction['distances'])
all_labels.append(prediction['labels'])
all_distances = np.concatenate(all_distances)
all_labels = np.concatenate(all_labels)
# 计算这3个总体值，还有给定阈值下的precision, recall, f1
cls_acc = skmetrics.accuracy_score(all_labels, all_distances >= 0.5)
ap = skmetrics.average_precision_score(all_labels, all_distances)
auc = skmetrics.roc_auc_score(all_labels, all_distances) # auc under roc
precisions = []
recalls = []
f1s = []
accs = []
for thresh in self._threshold_list:
precision = skmetrics.precision_score(all_labels, all_distances >= thresh, zero_division=0)
recall = skmetrics.recall_score(all_labels, all_distances >= thresh, zero_division=0)
f1 = 2 * precision * recall / (precision + recall + 1e-12)
acc = skmetrics.accuracy_score(all_labels, all_distances >= thresh)
precisions.append(precision)
recalls.append(recall)
f1s.append(f1)
accs.append(acc)
self._results = OrderedDict()
self._results['Acc@0.5'] = acc
self._results['Ap'] = ap
self._results['Auc'] = auc
self._results['Thresholds'] = self._threshold_list
self._results['Precisions'] = precisions
self._results['Recalls'] = recalls
self._results['F1_Scores'] = f1s
self._results['Accs'] = accs
return copy.deepcopy(self._results)

80
fastreid/evaluation/pair_score_evaluator.py
View File

@ -1,24 +1,86 @@
# -*- coding: utf-8 -*-
# @Time : 2021/10/27 15:56:52
# @Author : zuchen.wang@vipshop.com
# @File : pair_score_evaluator.py
# coding: utf-8
import copy
import itertools
import logging
from collections import OrderedDict
import numpy as np
import torch
from sklearn import metrics as skmetrics
from fastreid.modeling.losses.utils import normalize
from .pair_evaluator import PairEvaluator
from fastreid.utils import comm
from .evaluator import DatasetEvaluator
logger = logging.getLogger(__name__)
class PairScoreEvaluator(PairEvaluator):
class PairScoreEvaluator(DatasetEvaluator):
def __init__(self, cfg, output_dir=None):
self.cfg = cfg
self._output_dir = output_dir
self._cpu_device = torch.device('cpu')
self._predictions = []
if self.cfg.eval_only:
self._threshold_list = [x / 10 for x in range(5, 10)] + [x / 1000 for x in range(901, 1000)]
else:
self._threshold_list = [x / 10 for x in range(5, 9)] + [x / 100 for x in range(90, 100)]
def process(self, inputs, outputs):
prediction = {
'distances': outputs['cls_outputs'][:, 1].to(self._cpu_device).numpy(),
'labels': inputs["targets"].to(self._cpu_device).numpy()
'score': outputs['cls_outputs'][:, 1].to(self._cpu_device).numpy(),
'labels': inputs["targets"].to(self._cpu_device).numpy()
}
self._predictions.append(prediction)
def evaluate(self):
if comm.get_world_size() > 1:
comm.synchronize()
predictions = comm.gather(self._predictions, dst=0)
predictions = list(itertools.chain(*predictions))
if not comm.is_main_process():
return {}
else:
predictions = self._predictions
all_scores = []
all_labels = []
for prediction in predictions:
all_scores.append(prediction['score'])
all_labels.append(prediction['labels'])
all_scores = np.concatenate(all_scores)
all_labels = np.concatenate(all_labels)
# 计算这3个总体值，还有给定阈值下的precision, recall, f1
cls_acc = skmetrics.accuracy_score(all_labels, all_scores >= 0.5)
ap = skmetrics.average_precision_score(all_labels, all_scores)
auc = skmetrics.roc_auc_score(all_labels, all_scores) # auc under roc
precisions = []
recalls = []
f1s = []
accs = []
for thresh in self._threshold_list:
precision = skmetrics.precision_score(all_labels, all_scores >= thresh, zero_division=0)
recall = skmetrics.recall_score(all_labels, all_scores >= thresh, zero_division=0)
f1 = 2 * precision * recall / (precision + recall + 1e-12)
acc = skmetrics.accuracy_score(all_labels, all_scores >= thresh)
precisions.append(precision)
recalls.append(recall)
f1s.append(f1)
accs.append(acc)
self._results = OrderedDict()
self._results['Acc@0.5'] = acc
self._results['Ap'] = ap
self._results['Auc'] = auc
self._results['Thresholds'] = self._threshold_list
self._results['Precisions'] = precisions
self._results['Recalls'] = recalls
self._results['F1_Scores'] = f1s
self._results['Accs'] = accs
return copy.deepcopy(self._results)