From 2da31a875aea8f06b28a7b82231fd6e67c9d8f33 Mon Sep 17 00:00:00 2001 From: dongshuilong Date: Sat, 5 Jun 2021 17:49:36 +0800 Subject: [PATCH 1/3] fix metric issues --- ppcls/configs/Vehicle/ResNet50_ReID.yaml | 27 ++++++---- ppcls/metric/metrics.py | 64 +++++++++--------------- 2 files changed, 40 insertions(+), 51 deletions(-) diff --git a/ppcls/configs/Vehicle/ResNet50_ReID.yaml b/ppcls/configs/Vehicle/ResNet50_ReID.yaml index 768aa1604..4fac17361 100644 --- a/ppcls/configs/Vehicle/ResNet50_ReID.yaml +++ b/ppcls/configs/Vehicle/ResNet50_ReID.yaml @@ -16,8 +16,7 @@ Global: # used for static mode and model export image_shape: [3, 224, 224] save_inference_dir: "./inference" - num_split: 1 - feature_normalize: True + eval_mode: "retrieval" # model architecture Arch: @@ -99,10 +98,10 @@ DataLoader: loader: num_workers: 6 use_shared_memory: False - - Query: + Eval: + Query: # TOTO: modify to the latest trainer - dataset: + dataset: name: "VeriWild" image_root: "/work/dataset/VeRI-Wild/images" cls_label_path: "/work/dataset/VeRI-Wild/train_test_split/test_3000_id_query.txt" @@ -114,18 +113,18 @@ DataLoader: mean: [0.485, 0.456, 0.406] std: [0.229, 0.224, 0.225] order: '' - sampler: + sampler: name: DistributedBatchSampler batch_size: 64 drop_last: False shuffle: False - loader: + loader: num_workers: 6 use_shared_memory: False - Gallery: + Gallery: # TOTO: modify to the latest trainer - dataset: + dataset: name: "VeriWild" image_root: "/work/dataset/VeRI-Wild/images" cls_label_path: "/work/dataset/VeRI-Wild/train_test_split/test_3000_id.txt" @@ -137,15 +136,21 @@ DataLoader: mean: [0.485, 0.456, 0.406] std: [0.229, 0.224, 0.225] order: '' - sampler: + sampler: name: DistributedBatchSampler batch_size: 64 drop_last: False shuffle: False - loader: + loader: num_workers: 6 use_shared_memory: False +Metric: + Eval: + - Recallk: + topk: [1, 5] + - mAP: + Infer: infer_imgs: "docs/images/whl/demo.jpg" batch_size: 10 diff --git a/ppcls/metric/metrics.py b/ppcls/metric/metrics.py index d0044dafb..05723c123 100644 --- a/ppcls/metric/metrics.py +++ b/ppcls/metric/metrics.py @@ -15,6 +15,7 @@ import numpy as np import paddle import paddle.nn as nn +from functools import lru_cache # TODO: fix the format @@ -38,23 +39,13 @@ class TopkAcc(nn.Layer): class mAP(nn.Layer): - def __init__(self, max_rank=50): + def __init__(self): super().__init__() - self.max_rank = max_rank def forward(self, similarities_matrix, query_img_id, gallery_img_id): metric_dict = dict() - num_q, num_g = similarities_matrix.shape - q_pids = query_img_id.numpy().reshape((query_img_id.shape[0])) - g_pids = gallery_img_id.numpy().reshape((gallery_img_id.shape[0])) - if num_g < self.max_rank: - self.max_rank = num_g - print('Note: number of gallery samples is quite small, got {}'. - format(num_g)) - indices = paddle.argsort( - similarities_matrix, axis=1, descending=True).numpy() - _, all_AP, _ = get_metrics(indices, num_q, num_g, q_pids, g_pids, - self.max_rank) + _, all_AP, _ = get_metrics(similarities_matrix, query_img_id, + gallery_img_id) mAP = np.mean(all_AP) metric_dict["mAP"] = mAP @@ -62,23 +53,13 @@ class mAP(nn.Layer): class mINP(nn.Layer): - def __init__(self, max_rank=50): + def __init__(self): super().__init__() - self.max_rank = max_rank def forward(self, similarities_matrix, query_img_id, gallery_img_id): metric_dict = dict() - num_q, num_g = similarities_matrix.shape - q_pids = query_img_id.numpy().reshape((query_img_id.shape[0])) - g_pids = gallery_img_id.numpy().reshape((gallery_img_id.shape[0])) - if num_g < self.max_rank: - max_rank = num_g - print('Note: number of gallery samples is quite small, got {}'. - format(num_g)) - indices = paddle.argsort( - similarities_matrix, axis=1, descending=True).numpy() - _, _, all_INP = get_metrics(indices, num_q, num_g, q_pids, g_pids, - self.max_rank) + _, _, all_INP = get_metrics(similarities_matrix, query_img_id, + gallery_img_id) mINP = np.mean(all_INP) metric_dict["mINP"] = mINP @@ -86,34 +67,37 @@ class mINP(nn.Layer): class Recallk(nn.Layer): - def __init__(self, max_rank=50, topk=(1, 5)): + def __init__(self, topk=(1, 5)): super().__init__() - self.max_rank = max_rank assert isinstance(topk, (int, list)) if isinstance(topk, int): topk = [topk] self.topk = topk + self.max_rank = max(self.topk) if max(self.topk) > 50 else 50 def forward(self, similarities_matrix, query_img_id, gallery_img_id): metric_dict = dict() - num_q, num_g = similarities_matrix.shape - q_pids = query_img_id.numpy().reshape((query_img_id.shape[0])) - g_pids = gallery_img_id.numpy().reshape((gallery_img_id.shape[0])) - if num_g < self.max_rank: - max_rank = num_g - print('Note: number of gallery samples is quite small, got {}'. - format(num_g)) - indices = paddle.argsort( - similarities_matrix, axis=1, descending=True).numpy() - all_cmc, _, _ = get_metrics(indices, num_q, num_g, q_pids, g_pids, - self.max_rank) + all_cmc, _, _ = get_metrics(similarities_matrix, query_img_id, + gallery_img_id, self.max_rank) for k in self.topk: metric_dict["recall{}".format(k)] = all_cmc[k - 1] return metric_dict -def get_metrics(indices, num_q, num_g, q_pids, g_pids, max_rank=50): +@lru_cache() +def get_metrics(similarities_matrix, query_img_id, gallery_img_id, + max_rank=50): + num_q, num_g = similarities_matrix.shape + q_pids = query_img_id.numpy().reshape((query_img_id.shape[0])) + g_pids = gallery_img_id.numpy().reshape((gallery_img_id.shape[0])) + if num_g < max_rank: + max_rank = num_g + print('Note: number of gallery samples is quite small, got {}'.format( + num_g)) + indices = paddle.argsort( + similarities_matrix, axis=1, descending=True).numpy() + all_cmc = [] all_AP = [] all_INP = [] From fa2b698ad5d349660f49ec0e28abaa7f34164aa9 Mon Sep 17 00:00:00 2001 From: dongshuilong Date: Sat, 5 Jun 2021 18:07:26 +0800 Subject: [PATCH 2/3] fix metric bugs --- ppcls/configs/Vehicle/ResNet50_ReID.yaml | 2 -- ppcls/metric/metrics.py | 10 ++++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ppcls/configs/Vehicle/ResNet50_ReID.yaml b/ppcls/configs/Vehicle/ResNet50_ReID.yaml index 4fac17361..39862847e 100644 --- a/ppcls/configs/Vehicle/ResNet50_ReID.yaml +++ b/ppcls/configs/Vehicle/ResNet50_ReID.yaml @@ -1,6 +1,4 @@ # global configs -Trainer: - name: TrainerReID Global: checkpoints: null pretrained_model: null diff --git a/ppcls/metric/metrics.py b/ppcls/metric/metrics.py index 05723c123..ac2866392 100644 --- a/ppcls/metric/metrics.py +++ b/ppcls/metric/metrics.py @@ -39,8 +39,9 @@ class TopkAcc(nn.Layer): class mAP(nn.Layer): - def __init__(self): + def __init__(self, name="mAP"): super().__init__() + self.name = name def forward(self, similarities_matrix, query_img_id, gallery_img_id): metric_dict = dict() @@ -48,13 +49,14 @@ class mAP(nn.Layer): gallery_img_id) mAP = np.mean(all_AP) - metric_dict["mAP"] = mAP + metric_dict[self.name] = mAP return metric_dict class mINP(nn.Layer): - def __init__(self): + def __init__(self, name="mINP"): super().__init__() + self.name = name def forward(self, similarities_matrix, query_img_id, gallery_img_id): metric_dict = dict() @@ -62,7 +64,7 @@ class mINP(nn.Layer): gallery_img_id) mINP = np.mean(all_INP) - metric_dict["mINP"] = mINP + metric_dict[self.name] = mINP return metric_dict From f3f1c98943899eea76c19d9c8e75e2949956393a Mon Sep 17 00:00:00 2001 From: dongshuilong Date: Sat, 5 Jun 2021 18:33:50 +0800 Subject: [PATCH 3/3] fix yaml bugs --- ppcls/configs/Vehicle/ResNet50.yaml | 8 ++++---- ppcls/configs/Vehicle/ResNet50_ReID.yaml | 2 +- ppcls/metric/metrics.py | 10 ++++------ 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/ppcls/configs/Vehicle/ResNet50.yaml b/ppcls/configs/Vehicle/ResNet50.yaml index 23cb462cb..e7c4736db 100644 --- a/ppcls/configs/Vehicle/ResNet50.yaml +++ b/ppcls/configs/Vehicle/ResNet50.yaml @@ -148,9 +148,9 @@ Infer: Metric: Train: - - Topk: - k: [1, 5] + - TopkAcc: + topk: [1, 5] Eval: - - Topk: - k: [1, 5] + - TopkAcc: + topk: [1, 5] diff --git a/ppcls/configs/Vehicle/ResNet50_ReID.yaml b/ppcls/configs/Vehicle/ResNet50_ReID.yaml index 39862847e..a8d204744 100644 --- a/ppcls/configs/Vehicle/ResNet50_ReID.yaml +++ b/ppcls/configs/Vehicle/ResNet50_ReID.yaml @@ -147,7 +147,7 @@ Metric: Eval: - Recallk: topk: [1, 5] - - mAP: + - mAP: {} Infer: infer_imgs: "docs/images/whl/demo.jpg" diff --git a/ppcls/metric/metrics.py b/ppcls/metric/metrics.py index ac2866392..05723c123 100644 --- a/ppcls/metric/metrics.py +++ b/ppcls/metric/metrics.py @@ -39,9 +39,8 @@ class TopkAcc(nn.Layer): class mAP(nn.Layer): - def __init__(self, name="mAP"): + def __init__(self): super().__init__() - self.name = name def forward(self, similarities_matrix, query_img_id, gallery_img_id): metric_dict = dict() @@ -49,14 +48,13 @@ class mAP(nn.Layer): gallery_img_id) mAP = np.mean(all_AP) - metric_dict[self.name] = mAP + metric_dict["mAP"] = mAP return metric_dict class mINP(nn.Layer): - def __init__(self, name="mINP"): + def __init__(self): super().__init__() - self.name = name def forward(self, similarities_matrix, query_img_id, gallery_img_id): metric_dict = dict() @@ -64,7 +62,7 @@ class mINP(nn.Layer): gallery_img_id) mINP = np.mean(all_INP) - metric_dict[self.name] = mINP + metric_dict["mINP"] = mINP return metric_dict