# encoding: utf-8 """ @author: liaoxingyu @contact: sherlockliao01@gmail.com """ import numpy as np import copy from collections import defaultdict import sys import warnings try: from csrc.eval_cylib.eval_metrics_cy import evaluate_cy IS_CYTHON_AVAI = True print("Using Cython evaluation code as the backend") except ImportError: IS_CYTHON_AVAI = False warnings.warn("Cython evaluation is UNAVAILABLE, which is highly recommended") def eval_cuhk03(distmat, q_pids, g_pids, q_camids, g_camids, max_rank): """Evaluation with cuhk03 metric Key: one image for each gallery identity is randomly sampled for each query identity. Random sampling is performed num_repeats times. """ num_repeats = 10 num_q, num_g = distmat.shape if num_g < max_rank: max_rank = num_g print("Note: number of gallery samples is quite small, got {}".format(num_g)) indices = np.argsort(distmat, axis=1) matches = (g_pids[indices] == q_pids[:, np.newaxis]).astype(np.int32) # compute cmc curve for each query all_cmc = [] all_AP = [] num_valid_q = 0. # number of valid query for q_idx in range(num_q): # get query pid and camid q_pid = q_pids[q_idx] q_camid = q_camids[q_idx] # remove gallery samples that have the same pid and camid with query order = indices[q_idx] remove = (g_pids[order] == q_pid) & (g_camids[order] == q_camid) keep = np.invert(remove) # compute cmc curve raw_cmc = matches[q_idx][keep] # binary vector, positions with value 1 are correct matches if not np.any(raw_cmc): # this condition is true when query identity does not appear in gallery continue kept_g_pids = g_pids[order][keep] g_pids_dict = defaultdict(list) for idx, pid in enumerate(kept_g_pids): g_pids_dict[pid].append(idx) cmc, AP = 0., 0. for repeat_idx in range(num_repeats): mask = np.zeros(len(raw_cmc), dtype=np.bool) for _, idxs in g_pids_dict.items(): # randomly sample one image for each gallery person rnd_idx = np.random.choice(idxs) mask[rnd_idx] = True masked_raw_cmc = raw_cmc[mask] _cmc = masked_raw_cmc.cumsum() _cmc[_cmc > 1] = 1 cmc += _cmc[:max_rank].astype(np.float32) # compute AP num_rel = masked_raw_cmc.sum() tmp_cmc = masked_raw_cmc.cumsum() tmp_cmc = [x / (i + 1.) for i, x in enumerate(tmp_cmc)] tmp_cmc = np.asarray(tmp_cmc) * masked_raw_cmc AP += tmp_cmc.sum() / num_rel cmc /= num_repeats AP /= num_repeats all_cmc.append(cmc) all_AP.append(AP) num_valid_q += 1. assert num_valid_q > 0, "Error: all query identities do not appear in gallery" all_cmc = np.asarray(all_cmc).astype(np.float32) all_cmc = all_cmc.sum(0) / num_valid_q mAP = np.mean(all_AP) return all_cmc, mAP def eval_market1501(distmat, q_pids, g_pids, q_camids, g_camids, max_rank): """Evaluation with market1501 metric Key: for each query identity, its gallery images from the same camera view are discarded. """ num_q, num_g = distmat.shape if num_g < max_rank: max_rank = num_g print("Note: number of gallery samples is quite small, got {}".format(num_g)) indices = np.argsort(distmat, axis=1) matches = (g_pids[indices] == q_pids[:, np.newaxis]).astype(np.int32) # compute cmc curve for each query all_cmc = [] all_AP = [] num_valid_q = 0. # number of valid query for q_idx in range(num_q): # get query pid and camid q_pid = q_pids[q_idx] q_camid = q_camids[q_idx] # remove gallery samples that have the same pid and camid with query order = indices[q_idx] remove = (g_pids[order] == q_pid) & (g_camids[order] == q_camid) keep = np.invert(remove) # compute cmc curve raw_cmc = matches[q_idx][keep] # binary vector, positions with value 1 are correct matches if not np.any(raw_cmc): # this condition is true when query identity does not appear in gallery continue cmc = raw_cmc.cumsum() cmc[cmc > 1] = 1 all_cmc.append(cmc[:max_rank]) num_valid_q += 1. # compute average precision # reference: https://en.wikipedia.org/wiki/Evaluation_measures_(information_retrieval)#Average_precision num_rel = raw_cmc.sum() tmp_cmc = raw_cmc.cumsum() tmp_cmc = [x / (i + 1.) for i, x in enumerate(tmp_cmc)] tmp_cmc = np.asarray(tmp_cmc) * raw_cmc AP = tmp_cmc.sum() / num_rel all_AP.append(AP) assert num_valid_q > 0, "Error: all query identities do not appear in gallery" all_cmc = np.asarray(all_cmc).astype(np.float32) all_cmc = all_cmc.sum(0) / num_valid_q mAP = np.mean(all_AP) return all_cmc, mAP def evaluate_py(distmat, q_pids, g_pids, q_camids, g_camids, max_rank, use_metric_cuhk03): if use_metric_cuhk03: return eval_cuhk03(distmat, q_pids, g_pids, q_camids, g_camids, max_rank) else: return eval_market1501(distmat, q_pids, g_pids, q_camids, g_camids, max_rank) def evaluate(distmat, q_pids, g_pids, q_camids, g_camids, max_rank=50, use_metric_cuhk03=False, use_cython=True): if use_cython and IS_CYTHON_AVAI: return evaluate_cy(distmat, q_pids, g_pids, q_camids, g_camids, max_rank, use_metric_cuhk03) else: return evaluate_py(distmat, q_pids, g_pids, q_camids, g_camids, max_rank, use_metric_cuhk03)