import argparse import json import os from pathlib import Path from threading import Thread import numpy as np import torch import yaml from tqdm import tqdm from models.experimental import attempt_load from utils.datasets import create_dataloader from utils.general import coco80_to_coco91_class, check_dataset, check_file, check_img_size, check_requirements, \ box_iou, non_max_suppression, scale_coords, xyxy2xywh, xywh2xyxy, set_logging, increment_path, colorstr from utils.metrics import ap_per_class, ConfusionMatrix, range_bar_plot, range_p_r_bar_plot from utils.plots import plot_images, output_to_target, plot_study_txt, append_to_txt from utils.torch_utils import select_device, time_synchronized, TracedModel import pandas as pd from yolo_object_embeddings import ObjectEmbeddingVisualizer def object_size_to_range(obj_height_pixels: float, focal:int, class_id:int=1): class_height = {0:1.5, 1:1.8} # car Sedan height = 1.5 m , person height is 1.8m pixel_size = 17e-6 obj_height_m = class_height[class_id] return obj_height_m * focal * 1e-3 / (obj_height_pixels * pixel_size) def test(data, weights=None, batch_size=32, imgsz=640, conf_thres=0.001, iou_thres=0.6, # used for NMS save_json=False, single_cls=False, augment=False, verbose=False, model=None, dataloader=None, save_dir=Path(''), # for saving images save_txt=False, # for auto-labelling save_hybrid=False, # for hybrid auto-labelling save_conf=False, # save auto-label confidences plots=True, wandb_logger=None, compute_loss=None, half_precision=True, trace=False, is_coco=False, v5_metric=False, **kwargs): # Initialize/load model and set device hyp = kwargs['hyp'] training = model is not None if training: # called by train.py device = next(model.parameters()).device # get model device else: # called directly set_logging() device = select_device(opt.device, batch_size=batch_size) # Directories if opt.save_path == '': save_dir = Path(increment_path(Path(opt.project) / opt.name, exist_ok=opt.exist_ok)) # increment run else: save_dir = Path(increment_path(os.path.join(opt.save_path, Path(opt.project) , opt.name), exist_ok=opt.exist_ok)) try: # no suduer can fail (save_dir / 'labels' if save_txt else save_dir).mkdir(parents=True, exist_ok=True) # make dir except Exception as e: print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!",e) # Load model model = attempt_load(weights, map_location=device) # load FP32 model gs = max(int(model.stride.max()), 32) # grid size (max stride) imgsz = check_img_size(imgsz, s=gs) # check img_size if trace: model = TracedModel(model, device, imgsz, opt.input_channels) #torch.backends.cudnn.benchmark = True ##uses the inbuilt cudnn auto-tuner to find the fastest convolution algorithms. - # Half half = device.type != 'cpu' and half_precision # half precision only supported on CUDA @@ HK : TODO what are the consequences add : if half: model.half() # Configure model.eval() if isinstance(data, str): is_coco = data.endswith('coco.yaml') with open(data) as f: data = yaml.load(f, Loader=yaml.SafeLoader) check_dataset(data) # check nc = 1 if single_cls else int(data['nc']) # number of classes iouv = torch.linspace(0.5, 0.95, 10).to(device) # iou vector for mAP@0.5:0.95 niou = iouv.numel() # Logging log_imgs = 0 if wandb_logger and wandb_logger.wandb: log_imgs = min(wandb_logger.log_imgs, 100) # Dataloader embed_analyse = kwargs.get('embed_analyse', False) model_name = kwargs.get('model_name', '') if not training: if device.type != 'cpu': model(torch.zeros(1, opt.input_channels, imgsz, imgsz).to(device).type_as(next(model.parameters()))) # run once task = opt.task if opt.task in ('train', 'val', 'test') else 'val' # path to train/val/test images hyp = dict() hyp['person_size_small_medium_th'] = 32 * 32 hyp['car_size_small_medium_th'] = 44 * 44 hyp['img_percentile_removal'] = 0.3 hyp['beta'] = 0.3 hyp['gamma'] = 80 # dummy anyway augmentation is disabled hyp['gamma_liklihood'] = 0.01 hyp['random_pad'] = True hyp['copy_paste'] = False # augment=False explicit no augmentation to test dataloader = create_dataloader(data[task], imgsz, batch_size, gs, opt, hyp, pad=0.5, augment=False, rect=False, #rect was True # HK@@@ TODO : why pad =0.5?? only effective in rect=True in test time ? https://github.com/ultralytics/ultralytics/issues/13271 prefix=colorstr(f'{task}: '), rel_path_images=data['path'], num_cls=data['nc'])[0] labels = np.concatenate(dataloader.dataset.labels, 0) class_labels = torch.tensor(labels[:, 0]) # classes with open(save_dir / 'opt.yaml', 'w') as f: yaml.dump(vars(opt), f, sort_keys=False) with open(save_dir / 'hyp.yaml', 'w') as f: yaml.dump(hyp, f, sort_keys=False) if v5_metric: print("Testing with YOLOv5 AP metric...") seen = 0 confusion_matrix = ConfusionMatrix(nc=nc, conf=conf_thres, iou_thres=iou_thres) # HK per conf per iou_thresh names = {k: v for k, v in enumerate(model.names if hasattr(model, 'names') else model.module.names)} if not training: print(100 * '==') print('Test set labels {} count : {}'.format(names, torch.bincount(class_labels.long(), minlength=nc) + 1)) coco91class = coco80_to_coco91_class() s = ('%20s' + '%12s' * 6) % ('Class', 'Images', 'Labels', 'P', 'R', 'mAP@.5', 'mAP@.5:.95') p, r, f1, mp, mr, map50, map, t0, t1 = 0., 0., 0., 0., 0., 0., 0., 0., 0. loss = torch.zeros(3, device=device) jdict, stats, ap, ap_class, wandb_images = [], [], [], [], [] # res_all = list() predictions_df = pd.DataFrame(columns=[ 'image_id', 'pred_cls', 'bbox_x', 'bbox_y', 'bbox_w', 'bbox_h', 'score' ]) if embed_analyse: obj_embed_viz = ObjectEmbeddingVisualizer(model=model, device=device) features_acm = torch.empty((0, 1024)) # embedding dim of last scale 1024x20x20 labels_acm = np.array([]) stats_all_large, stats_person_medium = [], [] if dataloader.dataset.use_csv_meta_data_file: n_bins_of100m = 20 bin_size_100 = 100 bin_size_25 = 50 range_bins_map = {x.item():[0]*n_bins_of100m for x in pd.unique(dataloader.dataset.df_metadata['sensor_type'])} range_bins_precision_all_classes = {x.item():[np.array([0, 0])] *n_bins_of100m for x in pd.unique(dataloader.dataset.df_metadata['sensor_type'])} range_bins_recall_all_classes = {x.item():[np.array([0, 0])] *n_bins_of100m for x in pd.unique(dataloader.dataset.df_metadata['sensor_type'])} range_bins_support_gt = {x.item():[0]*n_bins_of100m for x in pd.unique(dataloader.dataset.df_metadata['sensor_type'])} gt_per_range_bins = {x.item(): [[] for _ in range(n_bins_of100m)] for x in pd.unique(dataloader.dataset.df_metadata['sensor_type'])}# collecting GT labels gt_path_per_range_bins = {x.item(): [[] for _ in range(n_bins_of100m)] for x in pd.unique(dataloader.dataset.df_metadata['sensor_type'])}# collecting GT labels bin_size_per_sensor = {} for sensor_type in pd.unique(dataloader.dataset.df_metadata['sensor_type']): exec('stats_all_sensor_type_{}'.format(sensor_type.item()) + '=[]') # 'stats_all_50' exec('stats_all_sensor_type_{}_with_range'.format(sensor_type.item()) + '=[]') # 'stats_all_50' sensor_focal = int(sensor_type.astype('str').split('_')[-1]) if sensor_focal > bin_size_100: bin_size_per_sensor.update({sensor_focal: bin_size_100}) else: bin_size_per_sensor.update({sensor_focal: bin_size_25}) for daytime in pd.unique(dataloader.dataset.df_metadata['part_in_day']): exec('stats_all_time_{}'.format(daytime.lower()) + '=[]') # 'stats_all_day' for weather_condition in pd.unique(dataloader.dataset.df_metadata['weather_condition']): if isinstance(weather_condition, str): exec('stats_all_weather_condition_{}'.format(weather_condition.lower()) + '=[]') # 'stats_all_day' sensor_type_vars = [key for key in vars().keys() if 'stats_all_sensor_type' in key and not '_with_range' in key] time_vars = [key for key in vars().keys() if 'stats_all_time' in key and not '_with_range' in key] weather_condition_vars = [key for key in vars().keys() if 'stats_all_weather_condition' in key and not '_with_range' in key] for batch_i, (img, targets, paths, shapes) in enumerate(tqdm(dataloader, desc=s)): img = img.to(device, non_blocking=True) img = img.half() if half else img.float() # uint8 to fp16/32 # img /= 255.0 # 0 - 255 to 0.0 - 1.0 c# already done inside dataloader targets = targets.to(device) nb, _, height, width = img.shape # batch size, channels, height, width with torch.no_grad(): # Run model t = time_synchronized() out, train_out = model(img, augment=augment) # inference out [batch, proposals, figures_of] figures_of :(4 coordination, obj conf, cls conf ) and training outputs(batch_size, anchor per scale, x,y dim of scale out 40x40 ,n_classes-conf+1-objectness+4-bbox ) over 3 scales diferent outputs (2,2,80,80,7), (2,2,40,40,7) : 640/8=40 t0 += time_synchronized() - t # out coco 80 classes : [1, 25200, 85] [batch, proposals_3_scales,4_box__coord+1_obj_score + n x classes] # Compute loss if compute_loss: loss += compute_loss([x.float() for x in train_out], targets)[1][:3] # box, obj, cls # Run NMS targets[:, 2:] *= torch.Tensor([width, height, width, height]).to(device) # to pixels lb = [targets[targets[:, 0] == i, 1:] for i in range(nb)] if save_hybrid else [] # for autolabelling t = time_synchronized() #NMS out = non_max_suppression(out, conf_thres=conf_thres, iou_thres=iou_thres, labels=lb, multi_label=True) # Does thresholding for class : list of detections, on (n,6) tensor per image [xyxy, conf, cls] # out = non_max_suppression(out, conf_thres=conf_thres, iou_thres=iou_thres, labels=lb, multi_label=False) # Does thresholding for class : list of detections, on (n,6) tensor per image [xyxy, conf, cls] t1 += time_synchronized() - t if trace and embed_analyse and np.sum([x.numel() for x in out])>0: # features are being saved/clone in the trace model version only TODO for others features, labels = obj_embed_viz.extract_object_grounded_features(feature_maps=model.features, predictions=out, image_shape=img.shape) features_acm = torch.cat((features_acm, features.detach().cpu()), dim=0) labels_acm = np.concatenate((labels_acm, labels), axis=0) # Statistics per image for si, pred in enumerate(out): # [bbox_coors, objectness_logit, class] labels = targets[targets[:, 0] == si, 1:] nl = len(labels) tcls = labels[:, 0].tolist() if nl else [] # target class path = Path(paths[si]) seen += 1 if len(pred) == 0: if nl: stats.append((torch.zeros(0, niou, dtype=torch.bool), torch.Tensor(), torch.Tensor(), tcls)) #niou for COCO 0.5:0.05:1 continue # Predictions predn = pred.clone() # *xyxy, conf, cls in predn [x y ,w ,h, conf, cls] taking top 300 after NMS scale_coords(img[si].shape[1:], predn[:, :4], shapes[si][0], shapes[si][1]) # native-space pred # Append to text file if save_txt: gn = torch.tensor(shapes[si][0])[[1, 0, 1, 0]] # normalization gain whwh for *xyxy, conf, cls in predn.tolist(): xywh = (xyxy2xywh(torch.tensor(xyxy).view(1, 4)) / gn).view(-1).tolist() # normalized xywh line = (cls, *xywh, conf) if save_conf else (cls, *xywh) # label format with open(save_dir / 'labels' / (path.stem + '.txt'), 'a') as f: f.write(('%g ' * len(line)).rstrip() % line + '\n') # W&B logging - Media Panel Plots if len(wandb_images) < log_imgs and wandb_logger.current_epoch > 0: # Check for test operation if wandb_logger.current_epoch % wandb_logger.bbox_interval == 0: box_data = [{"position": {"minX": xyxy[0], "minY": xyxy[1], "maxX": xyxy[2], "maxY": xyxy[3]}, "class_id": int(cls), "box_caption": "%s %.3f" % (names[cls], conf), "scores": {"class_score": conf}, "domain": "pixel"} for *xyxy, conf, cls in pred.tolist()] boxes = {"predictions": {"box_data": box_data, "class_labels": names}} # inference-space wandb_images.append(wandb_logger.wandb.Image(img[si], boxes=boxes, caption=path.name)) wandb_logger.log_training_progress(predn, path, names) if wandb_logger and wandb_logger.wandb_run else None # Append to pycocotools JSON dictionary if save_json: # [{"image_id": 42, "category_id": 18, "bbox": [258.15, 41.29, 348.26, 243.78], "score": 0.236}, ... image_id = int(path.stem) if path.stem.isnumeric() else path.stem box = xyxy2xywh(predn[:, :4]) # xywh box[:, :2] -= box[:, 2:] / 2 # xy center to top-left corner collect_info = list() for p, b in zip(pred.tolist(), box.tolist()): jdict.append({'image_id': image_id, 'category_id': coco91class[int(p[5])] if is_coco else int(p[5]), 'bbox': [round(x, 3) for x in b], 'score': round(p[4], 5)}) collect_info.append({'category_id': coco91class[int(p[5])] if is_coco else int(p[5]), 'bbox': [round(x, 3) for x in b], 'score': round(p[4], 5)}) predictions_df = pd.concat([ predictions_df, pd.DataFrame({ 'image_id': [image_id], 'pred_cls': [coco91class[int(p[5])] if is_coco else int(p[5])], 'bbox_x': [[round(x, 3) for x in b][0]], 'bbox_y': [[round(x, 3) for x in b][1]], 'bbox_w': [[round(x, 3) for x in b][2]], 'bbox_h': [[round(x, 3) for x in b][3]], 'score': [round(p[4], 5)] }) ], ignore_index=True) for it in labels.cpu().numpy(): # jdict.append({'image_id': image_id, # 'gt_cls': it[0], # 'bbox': [round(x, 3) for x in it[1:]]}) predictions_df = pd.concat([ predictions_df, pd.DataFrame({ 'image_id': [image_id], 'gt_cls': [it[0]], 'bbox': [[round(x, 3) for x in it[1:]]]}) ], ignore_index=True) # Assign all predictions as incorrect ; pred takes top 300 predictions conf over 10 ious [0.5:0.95:0.05] correct = torch.zeros(pred.shape[0], niou, dtype=torch.bool, device=device) if nl: detected = [] # target indices tcls_tensor = labels[:, 0] # target boxes tbox = xywh2xyxy(labels[:, 1:5]) scale_coords(img[si].shape[1:], tbox, shapes[si][0], shapes[si][1]) # native-space labels if plots: confusion_matrix.process_batch(predn, torch.cat((labels[:, 0:1], tbox), 1)) # Per target class for cls in torch.unique(tcls_tensor): ti = (cls == tcls_tensor).nonzero(as_tuple=False).view(-1) # target indices pi = (cls == pred[:, 5]).nonzero(as_tuple=False).view(-1) # prediction indices # Search for detections if pi.shape[0]: # Prediction to target ious ious, i = box_iou(predn[pi, :4], tbox[ti]).max(1) # best ious, indices # Append detections detected_set = set() for j in (ious > iouv[0]).nonzero(as_tuple=False): # iouv[0]=0.5 IOU for dectetions iouv in general are all 0.5:0.05:.. for COCO d = ti[i[j]] # detected target if d.item() not in detected_set: detected_set.add(d.item()) detected.append(d) correct[pi[j]] = ious[j] > iouv # iou_thres is 1xn if len(detected) == nl: # all targets already located in image break # Append statistics (correct, conf_objectness, pcls, tcls) Predicted class is Max-Likelihood among all classes logit and threshol goes over the objectness only stats.append((correct.cpu(), pred[:, 4].cpu(), pred[:, 5].cpu(), tcls)) # correct @ IOU=0.5 of pred box with target if save_json: predictions_df = pd.concat([ predictions_df, pd.DataFrame({ 'image_id': [image_id], 'correct': [correct[:, 0].cpu()] }) ], ignore_index=True) if 1: #not training or 1: # assert len(pred[:, :4]) == 1 x, y, w, h = xyxy2xywh(pred[:, :4])[0]##HK BUG !! need to go over all preds see which indexes aligned to which value if w * h > hyp['person_size_small_medium_th'] and w * h <= hyp['car_size_small_medium_th']: stats_person_medium.append((correct.cpu(), pred[:, 4].cpu(), pred[:, 5].cpu(), tcls)) # [(ix, w.cpu() * h.cpu()) for ix, (x, y, w, h) in enumerate(xyxy2xywh(pred[:, :4])) if w * h > hyp['person_size_small_medium_th'] and w * h <= hyp['car_size_small_medium_th']] if w * h > hyp['car_size_small_medium_th']: stats_all_large.append((correct.cpu(), pred[:, 4].cpu(), pred[:, 5].cpu(), tcls)) # sensor type if dataloader.dataset.use_csv_meta_data_file: try: weather_condition = (dataloader.dataset.df_metadata[dataloader.dataset.df_metadata['tir_frame_image_file_name'] == str(path).split('/')[-1]]['weather_condition'].item()) if isinstance(weather_condition, str): weather_condition = weather_condition.lower() exec([x for x in weather_condition_vars if str(weather_condition) in x][0] + '.append((correct.cpu(), pred[:, 4].cpu(), pred[:, 5].cpu(), tcls))') except Exception as e: print(f'{weather_condition} fname WARNING: Ignoring corrupted image and/or label {weather_condition}: {e}') time_in_day = dataloader.dataset.df_metadata[dataloader.dataset.df_metadata['tir_frame_image_file_name'] == str(path).split('/')[-1]]['part_in_day'].item().lower() # eval([x for x in time_vars if str(time_in_day) in x][0]).append((correct.cpu(), pred[:, 4].cpu(), pred[:, 5].cpu(), tcls)) exec([x for x in time_vars if str(time_in_day) in x][0] + '.append((correct.cpu(), pred[:, 4].cpu(), pred[:, 5].cpu(), tcls))') sensor_type = dataloader.dataset.df_metadata[dataloader.dataset.df_metadata['tir_frame_image_file_name'] == str(path).split('/')[-1]]['sensor_type'].item() # obj_range_m = torch.tensor( # [(object_size_to_range(obj_height_pixels=h.cpu(), focal=sensor_type, class_id=class_id.cpu().numpy().item())) # for class_id, (x, y, w, h) in zip(pred[:, 5], xyxy2xywh(pred[:, :4]))]) gt_range = [(object_size_to_range(obj_height_pixels=h, focal=sensor_type, class_id=class_id.numpy().item())) for class_id, (x, y, w, h) in zip(labels[:, 0].cpu(), labels[:, 1:5].cpu())] # coupling the range cell between any overlapped IOU >TH between pred bbox and GT bbox obj_range_m = list() i = 0 for class_id_pred, (x1_p, y1_p, x2_p, y2_p) in zip(pred[:, 5].cpu(), pred[:, :4].cpu()): range_candidate = torch.tensor( object_size_to_range(obj_height_pixels=xyxy2xywh(pred[:, :4])[0][-1].cpu(), focal=sensor_type, class_id=class_id_pred.cpu().numpy().item())) # Find any IOU overlapped between GT and prediction for class_id_gt, (x1_gt, y1_gt, x2_gt, y2_gt), (xc,yc,w,h) in zip(labels[:, 0].cpu(), xywh2xyxy(labels[:, 1:5].cpu()), labels[:, 1:5].cpu()): ious = box_iou(torch.tensor((x1_gt, y1_gt, x2_gt, y2_gt)).unsqueeze(axis=0), torch.tensor((x1_p, y1_p, x2_p, y2_p)).unsqueeze(axis=0)) i += 1 if ious > iouv.cpu()[0]: # range_candidate = torch.tensor( object_size_to_range(obj_height_pixels=h.cpu(), focal=sensor_type, class_id=class_id_pred.cpu().numpy().item())) break # the aligned GT/Pred was found no need to iterate more, this is the atmost candidate obj_range_m.append(range_candidate) # else: #ranges = func(sqrt(height*width)) # obj_range_m = torch.tensor([(object_size_to_range(obj_height_pixels=(np.sqrt(h.cpu()*w.cpu())), focal=sensor_type)) for ix, (x, y, w, h) in enumerate(xyxy2xywh(pred[:, :4]))]) # gt_range = [(object_size_to_range(obj_height_pixels=(np.sqrt(h*w)), focal=sensor_type)) for ix, (x, y, w, h) in enumerate(labels[:,1:5].cpu())] # if 1: gt_range = [_range // 100 for _range in gt_range] #gt_range = [_range // 100 for _range in gt_range] else: gt_range = [_range//bin_size_per_sensor[sensor_type] for _range in gt_range] for gt_lbl, rng_ in zip(labels[:,0], gt_range): if rng_ < n_bins_of100m : gt_per_range_bins[sensor_type][int(rng_.item())].append(int(gt_lbl.item())) # add to each range bin GT the GT counts gt_path_per_range_bins[sensor_type][int(rng_.item())].append(str(path)) # (obj_range_m.cpu().reshape(-1)) exec([x for x in sensor_type_vars if str(sensor_type) in x][0] + '.append((correct.cpu(), pred[:, 4].cpu(), pred[:, 5].cpu(), tcls))') exec([x+'_with_range' for x in sensor_type_vars if str(sensor_type) in x][0] + '.append((correct.cpu(), pred[:, 4].cpu(), pred[:, 5].cpu(), tcls, obj_range_m, pred[:, 5].shape[0]*[str(path)]))') # path is replicated to match each prediction TP/FP in the image # for daytime in pd.unique(dataloader.dataset.df_metadata['part_in_day']): # exec('stats_all_part_in_day{}'.format(daytime.lower()) + '=[]') # 'stats_all_day' # Plot images aa = np.repeat(img[0,:,:,:].cpu().permute(1,2,0).numpy(), 3, axis=2).astype('float32') cv2.imwrite('test/exp40/test_batch88_labels__.jpg', aa*255) if (plots and batch_i > 10): # conf_thresh_plot = 0.1 # the plot threshold the connfidence f = save_dir / f'test_batch{batch_i}_labels.jpg' # labels Thread(target=plot_images, args=(img, targets, paths, f, names), daemon=True).start() f = save_dir / f'test_batch{batch_i}_pred.jpg' # predictions Thread(target=plot_images, args=(img, output_to_target(out), paths, f, names), daemon=True).start() # Compute statistics stats = [np.concatenate(x, 0) for x in zip(*stats)] # to numpy if trace and embed_analyse: embeddings = obj_embed_viz.visualize_object_embeddings(features_acm, labels_acm, path=save_dir, tag=opt.conf_thres) if not training or 1: stats_person_medium = [np.concatenate(x, 0) for x in zip(*stats_person_medium)] # to numpy stats_all_large = [np.concatenate(x, 0) for x in zip(*stats_all_large)] # to numpy if dataloader.dataset.use_csv_meta_data_file: for time_var in time_vars: exec('{}=[np.concatenate(x, 0) for x in zip(*{})]'.format(time_var, time_var)) # 'stats_all_50' for sensor_type in sensor_type_vars: exec('{}=[np.concatenate(x, 0) for x in zip(*{})]'.format(sensor_type, sensor_type)) # 'stats_all_50' exec('{}_with_range=[np.concatenate(x, 0) for x in zip(*{}_with_range)]'.format(sensor_type, sensor_type)) # 'stats_all_50' for weather_condition in weather_condition_vars: exec('{}=[np.concatenate(x, 0) for x in zip(*{})]'.format(weather_condition, weather_condition)) # 'stats_all_50' if len(stats) and stats[0].any(): nt = np.bincount(stats[3].astype(np.int64), minlength=nc) # number of targets per class p, r, ap, f1, ap_class = ap_per_class(*stats, plot=plots, v5_metric=v5_metric, save_dir=save_dir, names=names, class_support=nt, tag=model_name) #based on correct @ IOU=0.5 of pred box with target ap50, ap = ap[:, 0], ap.mean(1) # AP@0.5, AP@0.5:0.95 mp, mr, map50, map = p.mean(), r.mean(), ap50.mean(), ap.mean() # if bool(stats_person_medium): # p_med, r_med, ap_med, f1_med, ap_class_med = ap_per_class(*stats_person_medium, plot=plots, v5_metric=v5_metric, save_dir=save_dir, names=names, tag='small_objects') # ap50_med, ap_med = ap_med[:, 0], ap_med.mean(1) # AP@0.5, AP@0.5:0.95 # mp_med, mr_med, map50_med, map_med = p_med.mean(), r_med.mean(), ap50_med.mean(), ap_med.mean() # nt_med = np.bincount(stats_person_medium[3].astype(np.int64), minlength=nc) # number of targets per class # # if bool(stats_all_large): # p_large, r_large, ap_large, f1_large, ap_class_large = ap_per_class(*stats_all_large, plot=plots, v5_metric=v5_metric, save_dir=save_dir, names=names, tag='large_objects') # ap50_large, ap_large = ap_large[:, 0], ap_large.mean(1) # AP@0.5, AP@0.5:0.95 # mp_large, mr_large, map50_large, map_large = p_large.mean(), r_large.mean(), ap50_large.mean(), ap_large.mean() # nt_large = np.bincount(stats_all_large[3].astype(np.int64), minlength=nc) # number of targets per class if dataloader.dataset.use_csv_meta_data_file: for time_var in time_vars: if bool(eval(time_var)): # TODO in the names of classes in labels add the support from nt_, also pass the support of overall and the title name like in the tag but only day /night to the title inside plot_pr_curve() exec("nt_{} = np.bincount({}[3].astype(np.int64), minlength={})".format(time_var, time_var, nc)) exec("p_{}, r_{}, ap_{}, f1_{}, ap_class_{} = ap_per_class(*{}, plot={}, v5_metric={}, save_dir={}, names={}, tag={}, class_support=nt_{})".format(time_var, time_var, time_var, time_var, time_var, time_var, plots, v5_metric, 'str(save_dir)', names, 'str(time_var)', time_var)) exec("ap50_{}, ap_{} = ap_{}[:, 0], ap_{}.mean(1)".format(time_var, time_var, time_var, time_var)) exec("mp_{}, mr_{}, map50_{}, map_{} = p_{}.mean(), r_{}.mean(), ap50_{}.mean(), ap_{}.mean()".format(time_var, time_var, time_var, time_var, time_var, time_var, time_var, time_var)) for weather_condition in weather_condition_vars: exec("nt_{} = np.bincount({}[3].astype(np.int64), minlength={})".format(weather_condition, weather_condition, nc)) exec("p_{}, r_{}, ap_{}, f1_{}, ap_class_{} = ap_per_class(*{}, plot={}, v5_metric={}, save_dir={}, names={}, tag={}, class_support=nt_{})".format(weather_condition, weather_condition, weather_condition, weather_condition, weather_condition, weather_condition, plots, v5_metric, 'str(save_dir)', names, 'str(weather_condition)', weather_condition)) exec("ap50_{}, ap_{} = ap_{}[:, 0], ap_{}.mean(1)".format(weather_condition, weather_condition, weather_condition, weather_condition)) exec("mp_{}, mr_{}, map50_{}, map_{} = p_{}.mean(), r_{}.mean(), ap50_{}.mean(), ap_{}.mean()".format(weather_condition, weather_condition, weather_condition, weather_condition, weather_condition, weather_condition, weather_condition, weather_condition)) if 0 : #debug ranges100_pred = np.array([]) print('gt class dist per range cell of 100s and sum of GTs ') print([(100*(ix+1), np.unique(x, return_counts=True), np.size(x)) for ix, x in enumerate(gt_per_range_bins[210])]) exec('ranges100_pred={}_with_range[4]//100'.format('stats_all_sensor_type_210')) exec('cls_pred={}_with_range[2]'.format('stats_all_sensor_type_210')) exec('predicted={}_with_range[0]'.format('stats_all_sensor_type_210')) print('True predictions on 210', eval('ranges100_pred.shape')) np.array([np.size(x) for ix, x in enumerate(gt_per_range_bins[210])]).T # predictions may be TRue or False print('detections preds per range cell') np.unique(ranges100_pred, return_counts=True)[1].T [np.bincount(x, minlength=2) for ix, x in enumerate(gt_per_range_bins[210])] range_bin_ = 1 np.unique(cls_pred[ranges100_pred == range_bin_], return_counts=True) # count of good bad predictions per class stats_all_sensor_type_210_with_range[2][ranges100_pred == range_bin_] np.unique(stats_all_sensor_type_210_with_range[0][:, 0][ranges100_pred == range_bin_], return_counts=True) for sensor_type in sensor_type_vars: if bool(eval(sensor_type)): exec("nt_{} = np.bincount({}[3].astype(np.int64), minlength={})".format(sensor_type, sensor_type, nc)) exec("p_{}, r_{}, ap_{}, f1_{}, ap_class_{} = ap_per_class(*{}, plot={}, v5_metric={}, save_dir={}, names={}, tag={}, class_support=nt_{})".format(sensor_type, sensor_type, sensor_type, sensor_type, sensor_type, sensor_type, plots, v5_metric, 'str(save_dir)', names, 'str(sensor_type)',sensor_type)) exec("ap50_{}, ap_{} = ap_{}[:, 0], ap_{}.mean(1)".format(sensor_type, sensor_type, sensor_type, sensor_type)) exec("mp_{}, mr_{}, map50_{}, map_{} = p_{}.mean(), r_{}.mean(), ap50_{}.mean(), ap_{}.mean()".format(sensor_type, sensor_type, sensor_type, sensor_type, sensor_type, sensor_type, sensor_type, sensor_type)) sensor_focal = int(sensor_type.split('_')[-1]) if 1 :#sensor_focal > 100: # ML exec('ranges={}_with_range[4]//100'.format(sensor_type)) for rng_100 in range(0,n_bins_of100m): nt_stat_list_per_range = np.array([0, 0]) r_stat_list_per_range = np.array([0, 0]) p_stat_list_per_range = np.array([0, 0]) map50_per_range = np.array(0) # ind = np.array([]) # exec('ind = np.where(ranges == rng_100)[0]') ind = eval('np.where(ranges == rng_100)[0]') stat_list_per_range = list() if ind.size>0: # if there were detections at that range bin for ele in range(3):# taking the relevant preds related to the distance bin, since P/R/AP are computed globally vs. all GT/targets then it is compared to all targets stat_list_per_range.append(eval(sensor_type)[ele][ind]) # GT at thta bin range if not bool(gt_per_range_bins[sensor_focal][rng_100]): # predictions but no GT => FP=>low Prcesion map50_per_range = np.array(0) nt_stat_list_per_range = np.array([0,0]) r_stat_list_per_range = np.array([0,0]) p_stat_list_per_range = np.array([0,0]) else: stat_list_per_range.append(np.array([x for x in gt_per_range_bins[sensor_focal][rng_100]])) # add all targets/labels nt_stat_list_per_range = np.bincount(stat_list_per_range[3].astype(np.int64), minlength=nc) # GT count per bin range of classes p_stat_list_per_range, r_stat_list_per_range, ap_stat_list_per_range, f1_stat_list_per_range, \ ap_class_stat_list_per_range = ap_per_class(*stat_list_per_range, plot=plots, v5_metric=v5_metric, save_dir='', names=names, tag='', class_support=nt_stat_list_per_range) ap50_per_range, ap_per_range = ap_stat_list_per_range[:, 0], ap_stat_list_per_range.mean(1) # AP@0.5, AP@0.5:0.95 mp_per_range, mr_per_range, map50_per_range, map_per_range = p_stat_list_per_range.mean(), r_stat_list_per_range.mean(), ap50_per_range.mean(), ap_per_range.mean() else:# no prediction at this range r_stat_list_per_range = np.array([0, 0]) p_stat_list_per_range = np.array([0, 0]) fn = len(gt_per_range_bins[sensor_focal][rng_100]) recall = 0 # no TP 0/TP+FN precision = 0 map50_per_range = np.array(0) if not bool(gt_per_range_bins[sensor_focal][rng_100]):# there are no GT no pred nt_stat_list_per_range = np.array([0,0]) # actual GT else: nt_stat_list_per_range = np.array(gt_per_range_bins[sensor_focal][rng_100]).sum() # there are GT but no pred # print(map50_per_range) range_bins_map[sensor_focal][rng_100] = map50_per_range.item() range_bins_precision_all_classes[sensor_focal][rng_100] = nt_stat_list_per_range.astype('bool').astype('int')*p_stat_list_per_range # broadcast the count of each calss in case one of the classes are missing range_bins_recall_all_classes[sensor_focal][rng_100] = nt_stat_list_per_range.astype('bool').astype('int')*r_stat_list_per_range range_bins_support_gt[sensor_focal][rng_100] = nt_stat_list_per_range.sum().item() # bug_diff = np.array(range_bins_support_gt[210]) - np.array( # [np.size(x) for ix, x in enumerate(gt_per_range_bins[210])]).T # In case no Preds than there is no GT count in the if condition anyway if pred=0=>TP=0 than P=R=0 range_bar_plot(n_bins_of100m, range_bins_map, save_dir, range_bins_support=range_bins_support_gt) range_p_r_bar_plot(n_bins_of100m, range_bins_precision_all_classes, range_bins_recall_all_classes, save_dir, range_bins_support=range_bins_support_gt, names=names, conf=opt.conf_thres) # for time_var in time_vars: # for sensor_type in sensor_type_vars: # nt = np.bincount(stats[3].astype(np.int64), minlength=nc) # number of targets per class else: nt = torch.zeros(1) nt_med = torch.zeros(1) nt_large = torch.zeros(1) # Print results pf = '%20s' + '%12i' * 2 + '%12.3g' * 4 # print format print(pf % ('all', seen, nt.sum(), mp, mr, map50, map)) # if not training or 1: # if bool(stats_person_medium): # try: # print(pf % ('all_medium', seen, nt_med.sum(), mp_med, mr_med, map50_med, map_med)) # except Exception as e: # print(e) # # if bool(stats_all_large): # try: # print(pf % ('all_large', seen, nt_large.sum(), mp_large, mr_large, map50_large, map_large)) # except Exception as e: # print(e) file_path = os.path.join(save_dir, 'class_stats.txt') #'Class', 'Images', 'Labels', 'P', 'R', 'mAP@.5', 'mAP@.5:.95' append_to_txt(file_path, 'all', seen, nt.sum(), mp, mr, map50, map) # Print results per class if 1 or (verbose or (nc < 50 and not training)) and nc > 1 and len(stats): for i, c in enumerate(ap_class): print(pf % (names[c], seen, nt[c], p[i], r[i], ap50[i], ap[i])) append_to_txt(file_path, names[c], seen, nt[c], p[i], r[i], ap50[i], ap[i]) # try: # if bool(stats_person_medium): # for i, c in enumerate(ap_class_med): # print(pf % (names[c]+ '_med', seen, nt_med[c], p_med[i], r_med[i], ap50_med[i], ap_med[i])) # append_to_txt(file_path, names[c] + '_med', seen, nt_med[c], p_med[i], r_med[i], ap50_med[i], ap_med[i]) # except Exception as e: # print(e) # # try: # if bool(stats_all_large): # for i, c in enumerate(ap_class_large): # print(pf % (names[c]+ '_large', seen, nt_large[c], p_large[i], r_large[i], ap50_large[i], ap_large[i])) # append_to_txt(file_path, names[c] + '_large', seen, nt_large[c], p_large[i], r_large[i], ap50_large[i], ap_large[i]) # except Exception as e: # print(e) # Print speeds t = tuple(x / seen * 1E3 for x in (t0, t1, t0 + t1)) + (imgsz, imgsz, batch_size) # tuple if not training: print('Speed: %.1f/%.1f/%.1f ms inference/NMS/total per %gx%g image at batch-size %g' % t) # Plots if plots: confusion_matrix.plot(save_dir=save_dir, names=list(names.values())) if wandb_logger and wandb_logger.wandb: val_batches = [wandb_logger.wandb.Image(str(f), caption=f.name) for f in sorted(save_dir.glob('test*.jpg'))] wandb_logger.log({"Validation": val_batches}) if wandb_images: wandb_logger.log({"Bounding Box Debugger/Images": wandb_images}) # Save JSON if save_json and len(jdict): # @@ HK TODO: w = Path(weights[0] if isinstance(weights, list) else weights).stem if weights is not None else '' # weights anno_json = './coco/annotations/instances_val2017.json' # annotations json pred_json = str(save_dir / f"{w}_predictions.json") # predictions json pred_df_file = str(save_dir / f"{w}_predictions.csv") # predictions json print('\nEvaluating pycocotools mAP... saving %s...' % pred_json) with open(pred_json, 'w') as f: json.dump(jdict, f) try: # https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocoEvalDemo.ipynb from pycocotools.coco import COCO from pycocotools.cocoeval import COCOeval anno = COCO(anno_json) # init annotations api pred = anno.loadRes(pred_json) # init predictions api eval1 = COCOeval(anno, pred, 'bbox') if is_coco: eval1.params.imgIds = [int(Path(x).stem) for x in dataloader.dataset.img_files] # image IDs to evaluate eval1.evaluate() eval1.accumulate() eval1.summarize() map, map50 = eval1.stats[:2] # update results (mAP@0.5:0.95, mAP@0.5) except Exception as e: print(f'pycocotools unable to run: {e}') # Return results model.float() # for training if not training: s = f"\n{len(list(save_dir.glob('labels/*.txt')))} labels saved to {save_dir / 'labels'}" if save_txt else '' print(f"Results saved to {save_dir}{s}") maps = np.zeros(nc) + map for i, c in enumerate(ap_class): maps[c] = ap[i] if save_json: predictions_df.to_csv(pred_df_file, index=False) return (mp, mr, map50, map, *(loss.cpu() / len(dataloader)).tolist()), maps, t def sensor_type_breakdown_kpi(gt_per_range_bins, n_bins_of100m, names, nc, plots, range_bins_map, range_bins_support, save_dir, bin_size_per_sensor, v5_metric, **kwargs): # print(sensor_type_50) # print(sensor_type_210) # print(f_sensor_type_50) # print(f_sensor_type_210) # exec('{}=sensor_type_50'.format(f_sensor_type_50)) # exec('{}=sensor_type_210'.format(f_sensor_type_210)) # print([key for key in vars().keys() if 'stats_all_sensor_type' in key and not '_with_range' in key]) sensor_type_vars = [key for key in vars().keys() if 'stats_all_sensor_type' in key and not '_with_range' in key] # exec('{}=sensor_type_50'.format(f_sensor_type_50)) for sensor_type in sensor_type_vars: #sensor_type_vars: if bool(sensor_type): exec("nt_{} = np.bincount({}[3].astype(np.int64), minlength={})".format(sensor_type, sensor_type, nc)) exec( "p_{}, r_{}, ap_{}, f1_{}, ap_class_{} = ap_per_class(*{}, plot={}, v5_metric={}, save_dir={}, names={}, tag={}, class_support=nt_{})".format( sensor_type, sensor_type, sensor_type, sensor_type, sensor_type, sensor_type, plots, v5_metric, 'str(save_dir)', names, 'str(sensor_type)', sensor_type)) exec("ap50_{}, ap_{} = ap_{}[:, 0], ap_{}.mean(1)".format(sensor_type, sensor_type, sensor_type, sensor_type)) exec("mp_{}, mr_{}, map50_{}, map_{} = p_{}.mean(), r_{}.mean(), ap50_{}.mean(), ap_{}.mean()".format( sensor_type, sensor_type, sensor_type, sensor_type, sensor_type, sensor_type, sensor_type, sensor_type)) sensor_focal = int(sensor_type.split('_')[-1]) if 1: # sensor_focal > 100: # ML exec('ranges={}_with_range[3]//{}'.format(sensor_type, bin_size_per_sensor[sensor_focal])) for rng_100 in range(0, n_bins_of100m): # ind = np.array([]) # exec('ind = np.where(ranges == rng_100)[0]') ind = eval('np.where(ranges == rng_100)[0]') stat_list_per_range = list() if ind.any(): # if there were detections at that range bin for ele in range( 3): # taking the relevant preds related to the distance bin, since P/R/AP are computed globally vs. all GT/targets then it is compared to all targets stat_list_per_range.append(eval(sensor_type)[ele][ind]) # GT at thta bin range if not bool( gt_per_range_bins[sensor_focal][rng_100]): # predictions but no GT => FP=>low Prcesion map50_per_range = np.array(0) nt_stat_list_per_range = np.array(0) else: stat_list_per_range.append(np.array( [x for x in gt_per_range_bins[sensor_focal][rng_100]])) # add all targets/labels nt_stat_list_per_range = np.bincount(stat_list_per_range[3].astype(np.int64), minlength=nc) p_stat_list_per_range, r_stat_list_per_range, ap_stat_list_per_range, f1_stat_list_per_range, \ ap_class_stat_list_per_range = ap_per_class(*stat_list_per_range, plot=plots, v5_metric=v5_metric, save_dir='', names=names, tag='', class_support=nt_stat_list_per_range) ap50_per_range, ap_per_range = ap_stat_list_per_range[:, 0], ap_stat_list_per_range.mean( 1) # AP@0.5, AP@0.5:0.95 mp_per_range, mr_per_range, map50_per_range, map_per_range = p_stat_list_per_range.mean(), r_stat_list_per_range.mean(), ap50_per_range.mean(), ap_per_range.mean() else: # no prediction at this range if not bool(gt_per_range_bins[sensor_focal][rng_100]): # there are GT but no pred nt_stat_list_per_range = np.array(0) fn = len(gt_per_range_bins[sensor_focal][rng_100]) recall = 0 # no TP 0/TP+FN precision = 0 map50_per_range = np.array(0) print(map50_per_range) range_bins_map[sensor_focal][rng_100] = map50_per_range.item() range_bins_support[sensor_focal][rng_100] = nt_stat_list_per_range.sum().item() range_bar_plot(n_bins=17, range_bins=range_bins_map, save_dir=save_dir, range_bins_support=range_bins_support) # for time_var in time_vars: if __name__ == '__main__': parser = argparse.ArgumentParser(prog='test.py') parser.add_argument('--weights', nargs='+', type=str, default='yolov7.pt', help='model.pt path(s)') parser.add_argument('--data', type=str, default='data/coco.yaml', help='*.data path') parser.add_argument('--batch-size', type=int, default=32, help='size of each image batch') parser.add_argument('--img-size', type=int, default=640, help='inference size (pixels)') parser.add_argument('--conf-thres', type=float, default=0.001, help='object confidence threshold') parser.add_argument('--iou-thres', type=float, default=0.6, help='IOU threshold for NMS') parser.add_argument('--task', default='val', help='train, val, test, speed or study') parser.add_argument('--device', default='', help='cuda device, i.e. 0 or 0,1,2,3 or cpu') parser.add_argument('--single-cls', action='store_true', help='treat as single-class dataset') parser.add_argument('--augment', action='store_true', help='augmented inference') parser.add_argument('--verbose', action='store_true', help='report mAP by class') parser.add_argument('--save-txt', action='store_true', help='save results to *.txt') parser.add_argument('--save-hybrid', action='store_true', help='save label+prediction hybrid results to *.txt') parser.add_argument('--save-conf', action='store_true', help='save confidences in --save-txt labels') parser.add_argument('--save-json', action='store_true', help='save a cocoapi-compatible JSON results file') parser.add_argument('--project', default='runs/test', help='save to project/name') parser.add_argument('--name', default='exp', help='save to project/name') parser.add_argument('--exist-ok', action='store_true', help='existing project/name ok, do not increment') parser.add_argument('--no-trace', action='store_true', help='don`t trace model') parser.add_argument('--v5-metric', action='store_true', help='assume maximum recall as 1.0 in AP calculation') parser.add_argument('--norm-type', type=str, default='standardization', choices=['standardization', 'single_image_0_to_1', 'single_image_mean_std','single_image_percentile_0_255', 'single_image_percentile_0_1', 'remove+global_outlier_0_1'], help='Normalization approach') parser.add_argument('--no-tir-signal', action='store_true', help='') parser.add_argument('--tir-channel-expansion', action='store_true', help='drc_per_ch_percentile') parser.add_argument('--input-channels', type=int, default=3, help='') parser.add_argument('--save-path', default='', help='save to project/name') parser.add_argument('--csv-metadata-path', default='', help='save to project/name') parser.add_argument('--embed-analyse', action='store_true', help='') opt = parser.parse_args() if opt.tir_channel_expansion: # operates over 3 channels opt.input_channels = 3 if opt.tir_channel_expansion and opt.norm_type != 'single_image_percentile_0_1': # operates over 3 channels print('Not a good combination') opt.save_json |= opt.data.endswith('coco.yaml') opt.data = check_file(opt.data) # check file print(opt) #check_requirements() hyp = dict() model_name = str(opt.weights)[str(opt.weights).find('yolo'):].split('/')[0] if opt.task in ('train', 'val', 'test'): # run normally test(opt.data, opt.weights, opt.batch_size, opt.img_size, opt.conf_thres, opt.iou_thres, opt.save_json, opt.single_cls, opt.augment, opt.verbose, save_txt=opt.save_txt | opt.save_hybrid, save_hybrid=opt.save_hybrid, save_conf=opt.save_conf, trace=not opt.no_trace, v5_metric=opt.v5_metric, hyp=hyp, embed_analyse=opt.embed_analyse, model_name=model_name) elif opt.task == 'speed': # speed benchmarks for w in opt.weights: test(opt.data, w, opt.batch_size, opt.img_size, 0.25, 0.45, save_json=False, plots=False, v5_metric=opt.v5_metric) elif opt.task == 'study': # run over a range of settings and save/plot # python test.py --task study --data coco.yaml --iou 0.65 --weights yolov7.pt x = list(range(256, 1536 + 128, 128)) # x axis (image sizes) for w in opt.weights: f = f'study_{Path(opt.data).stem}_{Path(w).stem}.txt' # filename to save to y = [] # y axis for i in x: # img-size print(f'\nRunning {f} point {i}...') r, _, t = test(opt.data, w, opt.batch_size, i, opt.conf_thres, opt.iou_thres, opt.save_json, plots=False, v5_metric=opt.v5_metric) y.append(r + t) # results and times np.savetxt(f, y, fmt='%10.4g') # save os.system('zip -r study.zip study_*.txt') plot_study_txt(x=x) # plot """ --weights ./yolov7/yolov7.pt --device 0 --batch-size 16 --data data/coco_2_tir.yaml --img-size 640 --conf 0.6 --verbose --save-txt --save-hybrid --norm-type single_image_percentile_0_1 test based on RGB coco model --weights ./yolov7/yolov7.pt --device 0 --batch-size 64 --data data/coco_2_tir.yaml --img-size 640 --conf 0.25 --verbose --save-txt --norm-type single_image_percentile_0_1 --project test --task train --weights ./yolov7/yolov7.pt --device 0 --batch-size 64 --data data/tir_od.yaml --img-size 640 --conf 0.25 --verbose --save-txt --norm-type single_image_percentile_0_1 --project test --task val # Using pretrained model --weights /mnt/Data/hanoch/runs/train/yolov7434/weights/epoch_099.pt --device 0 --batch-size 4 --data data/tir_od_test_set.yaml --img-size 640 --conf 0.25 --verbose --norm-type single_image_percentile_0_1 --project test --task test #vbased on 7555 mAP=82.3 --weights /mnt/Data/hanoch/runs/train/yolov7563/weights/best.pt --device 0 --batch-size 16 --data data/tir_od_test_set.yaml --img-size 640 --conf 0.02 --verbose --norm-type single_image_percentile_0_1 --input-channels 1 --project test --task test --iou-thres 0.4 /home/hanoch/projects/tir_od/runs/train/yolov7563/weights --weights /mnt/Data/hanoch/runs/train/yolov7575/weights/best.pt --device 0 --batch-size 16 --data data/tir_od_test_set.yaml --img-size 640 --conf 0.001 --verbose --norm-type single_image_percentile_0_1 --input-channels 1 --project test --task test --iou-thres 0.6 --weights /home/hanoch/projects/tir_od/runs/gpu02/yolov74/weights --device 0 --batch-size 16 --data data/tir_od_test_set.yaml --img-size 640 --conf 0.001 --verbose --norm-type single_image_percentile_0_1 --input-channels 1 --project test --task test --iou-thres 0.6 3class /mnt/Data/hanoch/runs/train/yolov71058 --weights /mnt/Data/hanoch/runs/train/yolov71058/weights/best.pt --device 0 --batch-size 16 --data data/tir_od_center_roi_aug_list_train_cls.yaml --img-size 640 --conf 0.02 --verbose --norm-type single_image_percentile_0_1 --input-channels 1 --project test --task val --iou-thres 0.6 tir_tiff_w_center_roi_validation_set_train_cls_usa.txt # per day/nigh SY/ML --weights /mnt/Data/hanoch/runs/train/yolov7999/weights/best.pt --device 0 --batch-size 16 --data data/tir_od_test_set.yaml --img-size 640 --conf 0.001 --verbose --norm-type single_image_percentile_0_1 --input-channels 1 --project test --task test --iou-thres 0.6 --csv-metadata-path tir_od/tir_center_merged_seq_tiff_last_original_png.xlsx P/R --weights /mnt/Data/hanoch/runs/train/yolov7999/weights/best.pt --device 0 --batch-size 16 --data data/tir_od_test_set.yaml --img-size 640 --verbose --norm-type single_image_percentile_0_1 --input-channels 1 --project test --task test --iou-thres 0.6 --csv-metadata-path tir_od/tir_center_merged_seq_tiff_last_original_png.xlsx --conf 0.65 mAP: --weights /mnt/Data/hanoch/runs/train/yolov7999/weights/best.pt --device 0 --batch-size 16 --data data/tir_od_test_set.yaml --img-size 640 --verbose --norm-type single_image_percentile_0_1 --input-channels 1 --project test --task test --iou-thres 0.6 --csv-metadata-path tir_od/tir_center_merged_seq_tiff_last_original_png.xlsx --conf 0.01 Fixed wether csv P/R --weights /mnt/Data/hanoch/runs/train/yolov7999/weights/best.pt --device 0 --batch-size 16 --data data/tir_od_test_set.yaml --img-size 640 --verbose --norm-type single_image_percentile_0_1 --input-channels 1 --project test --task test --csv-metadata-path tir_od/tir_tiff_seq_png_3_class_fixed_whether.xlsx --iou-thres 0.6 --conf 0.65 FOG --weights /mnt/Data/hanoch/runs/train/yolov7999/weights/best.pt --device 0 --batch-size 16 --data data/tir_od_fog_set.yaml --img-size 640 --verbose --norm-type single_image_percentile_0_1 --input-channels 1 --project test --task test --csv-metadata-path tir_od/tir_tiff_seq_png_3_class_fixed_whether.xlsx --conf 0.65 --iou-thres 0.6 Locomotive --weights /mnt/Data/hanoch/runs/train/yolov71107/weights/best.pt --device 0 --batch-size 16 --data data/tir_od_test_set_3_class_train.yaml --img-size 640 --verbose --norm-type single_image_percentile_0_1 --input-channels 1 --project test --task test --iou-thres 0.6 --conf 0.1 --embed-analyse --weights /mnt/Data/hanoch/runs/train/yolov71133/weights/best.pt --device 0 --batch-size 16 --data data/tir_od_test_set_3_class_train.yaml --img-size 640 --verbose --norm-type single_image_percentile_0_1 --input-channels 1 --project test --task test --iou-thres 0.6 --conf 0.4 ------- Error analysis ------------ 1st run with conf_th=0.0001 then observe the desired threshold, re-run with the desired threshold abd observe images with bboxes given the deired threshold """