yolov7/test.py

import argparse
import json
import os
from pathlib import Path
from threading import Thread

import numpy as np
import torch
import yaml
from tqdm import tqdm

from models.experimental import attempt_load
from utils.datasets import create_dataloader
from utils.general import coco80_to_coco91_class, check_dataset, check_file, check_img_size, check_requirements, \
    box_iou, non_max_suppression, scale_coords, xyxy2xywh, xywh2xyxy, set_logging, increment_path, colorstr
from utils.metrics import ap_per_class, ConfusionMatrix, range_bar_plot, range_p_r_bar_plot
from utils.plots import plot_images, output_to_target, plot_study_txt, append_to_txt
from utils.torch_utils import select_device, time_synchronized, TracedModel
import pandas as pd
from yolo_object_embeddings import ObjectEmbeddingVisualizer

def object_size_to_range(obj_height_pixels: float, focal:int, class_id:int=1):
    class_height = {0:1.5, 1:1.8} # car Sedan height = 1.5 m , person height is 1.8m
    pixel_size = 17e-6
    obj_height_m = class_height[class_id]
    return obj_height_m * focal * 1e-3 / (obj_height_pixels * pixel_size)


def test(data,
         weights=None,
         batch_size=32,
         imgsz=640,
         conf_thres=0.001,
         iou_thres=0.6,  # used for NMS
         save_json=False,
         single_cls=False,
         augment=False,
         verbose=False,
         model=None,
         dataloader=None,
         save_dir=Path(''),  # for saving images
         save_txt=False,  # for auto-labelling
         save_hybrid=False,  # for hybrid auto-labelling
         save_conf=False,  # save auto-label confidences
         plots=True,
         wandb_logger=None,
         compute_loss=None,
         half_precision=True,
         trace=False,
         is_coco=False,
         v5_metric=False,
         **kwargs):
    # Initialize/load model and set device

    hyp = kwargs['hyp']
    training = model is not None
    if training:  # called by train.py
        device = next(model.parameters()).device  # get model device

    else:  # called directly
        set_logging()
        device = select_device(opt.device, batch_size=batch_size)

        # Directories

        if opt.save_path == '':
            save_dir = Path(increment_path(Path(opt.project) / opt.name, exist_ok=opt.exist_ok))  # increment run
        else:
            save_dir = Path(increment_path(os.path.join(opt.save_path, Path(opt.project) , opt.name), exist_ok=opt.exist_ok))

        try: # no suduer can fail
            (save_dir / 'labels' if save_txt else save_dir).mkdir(parents=True, exist_ok=True)  # make dir
        except Exception as e:
            print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!",e)

        # Load model
        model = attempt_load(weights, map_location=device)  # load FP32 model
        gs = max(int(model.stride.max()), 32)  # grid size (max stride)
        imgsz = check_img_size(imgsz, s=gs)  # check img_size
        
        if trace:
            model = TracedModel(model, device, imgsz, opt.input_channels)

    #torch.backends.cudnn.benchmark = True  ##uses the inbuilt cudnn auto-tuner to find the fastest convolution algorithms. -
    # Half
    half = device.type != 'cpu' and half_precision  # half precision only supported on CUDA @@ HK : TODO what are the consequences  add :
    if half:
        model.half()

    # Configure
    model.eval()
    if isinstance(data, str):
        is_coco = data.endswith('coco.yaml')
        with open(data) as f:
            data = yaml.load(f, Loader=yaml.SafeLoader)
    check_dataset(data)  # check
    nc = 1 if single_cls else int(data['nc'])  # number of classes
    iouv = torch.linspace(0.5, 0.95, 10).to(device)  # iou vector for mAP@0.5:0.95
    niou = iouv.numel()

    # Logging
    log_imgs = 0
    if wandb_logger and wandb_logger.wandb:
        log_imgs = min(wandb_logger.log_imgs, 100)
    # Dataloader

    embed_analyse = kwargs.get('embed_analyse', False)
    if not training:
        if device.type != 'cpu':
            model(torch.zeros(1, opt.input_channels, imgsz, imgsz).to(device).type_as(next(model.parameters())))  # run once
        task = opt.task if opt.task in ('train', 'val', 'test') else 'val'  # path to train/val/test images
        hyp = dict()
        hyp['person_size_small_medium_th'] = 32 * 32
        hyp['car_size_small_medium_th'] = 44 * 44

        hyp['img_percentile_removal'] = 0.3
        hyp['beta'] = 0.3
        hyp['gamma'] = 80 # dummy anyway augmentation is disabled
        hyp['gamma_liklihood'] = 0.01
        hyp['random_pad'] = True
        hyp['copy_paste'] = False
        # augment=False explicit no augmentation to test
        dataloader = create_dataloader(data[task], imgsz, batch_size, gs, opt, hyp, pad=0.5, augment=False, rect=False, #rect was True  # HK@@@ TODO : why pad =0.5?? only effective in rect=True in test time ? https://github.com/ultralytics/ultralytics/issues/13271
                                       prefix=colorstr(f'{task}: '), rel_path_images=data['path'], num_cls=data['nc'])[0]

        labels = np.concatenate(dataloader.dataset.labels, 0)
        class_labels = torch.tensor(labels[:, 0])  # classes


    if v5_metric:
        print("Testing with YOLOv5 AP metric...")
    
    seen = 0
    confusion_matrix = ConfusionMatrix(nc=nc, conf=conf_thres, iou_thres=iou_thres) # HK per conf per iou_thresh
    names = {k: v for k, v in enumerate(model.names if hasattr(model, 'names') else model.module.names)}

    if not training:
        print(100 * '==')
        print('Test set labels {} count : {}'.format(names, torch.bincount(class_labels.long(), minlength=nc) + 1))

    coco91class = coco80_to_coco91_class()
    s = ('%20s' + '%12s' * 6) % ('Class', 'Images', 'Labels', 'P', 'R', 'mAP@.5', 'mAP@.5:.95')
    p, r, f1, mp, mr, map50, map, t0, t1 = 0., 0., 0., 0., 0., 0., 0., 0., 0.
    loss = torch.zeros(3, device=device)
    jdict, stats, ap, ap_class, wandb_images = [], [], [], [], []
    # res_all = list()
    predictions_df = pd.DataFrame(columns=[
        'image_id',
        'pred_cls',
        'bbox_x',
        'bbox_y',
        'bbox_w',
        'bbox_h',
        'score'
    ])

    if embed_analyse:
        obj_embed_viz = ObjectEmbeddingVisualizer(model=model, device=device)
        features_acm = torch.empty((0, 1024)) # embedding dim of last scale 1024x20x20
        labels_acm = np.array([])

    stats_all_large, stats_person_medium = [], []
    if dataloader.dataset.use_csv_meta_data_file:

        n_bins_of100m = 20
        bin_size_100 = 100
        bin_size_25 = 50

        range_bins_map = {x.item():[0]*n_bins_of100m for x in pd.unique(dataloader.dataset.df_metadata['sensor_type'])}
        range_bins_precision_all_classes = {x.item():[np.array([0, 0])] *n_bins_of100m for x in pd.unique(dataloader.dataset.df_metadata['sensor_type'])}
        range_bins_recall_all_classes = {x.item():[np.array([0, 0])] *n_bins_of100m for x in pd.unique(dataloader.dataset.df_metadata['sensor_type'])}
        range_bins_support_gt = {x.item():[0]*n_bins_of100m for x in pd.unique(dataloader.dataset.df_metadata['sensor_type'])}
        gt_per_range_bins = {x.item(): [[] for _ in range(n_bins_of100m)] for x in pd.unique(dataloader.dataset.df_metadata['sensor_type'])}# collecting GT labels
        gt_path_per_range_bins = {x.item(): [[] for _ in range(n_bins_of100m)] for x in pd.unique(dataloader.dataset.df_metadata['sensor_type'])}# collecting GT labels
        bin_size_per_sensor = {}

        for sensor_type in pd.unique(dataloader.dataset.df_metadata['sensor_type']):
            exec('stats_all_sensor_type_{}'.format(sensor_type.item()) + '=[]')  # 'stats_all_50'
            exec('stats_all_sensor_type_{}_with_range'.format(sensor_type.item()) + '=[]')  # 'stats_all_50'

            sensor_focal = int(sensor_type.astype('str').split('_')[-1])

            if sensor_focal > bin_size_100:
                bin_size_per_sensor.update({sensor_focal: bin_size_100})
            else:
                bin_size_per_sensor.update({sensor_focal: bin_size_25})

        for daytime in pd.unique(dataloader.dataset.df_metadata['part_in_day']):
            exec('stats_all_time_{}'.format(daytime.lower()) + '=[]')  # 'stats_all_day'

        for weather_condition in pd.unique(dataloader.dataset.df_metadata['weather_condition']):
            if isinstance(weather_condition, str):
                exec('stats_all_weather_condition_{}'.format(weather_condition.lower()) + '=[]')  # 'stats_all_day'

        sensor_type_vars = [key for key in vars().keys() if 'stats_all_sensor_type' in key and not '_with_range' in key]
        time_vars = [key for key in vars().keys() if 'stats_all_time' in key and not '_with_range' in key]
        weather_condition_vars = [key for key in vars().keys() if 'stats_all_weather_condition' in key and not '_with_range' in key]

    for batch_i, (img, targets, paths, shapes) in enumerate(tqdm(dataloader, desc=s)):
        img = img.to(device, non_blocking=True)
        img = img.half() if half else img.float()
        # uint8 to fp16/32
        # img /= 255.0  # 0 - 255 to 0.0 - 1.0 c# already done inside dataloader
        targets = targets.to(device)
        nb, _, height, width = img.shape  # batch size, channels, height, width

        with torch.no_grad():
            # Run model
            t = time_synchronized()
            out, train_out = model(img, augment=augment)  # inference out [batch, proposals, figures_of] figures_of :(4 coordination, obj conf, cls conf ) and training outputs(batch_size, anchor per scale, x,y dim of scale out 40x40 ,n_classes-conf+1-objectness+4-bbox ) over 3 scales diferent outputs (2,2,80,80,7), (2,2,40,40,7)  : 640/8=40
            t0 += time_synchronized() - t
            # out coco 80 classes : [1, 25200, 85] [batch, proposals_3_scales,4_box__coord+1_obj_score + n x classes]
            # Compute loss
            if compute_loss:
                loss += compute_loss([x.float() for x in train_out], targets)[1][:3]  # box, obj, cls

            # Run NMS
            targets[:, 2:] *= torch.Tensor([width, height, width, height]).to(device)  # to pixels
            lb = [targets[targets[:, 0] == i, 1:] for i in range(nb)] if save_hybrid else []  # for autolabelling
            t = time_synchronized() #NMS
            out = non_max_suppression(out, conf_thres=conf_thres, iou_thres=iou_thres, labels=lb, multi_label=True) # Does thresholding for class  : list of detections, on (n,6) tensor per image [xyxy, conf, cls]
            # out = non_max_suppression(out, conf_thres=conf_thres, iou_thres=iou_thres, labels=lb, multi_label=False) # Does thresholding for class  : list of detections, on (n,6) tensor per image [xyxy, conf, cls]
            t1 += time_synchronized() - t

        if trace and embed_analyse and np.sum([x.numel() for x in out])>0: # features are being saved/clone in the trace model version only TODO for others
            features, labels = obj_embed_viz.extract_object_grounded_features(feature_maps=model.features,
                                                           predictions=out,
                                                           image_shape=img.shape)
            features_acm = torch.cat((features_acm, features.detach().cpu()), dim=0)
            labels_acm = np.concatenate((labels_acm, labels), axis=0)

        # Statistics per image
        for si, pred in enumerate(out): # [bbox_coors, objectness_logit, class]

            labels = targets[targets[:, 0] == si, 1:]
            nl = len(labels)
            tcls = labels[:, 0].tolist() if nl else []  # target class
            path = Path(paths[si])
            seen += 1

            if len(pred) == 0:
                if nl:
                    stats.append((torch.zeros(0, niou, dtype=torch.bool), torch.Tensor(), torch.Tensor(), tcls))    #niou for COCO 0.5:0.05:1
                continue

            # Predictions
            predn = pred.clone() # *xyxy, conf, cls in predn  [x y ,w ,h, conf, cls] taking top 300 after NMS
            scale_coords(img[si].shape[1:], predn[:, :4], shapes[si][0], shapes[si][1])  # native-space pred

            # Append to text file
            if save_txt:
                gn = torch.tensor(shapes[si][0])[[1, 0, 1, 0]]  # normalization gain whwh
                for *xyxy, conf, cls in predn.tolist():
                    xywh = (xyxy2xywh(torch.tensor(xyxy).view(1, 4)) / gn).view(-1).tolist()  # normalized xywh
                    line = (cls, *xywh, conf) if save_conf else (cls, *xywh)  # label format
                    with open(save_dir / 'labels' / (path.stem + '.txt'), 'a') as f:
                        f.write(('%g ' * len(line)).rstrip() % line + '\n')

            # W&B logging - Media Panel Plots
            if len(wandb_images) < log_imgs and wandb_logger.current_epoch > 0:  # Check for test operation
                if wandb_logger.current_epoch % wandb_logger.bbox_interval == 0:
                    box_data = [{"position": {"minX": xyxy[0], "minY": xyxy[1], "maxX": xyxy[2], "maxY": xyxy[3]},
                                 "class_id": int(cls),
                                 "box_caption": "%s %.3f" % (names[cls], conf),
                                 "scores": {"class_score": conf},
                                 "domain": "pixel"} for *xyxy, conf, cls in pred.tolist()]
                    boxes = {"predictions": {"box_data": box_data, "class_labels": names}}  # inference-space
                    wandb_images.append(wandb_logger.wandb.Image(img[si], boxes=boxes, caption=path.name))
            wandb_logger.log_training_progress(predn, path, names) if wandb_logger and wandb_logger.wandb_run else None

            # Append to pycocotools JSON dictionary
            if save_json:
                # [{"image_id": 42, "category_id": 18, "bbox": [258.15, 41.29, 348.26, 243.78], "score": 0.236}, ...
                image_id = int(path.stem) if path.stem.isnumeric() else path.stem
                box = xyxy2xywh(predn[:, :4])  # xywh
                box[:, :2] -= box[:, 2:] / 2  # xy center to top-left corner
                collect_info = list()
                for p, b in zip(pred.tolist(), box.tolist()):
                    jdict.append({'image_id': image_id,
                                  'category_id': coco91class[int(p[5])] if is_coco else int(p[5]),
                                  'bbox': [round(x, 3) for x in b],
                                  'score': round(p[4], 5)})

                    collect_info.append({'category_id': coco91class[int(p[5])] if is_coco else int(p[5]),
                                  'bbox': [round(x, 3) for x in b],
                                  'score': round(p[4], 5)})

                    predictions_df = pd.concat([
                        predictions_df,
                        pd.DataFrame({
                            'image_id': [image_id],
                            'pred_cls': [coco91class[int(p[5])] if is_coco else int(p[5])],
                            'bbox_x': [[round(x, 3) for x in b][0]],
                            'bbox_y': [[round(x, 3) for x in b][1]],
                            'bbox_w': [[round(x, 3) for x in b][2]],
                            'bbox_h': [[round(x, 3) for x in b][3]],
                            'score':  [round(p[4], 5)]
                        })
                    ], ignore_index=True)

                    for it in labels.cpu().numpy():
                        # jdict.append({'image_id': image_id,
                        #               'gt_cls': it[0],
                        #               'bbox': [round(x, 3) for x in it[1:]]})

                        predictions_df = pd.concat([
                            predictions_df,
                            pd.DataFrame({
                                'image_id': [image_id],
                                'gt_cls': [it[0]],
                                'bbox': [[round(x, 3) for x in it[1:]]]})
                        ], ignore_index=True)

            # Assign all predictions as incorrect ; pred takes top 300 predictions conf over 10 ious [0.5:0.95:0.05]
            correct = torch.zeros(pred.shape[0], niou, dtype=torch.bool, device=device)
            if nl:
                detected = []  # target indices
                tcls_tensor = labels[:, 0]

                # target boxes
                tbox = xywh2xyxy(labels[:, 1:5])
                scale_coords(img[si].shape[1:], tbox, shapes[si][0], shapes[si][1])  # native-space labels
                if plots:
                    confusion_matrix.process_batch(predn, torch.cat((labels[:, 0:1], tbox), 1))

                # Per target class
                for cls in torch.unique(tcls_tensor):
                    ti = (cls == tcls_tensor).nonzero(as_tuple=False).view(-1)  # target indices
                    pi = (cls == pred[:, 5]).nonzero(as_tuple=False).view(-1)  # prediction indices

                    # Search for detections
                    if pi.shape[0]:
                        # Prediction to target ious
                        ious, i = box_iou(predn[pi, :4], tbox[ti]).max(1)  # best ious, indices

                        # Append detections
                        detected_set = set()
                        for j in (ious > iouv[0]).nonzero(as_tuple=False): # iouv[0]=0.5 IOU for dectetions iouv in general are all 0.5:0.05:.. for COCO
                            d = ti[i[j]]  # detected target
                            if d.item() not in detected_set:
                                detected_set.add(d.item())
                                detected.append(d)
                                correct[pi[j]] = ious[j] > iouv  # iou_thres is 1xn
                                if len(detected) == nl:  # all targets already located in image
                                    break

            # Append statistics (correct, conf_objectness, pcls, tcls) Predicted class is Max-Likelihood among all classes logit and threshol goes over the objectness only
            stats.append((correct.cpu(), pred[:, 4].cpu(), pred[:, 5].cpu(), tcls)) # correct @ IOU=0.5 of pred box with target

            if save_json:
                predictions_df = pd.concat([
                    predictions_df,
                    pd.DataFrame({
                        'image_id': [image_id],
                        'correct': [correct[:, 0].cpu()]
                    })
                ], ignore_index=True)


            if 1: #not training or 1:
                # assert len(pred[:, :4]) == 1
                x, y, w, h = xyxy2xywh(pred[:, :4])[0]##HK BUG !! need to go over all preds see which indexes aligned to which value
                if w * h > hyp['person_size_small_medium_th'] and  w * h <= hyp['car_size_small_medium_th']:
                    stats_person_medium.append((correct.cpu(), pred[:, 4].cpu(), pred[:, 5].cpu(), tcls))
                #     [(ix, w.cpu() * h.cpu()) for ix, (x, y, w, h) in enumerate(xyxy2xywh(pred[:, :4])) if w * h > hyp['person_size_small_medium_th'] and  w * h <= hyp['car_size_small_medium_th']]
                if w * h > hyp['car_size_small_medium_th']:
                    stats_all_large.append((correct.cpu(), pred[:, 4].cpu(), pred[:, 5].cpu(), tcls))

                #     sensor type
                if dataloader.dataset.use_csv_meta_data_file:
                    try:

                        weather_condition = (dataloader.dataset.df_metadata[dataloader.dataset.df_metadata['tir_frame_image_file_name'] == str(path).split('/')[-1]]['weather_condition'].item())
                        if isinstance(weather_condition, str):
                            weather_condition = weather_condition.lower()
                            exec([x for x in weather_condition_vars if str(weather_condition) in x][0] + '.append((correct.cpu(), pred[:, 4].cpu(), pred[:, 5].cpu(), tcls))')
                    except Exception as e:
                        print(f'{weather_condition} fname WARNING: Ignoring corrupted image and/or label {weather_condition}: {e}')

                    time_in_day = dataloader.dataset.df_metadata[dataloader.dataset.df_metadata['tir_frame_image_file_name'] == str(path).split('/')[-1]]['part_in_day'].item().lower()
                    # eval([x for x in time_vars if str(time_in_day) in x][0]).append((correct.cpu(), pred[:, 4].cpu(), pred[:, 5].cpu(), tcls))
                    exec([x for x in time_vars if str(time_in_day) in x][0] + '.append((correct.cpu(), pred[:, 4].cpu(), pred[:, 5].cpu(), tcls))')

                    sensor_type = dataloader.dataset.df_metadata[dataloader.dataset.df_metadata['tir_frame_image_file_name'] == str(path).split('/')[-1]]['sensor_type'].item()
                    # obj_range_m = torch.tensor(
                    #     [(object_size_to_range(obj_height_pixels=h.cpu(), focal=sensor_type, class_id=class_id.cpu().numpy().item()))
                    #      for class_id, (x, y, w, h) in zip(pred[:, 5], xyxy2xywh(pred[:, :4]))])
                    gt_range = [(object_size_to_range(obj_height_pixels=h, focal=sensor_type, class_id=class_id.numpy().item())) for
                                class_id, (x, y, w, h) in zip(labels[:, 0].cpu(), labels[:, 1:5].cpu())]

                    # coupling the range cell between any overlapped IOU >TH between pred bbox and GT bbox
                    obj_range_m = list()
                    i = 0
                    for class_id_pred, (x1_p, y1_p, x2_p, y2_p) in zip(pred[:, 5].cpu(), pred[:, :4].cpu()):

                        range_candidate = torch.tensor(
                            object_size_to_range(obj_height_pixels=xyxy2xywh(pred[:, :4])[0][-1].cpu(),
                                                 focal=sensor_type,
                                                 class_id=class_id_pred.cpu().numpy().item()))
                        # Find any IOU overlapped between GT and prediction
                        for class_id_gt, (x1_gt, y1_gt, x2_gt, y2_gt), (xc,yc,w,h) in zip(labels[:, 0].cpu(),
                                                                             xywh2xyxy(labels[:, 1:5].cpu()), labels[:, 1:5].cpu()):
                            ious = box_iou(torch.tensor((x1_gt, y1_gt, x2_gt, y2_gt)).unsqueeze(axis=0), torch.tensor((x1_p, y1_p, x2_p, y2_p)).unsqueeze(axis=0))
                            i += 1
                            if ious > iouv.cpu()[0]: #
                                range_candidate = torch.tensor(
                                    object_size_to_range(obj_height_pixels=h.cpu(),
                                                         focal=sensor_type,
                                                         class_id=class_id_pred.cpu().numpy().item()))
                                break  # the aligned GT/Pred was found no need to iterate more, this is the atmost candidate
                        obj_range_m.append(range_candidate)
                    # else: #ranges = func(sqrt(height*width))
                    #     obj_range_m = torch.tensor([(object_size_to_range(obj_height_pixels=(np.sqrt(h.cpu()*w.cpu())), focal=sensor_type)) for ix, (x, y, w, h) in enumerate(xyxy2xywh(pred[:, :4]))])
                    #     gt_range = [(object_size_to_range(obj_height_pixels=(np.sqrt(h*w)), focal=sensor_type)) for ix, (x, y, w, h) in enumerate(labels[:,1:5].cpu())]
                    #
                    if 1:
                        gt_range = [_range // 100 for _range in gt_range] #gt_range = [_range // 100 for _range in gt_range]
                    else:
                        gt_range = [_range//bin_size_per_sensor[sensor_type] for _range in gt_range]

                    for gt_lbl, rng_ in zip(labels[:,0], gt_range):
                        if rng_ < n_bins_of100m :
                            gt_per_range_bins[sensor_type][int(rng_.item())].append(int(gt_lbl.item()))  # add to each range bin GT the GT counts
                            gt_path_per_range_bins[sensor_type][int(rng_.item())].append(str(path))
                        # (obj_range_m.cpu().reshape(-1))
                    exec([x for x in sensor_type_vars if str(sensor_type) in x][0] + '.append((correct.cpu(), pred[:, 4].cpu(), pred[:, 5].cpu(), tcls))')
                    exec([x+'_with_range' for x in sensor_type_vars if str(sensor_type) in x][0] + '.append((correct.cpu(), pred[:, 4].cpu(), pred[:, 5].cpu(), tcls, obj_range_m, pred[:, 5].shape[0]*[str(path)]))') # path is replicated to match each prediction TP/FP in the image



                    # for daytime in pd.unique(dataloader.dataset.df_metadata['part_in_day']):
                    #     exec('stats_all_part_in_day{}'.format(daytime.lower()) + '=[]')  # 'stats_all_day'



        # Plot images  aa = np.repeat(img[0,:,:,:].cpu().permute(1,2,0).numpy(), 3, axis=2).astype('float32') cv2.imwrite('test/exp40/test_batch88_labels__.jpg', aa*255)
        if plots and batch_i > 10 or 1:
            # conf_thresh_plot = 0.1 # the plot threshold the connfidence
            f = save_dir / f'test_batch{batch_i}_labels.jpg'  # labels
            Thread(target=plot_images, args=(img, targets, paths, f, names), daemon=True).start()
            f = save_dir / f'test_batch{batch_i}_pred.jpg'  # predictions
            Thread(target=plot_images, args=(img, output_to_target(out), paths, f, names), daemon=True).start()

    # Compute statistics
    stats = [np.concatenate(x, 0) for x in zip(*stats)]  # to numpy
    if trace and embed_analyse:
        embeddings = obj_embed_viz.visualize_object_embeddings(features_acm,
                                                               labels_acm,
                                                               path=save_dir,
                                                               tag=opt.conf_thres)

    if not training or 1:
        stats_person_medium = [np.concatenate(x, 0) for x in zip(*stats_person_medium)]  # to numpy
        stats_all_large = [np.concatenate(x, 0) for x in zip(*stats_all_large)]  # to numpy
        if dataloader.dataset.use_csv_meta_data_file:
            for time_var in time_vars:
                exec('{}=[np.concatenate(x, 0) for x in zip(*{})]'.format(time_var, time_var))  # 'stats_all_50'

            for sensor_type in sensor_type_vars:
                exec('{}=[np.concatenate(x, 0) for x in zip(*{})]'.format(sensor_type, sensor_type))  # 'stats_all_50'
                exec('{}_with_range=[np.concatenate(x, 0) for x in zip(*{}_with_range)]'.format(sensor_type, sensor_type))  # 'stats_all_50'

            for weather_condition in weather_condition_vars:
                exec('{}=[np.concatenate(x, 0) for x in zip(*{})]'.format(weather_condition, weather_condition))  # 'stats_all_50'

    if len(stats) and stats[0].any(): # P, R @  # max F1 index if any correct prediction
        p, r, ap, f1, ap_class = ap_per_class(*stats, plot=plots, v5_metric=v5_metric, save_dir=save_dir, names=names) #based on correct @ IOU=0.5 of pred box with target
        ap50, ap = ap[:, 0], ap.mean(1)  # AP@0.5, AP@0.5:0.95
        mp, mr, map50, map = p.mean(), r.mean(), ap50.mean(), ap.mean()
        nt = np.bincount(stats[3].astype(np.int64), minlength=nc)  # number of targets per class


        # if bool(stats_person_medium):
        #     p_med, r_med, ap_med, f1_med, ap_class_med = ap_per_class(*stats_person_medium, plot=plots, v5_metric=v5_metric, save_dir=save_dir, names=names, tag='small_objects')
        #     ap50_med, ap_med = ap_med[:, 0], ap_med.mean(1)  # AP@0.5, AP@0.5:0.95
        #     mp_med, mr_med, map50_med, map_med = p_med.mean(), r_med.mean(), ap50_med.mean(), ap_med.mean()
        #     nt_med = np.bincount(stats_person_medium[3].astype(np.int64), minlength=nc)  # number of targets per class
        #
        # if bool(stats_all_large):
        #     p_large, r_large, ap_large, f1_large, ap_class_large = ap_per_class(*stats_all_large, plot=plots, v5_metric=v5_metric, save_dir=save_dir, names=names, tag='large_objects')
        #     ap50_large, ap_large = ap_large[:, 0], ap_large.mean(1)  # AP@0.5, AP@0.5:0.95
        #     mp_large, mr_large, map50_large, map_large = p_large.mean(), r_large.mean(), ap50_large.mean(), ap_large.mean()
        #     nt_large = np.bincount(stats_all_large[3].astype(np.int64), minlength=nc)  # number of targets per class

        if dataloader.dataset.use_csv_meta_data_file:
            for time_var in time_vars:
                if bool(eval(time_var)):
                    # TODO in the names of classes in labels add the support from nt_, also pass the support of overall and the title name like in the tag but only day /night to the title inside plot_pr_curve()
                    exec("nt_{} = np.bincount({}[3].astype(np.int64), minlength={})".format(time_var, time_var, nc))
                    exec("p_{}, r_{}, ap_{}, f1_{}, ap_class_{} = ap_per_class(*{}, plot={}, v5_metric={}, save_dir={}, names={}, tag={}, class_support=nt_{})".format(time_var,
                                                                time_var, time_var, time_var, time_var, time_var, plots, v5_metric, 'str(save_dir)', names, 'str(time_var)', time_var))

                    exec("ap50_{}, ap_{} = ap_{}[:, 0], ap_{}.mean(1)".format(time_var, time_var, time_var, time_var))
                    exec("mp_{}, mr_{}, map50_{}, map_{} = p_{}.mean(), r_{}.mean(), ap50_{}.mean(), ap_{}.mean()".format(time_var, time_var, time_var, time_var, time_var, time_var, time_var, time_var))

                for weather_condition in weather_condition_vars:
                    exec("nt_{} = np.bincount({}[3].astype(np.int64), minlength={})".format(weather_condition, weather_condition, nc))
                    exec("p_{}, r_{}, ap_{}, f1_{}, ap_class_{} = ap_per_class(*{}, plot={}, v5_metric={}, save_dir={}, names={}, tag={}, class_support=nt_{})".format(weather_condition,
                                    weather_condition, weather_condition, weather_condition, weather_condition, weather_condition, plots, v5_metric, 'str(save_dir)', names, 'str(weather_condition)', weather_condition))

                    exec("ap50_{}, ap_{} = ap_{}[:, 0], ap_{}.mean(1)".format(weather_condition, weather_condition, weather_condition, weather_condition))
                    exec("mp_{}, mr_{}, map50_{}, map_{} = p_{}.mean(), r_{}.mean(), ap50_{}.mean(), ap_{}.mean()".format(weather_condition, weather_condition,
                                                            weather_condition, weather_condition, weather_condition, weather_condition, weather_condition, weather_condition))

            if 0 : #debug
                ranges100_pred = np.array([])
                print('gt class dist per range cell of 100s and sum of GTs ')
                print([(100*(ix+1), np.unique(x, return_counts=True), np.size(x)) for ix, x in enumerate(gt_per_range_bins[210])])
                exec('ranges100_pred={}_with_range[4]//100'.format('stats_all_sensor_type_210'))
                exec('cls_pred={}_with_range[2]'.format('stats_all_sensor_type_210'))
                exec('predicted={}_with_range[0]'.format('stats_all_sensor_type_210'))

                print('True predictions on 210', eval('ranges100_pred.shape'))
                np.array([np.size(x) for ix, x in enumerate(gt_per_range_bins[210])]).T # predictions may be TRue or False
                print('detections preds per range cell')
                np.unique(ranges100_pred, return_counts=True)[1].T

                [np.bincount(x, minlength=2) for ix, x in enumerate(gt_per_range_bins[210])]
                range_bin_ = 1
                np.unique(cls_pred[ranges100_pred == range_bin_], return_counts=True)
                # count of good bad predictions per class
                stats_all_sensor_type_210_with_range[2][ranges100_pred == range_bin_]

                np.unique(stats_all_sensor_type_210_with_range[0][:, 0][ranges100_pred == range_bin_],
                          return_counts=True)

            for sensor_type in sensor_type_vars:
                if bool(eval(sensor_type)):
                    exec("nt_{} = np.bincount({}[3].astype(np.int64), minlength={})".format(sensor_type, sensor_type, nc))

                    exec("p_{}, r_{}, ap_{}, f1_{}, ap_class_{} = ap_per_class(*{}, plot={}, v5_metric={}, save_dir={}, names={}, tag={}, class_support=nt_{})".format(sensor_type,
                                                                sensor_type, sensor_type, sensor_type, sensor_type, sensor_type, plots, v5_metric, 'str(save_dir)', names, 'str(sensor_type)',sensor_type))

                    exec("ap50_{}, ap_{} = ap_{}[:, 0], ap_{}.mean(1)".format(sensor_type, sensor_type, sensor_type, sensor_type))
                    exec("mp_{}, mr_{}, map50_{}, map_{} = p_{}.mean(), r_{}.mean(), ap50_{}.mean(), ap_{}.mean()".format(sensor_type, sensor_type, sensor_type, sensor_type, sensor_type, sensor_type, sensor_type, sensor_type))

                    sensor_focal = int(sensor_type.split('_')[-1])
                    if  1 :#sensor_focal > 100: # ML
                        exec('ranges={}_with_range[4]//100'.format(sensor_type))

                        for rng_100 in range(0,n_bins_of100m):
                            nt_stat_list_per_range = np.array([0, 0])
                            r_stat_list_per_range = np.array([0, 0])
                            p_stat_list_per_range = np.array([0, 0])
                            map50_per_range = np.array(0)

                            # ind = np.array([])
                            # exec('ind = np.where(ranges == rng_100)[0]')
                            ind = eval('np.where(ranges == rng_100)[0]')
                            stat_list_per_range = list()
                            if ind.size>0: # if there were detections at that range bin
                                for ele in range(3):# taking the relevant preds related to the distance bin, since P/R/AP are computed globally vs. all GT/targets then it is compared to all targets
                                    stat_list_per_range.append(eval(sensor_type)[ele][ind])
                                # GT at thta bin range
                                if not bool(gt_per_range_bins[sensor_focal][rng_100]): # predictions but no GT => FP=>low Prcesion
                                    map50_per_range = np.array(0)
                                    nt_stat_list_per_range = np.array([0,0])
                                    r_stat_list_per_range = np.array([0,0])
                                    p_stat_list_per_range = np.array([0,0])
                                else:
                                    stat_list_per_range.append(np.array([x for x in gt_per_range_bins[sensor_focal][rng_100]])) # add all targets/labels
                                    nt_stat_list_per_range = np.bincount(stat_list_per_range[3].astype(np.int64), minlength=nc) # GT count per bin range of classes

                                    p_stat_list_per_range, r_stat_list_per_range, ap_stat_list_per_range, f1_stat_list_per_range, \
                                    ap_class_stat_list_per_range = ap_per_class(*stat_list_per_range, plot=plots, v5_metric=v5_metric,
                                                                    save_dir='', names=names, tag='',
                                                                                class_support=nt_stat_list_per_range)

                                    ap50_per_range, ap_per_range = ap_stat_list_per_range[:, 0], ap_stat_list_per_range.mean(1)  # AP@0.5, AP@0.5:0.95
                                    mp_per_range, mr_per_range, map50_per_range, map_per_range = p_stat_list_per_range.mean(), r_stat_list_per_range.mean(), ap50_per_range.mean(), ap_per_range.mean()
                            else:# no prediction at this range
                                r_stat_list_per_range = np.array([0, 0])
                                p_stat_list_per_range = np.array([0, 0])
                                fn = len(gt_per_range_bins[sensor_focal][rng_100])
                                recall = 0  # no TP 0/TP+FN
                                precision = 0
                                map50_per_range = np.array(0)
                                if not bool(gt_per_range_bins[sensor_focal][rng_100]):# there are no GT no pred
                                    nt_stat_list_per_range = np.array([0,0])  # actual GT
                                else:
                                    nt_stat_list_per_range = np.array(gt_per_range_bins[sensor_focal][rng_100]).sum()
                            # there are GT but no pred
                            # print(map50_per_range)

                            range_bins_map[sensor_focal][rng_100] = map50_per_range.item()
                            range_bins_precision_all_classes[sensor_focal][rng_100] = nt_stat_list_per_range.astype('bool').astype('int')*p_stat_list_per_range # broadcast the count of each calss in case one of the classes are missing
                            range_bins_recall_all_classes[sensor_focal][rng_100] = nt_stat_list_per_range.astype('bool').astype('int')*r_stat_list_per_range
                            range_bins_support_gt[sensor_focal][rng_100] = nt_stat_list_per_range.sum().item()

            # bug_diff = np.array(range_bins_support_gt[210]) - np.array(
            #     [np.size(x) for ix, x in enumerate(gt_per_range_bins[210])]).T
            # In case no Preds than there is no GT count in the if condition anyway if pred=0=>TP=0 than P=R=0
            range_bar_plot(n_bins_of100m, range_bins_map, save_dir, range_bins_support=range_bins_support_gt)
            range_p_r_bar_plot(n_bins_of100m, range_bins_precision_all_classes, range_bins_recall_all_classes,
                               save_dir, range_bins_support=range_bins_support_gt, names=names, conf=opt.conf_thres)
            # for time_var in time_vars:
            #     for sensor_type in sensor_type_vars:

        # nt = np.bincount(stats[3].astype(np.int64), minlength=nc)  # number of targets per class
    else:
        nt = torch.zeros(1)
        nt_med = torch.zeros(1)
        nt_large = torch.zeros(1)

    # Print results
    pf = '%20s' + '%12i' * 2 + '%12.3g' * 4  # print format
    print(pf % ('all', seen, nt.sum(), mp, mr, map50, map))
    # if not training or 1:
        # if bool(stats_person_medium):
        #     try:
        #         print(pf % ('all_medium', seen, nt_med.sum(), mp_med, mr_med, map50_med, map_med))
        #     except Exception as e:
        #         print(e)
        #
        # if bool(stats_all_large):
        #     try:
        #         print(pf % ('all_large', seen, nt_large.sum(), mp_large, mr_large, map50_large, map_large))
        #     except Exception as e:
        #         print(e)

    file_path = os.path.join(save_dir, 'class_stats.txt') #'Class', 'Images', 'Labels', 'P', 'R', 'mAP@.5', 'mAP@.5:.95'
    append_to_txt(file_path, 'all', seen, nt.sum(), mp, mr, map50, map)

    # Print results per class
    if 1 or (verbose or (nc < 50 and not training)) and nc > 1 and len(stats):
        for i, c in enumerate(ap_class):
            print(pf % (names[c], seen, nt[c], p[i], r[i], ap50[i], ap[i]))
            append_to_txt(file_path, names[c], seen, nt[c], p[i], r[i], ap50[i], ap[i])
        # try:
        #     if bool(stats_person_medium):
        #         for i, c in enumerate(ap_class_med):
        #             print(pf % (names[c]+ '_med', seen, nt_med[c], p_med[i], r_med[i], ap50_med[i], ap_med[i]))
        #             append_to_txt(file_path, names[c] + '_med', seen, nt_med[c], p_med[i], r_med[i], ap50_med[i], ap_med[i])
        # except Exception as e:
        #     print(e)
        #
        # try:
        #     if bool(stats_all_large):
        #         for i, c in enumerate(ap_class_large):
        #             print(pf % (names[c]+ '_large', seen, nt_large[c], p_large[i], r_large[i], ap50_large[i], ap_large[i]))
        #             append_to_txt(file_path, names[c] + '_large', seen, nt_large[c], p_large[i], r_large[i], ap50_large[i], ap_large[i])
        # except Exception as e:
        #     print(e)

    # Print speeds
    t = tuple(x / seen * 1E3 for x in (t0, t1, t0 + t1)) + (imgsz, imgsz, batch_size)  # tuple
    if not training:
        print('Speed: %.1f/%.1f/%.1f ms inference/NMS/total per %gx%g image at batch-size %g' % t)

    # Plots
    if plots:
        confusion_matrix.plot(save_dir=save_dir, names=list(names.values()))
        if wandb_logger and wandb_logger.wandb:
            val_batches = [wandb_logger.wandb.Image(str(f), caption=f.name) for f in sorted(save_dir.glob('test*.jpg'))]
            wandb_logger.log({"Validation": val_batches})
    if wandb_images:
        wandb_logger.log({"Bounding Box Debugger/Images": wandb_images})

    # Save JSON
    if save_json and len(jdict): # @@ HK TODO:
        w = Path(weights[0] if isinstance(weights, list) else weights).stem if weights is not None else ''  # weights
        anno_json = './coco/annotations/instances_val2017.json'  # annotations json
        pred_json = str(save_dir / f"{w}_predictions.json")  # predictions json
        pred_df_file = str(save_dir / f"{w}_predictions.csv")  # predictions json
        print('\nEvaluating pycocotools mAP... saving %s...' % pred_json)
        with open(pred_json, 'w') as f:
            json.dump(jdict, f)

        try:  # https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocoEvalDemo.ipynb
            from pycocotools.coco import COCO
            from pycocotools.cocoeval import COCOeval

            anno = COCO(anno_json)  # init annotations api
            pred = anno.loadRes(pred_json)  # init predictions api
            eval1 = COCOeval(anno, pred, 'bbox')
            if is_coco:
                eval1.params.imgIds = [int(Path(x).stem) for x in dataloader.dataset.img_files]  # image IDs to evaluate
            eval1.evaluate()
            eval1.accumulate()
            eval1.summarize()
            map, map50 = eval1.stats[:2]  # update results (mAP@0.5:0.95, mAP@0.5)
        except Exception as e:
            print(f'pycocotools unable to run: {e}')

    # Return results
    model.float()  # for training
    if not training:
        s = f"\n{len(list(save_dir.glob('labels/*.txt')))} labels saved to {save_dir / 'labels'}" if save_txt else ''
        print(f"Results saved to {save_dir}{s}")
    maps = np.zeros(nc) + map
    for i, c in enumerate(ap_class):
        maps[c] = ap[i]

    if save_json:
        predictions_df.to_csv(pred_df_file, index=False)

    return (mp, mr, map50, map, *(loss.cpu() / len(dataloader)).tolist()), maps, t


def sensor_type_breakdown_kpi(gt_per_range_bins, n_bins_of100m, names, nc, plots, range_bins_map, range_bins_support,
                              save_dir, bin_size_per_sensor, v5_metric, **kwargs):

    # print(sensor_type_50)
    # print(sensor_type_210)

    # print(f_sensor_type_50)
    # print(f_sensor_type_210)
    # exec('{}=sensor_type_50'.format(f_sensor_type_50))
    # exec('{}=sensor_type_210'.format(f_sensor_type_210))
    # print([key for key in vars().keys() if 'stats_all_sensor_type' in key and not '_with_range' in key])
    sensor_type_vars = [key for key in vars().keys() if 'stats_all_sensor_type' in key and not '_with_range' in key]

    # exec('{}=sensor_type_50'.format(f_sensor_type_50))
    for sensor_type in sensor_type_vars: #sensor_type_vars:
        if bool(sensor_type):
            exec("nt_{} = np.bincount({}[3].astype(np.int64), minlength={})".format(sensor_type, sensor_type, nc))

            exec(
                "p_{}, r_{}, ap_{}, f1_{}, ap_class_{} = ap_per_class(*{}, plot={}, v5_metric={}, save_dir={}, names={}, tag={}, class_support=nt_{})".format(
                    sensor_type,
                    sensor_type, sensor_type, sensor_type, sensor_type, sensor_type, plots, v5_metric, 'str(save_dir)',
                    names, 'str(sensor_type)', sensor_type))

            exec("ap50_{}, ap_{} = ap_{}[:, 0], ap_{}.mean(1)".format(sensor_type, sensor_type, sensor_type,
                                                                      sensor_type))
            exec("mp_{}, mr_{}, map50_{}, map_{} = p_{}.mean(), r_{}.mean(), ap50_{}.mean(), ap_{}.mean()".format(
                sensor_type, sensor_type, sensor_type, sensor_type, sensor_type, sensor_type, sensor_type, sensor_type))

            sensor_focal = int(sensor_type.split('_')[-1])
            if 1:  # sensor_focal > 100: # ML
                exec('ranges={}_with_range[3]//{}'.format(sensor_type, bin_size_per_sensor[sensor_focal]))
                for rng_100 in range(0, n_bins_of100m):
                    # ind = np.array([])
                    # exec('ind = np.where(ranges == rng_100)[0]')
                    ind = eval('np.where(ranges == rng_100)[0]')
                    stat_list_per_range = list()
                    if ind.any():  # if there were detections at that range bin
                        for ele in range(
                                3):  # taking the relevant preds related to the distance bin, since P/R/AP are computed globally vs. all GT/targets then it is compared to all targets
                            stat_list_per_range.append(eval(sensor_type)[ele][ind])
                        # GT at thta bin range
                        if not bool(
                                gt_per_range_bins[sensor_focal][rng_100]):  # predictions but no GT => FP=>low Prcesion
                            map50_per_range = np.array(0)
                            nt_stat_list_per_range = np.array(0)
                        else:
                            stat_list_per_range.append(np.array(
                                [x for x in gt_per_range_bins[sensor_focal][rng_100]]))  # add all targets/labels
                            nt_stat_list_per_range = np.bincount(stat_list_per_range[3].astype(np.int64), minlength=nc)

                            p_stat_list_per_range, r_stat_list_per_range, ap_stat_list_per_range, f1_stat_list_per_range, \
                                ap_class_stat_list_per_range = ap_per_class(*stat_list_per_range, plot=plots,
                                                                            v5_metric=v5_metric,
                                                                            save_dir='', names=names, tag='',
                                                                            class_support=nt_stat_list_per_range)

                            ap50_per_range, ap_per_range = ap_stat_list_per_range[:, 0], ap_stat_list_per_range.mean(
                                1)  # AP@0.5, AP@0.5:0.95
                            mp_per_range, mr_per_range, map50_per_range, map_per_range = p_stat_list_per_range.mean(), r_stat_list_per_range.mean(), ap50_per_range.mean(), ap_per_range.mean()
                    else:  # no prediction at this range
                        if not bool(gt_per_range_bins[sensor_focal][rng_100]):  # there are GT but no pred
                            nt_stat_list_per_range = np.array(0)
                            fn = len(gt_per_range_bins[sensor_focal][rng_100])
                            recall = 0  # no TP 0/TP+FN
                            precision = 0
                            map50_per_range = np.array(0)
                    print(map50_per_range)
                    range_bins_map[sensor_focal][rng_100] = map50_per_range.item()
                    range_bins_support[sensor_focal][rng_100] = nt_stat_list_per_range.sum().item()
    range_bar_plot(n_bins=17, range_bins=range_bins_map, save_dir=save_dir, range_bins_support=range_bins_support)
    # for time_var in time_vars:


if __name__ == '__main__':
    parser = argparse.ArgumentParser(prog='test.py')
    parser.add_argument('--weights', nargs='+', type=str, default='yolov7.pt', help='model.pt path(s)')
    parser.add_argument('--data', type=str, default='data/coco.yaml', help='*.data path')
    parser.add_argument('--batch-size', type=int, default=32, help='size of each image batch')
    parser.add_argument('--img-size', type=int, default=640, help='inference size (pixels)')
    parser.add_argument('--conf-thres', type=float, default=0.001, help='object confidence threshold')
    parser.add_argument('--iou-thres', type=float, default=0.6, help='IOU threshold for NMS')
    parser.add_argument('--task', default='val', help='train, val, test, speed or study')
    parser.add_argument('--device', default='', help='cuda device, i.e. 0 or 0,1,2,3 or cpu')
    parser.add_argument('--single-cls', action='store_true', help='treat as single-class dataset')
    parser.add_argument('--augment', action='store_true', help='augmented inference')
    parser.add_argument('--verbose', action='store_true', help='report mAP by class')
    parser.add_argument('--save-txt', action='store_true', help='save results to *.txt')
    parser.add_argument('--save-hybrid', action='store_true', help='save label+prediction hybrid results to *.txt')
    parser.add_argument('--save-conf', action='store_true', help='save confidences in --save-txt labels')
    parser.add_argument('--save-json', action='store_true', help='save a cocoapi-compatible JSON results file')
    parser.add_argument('--project', default='runs/test', help='save to project/name')
    parser.add_argument('--name', default='exp', help='save to project/name')
    parser.add_argument('--exist-ok', action='store_true', help='existing project/name ok, do not increment')
    parser.add_argument('--no-trace', action='store_true', help='don`t trace model')
    parser.add_argument('--v5-metric', action='store_true', help='assume maximum recall as 1.0 in AP calculation')
    parser.add_argument('--norm-type', type=str, default='standardization',
                        choices=['standardization', 'single_image_0_to_1', 'single_image_mean_std','single_image_percentile_0_255',
                                 'single_image_percentile_0_1', 'remove+global_outlier_0_1'],
                        help='Normalization approach')

    parser.add_argument('--no-tir-signal', action='store_true', help='')

    parser.add_argument('--tir-channel-expansion', action='store_true', help='drc_per_ch_percentile')

    parser.add_argument('--input-channels', type=int, default=3, help='')

    parser.add_argument('--save-path', default='', help='save to project/name')

    parser.add_argument('--csv-metadata-path', default='', help='save to project/name')

    parser.add_argument('--embed-analyse', action='store_true', help='')


    opt = parser.parse_args()

    if opt.tir_channel_expansion: # operates over 3 channels
        opt.input_channels = 3

    if opt.tir_channel_expansion and opt.norm_type != 'single_image_percentile_0_1': # operates over 3 channels
        print('Not a good combination')

    opt.save_json |= opt.data.endswith('coco.yaml')
    opt.data = check_file(opt.data)  # check file
    print(opt)
    #check_requirements()
    hyp = dict()

    if opt.task in ('train', 'val', 'test'):  # run normally
        test(opt.data,
             opt.weights,
             opt.batch_size,
             opt.img_size,
             opt.conf_thres,
             opt.iou_thres,
             opt.save_json,
             opt.single_cls,
             opt.augment,
             opt.verbose,
             save_txt=opt.save_txt | opt.save_hybrid,
             save_hybrid=opt.save_hybrid,
             save_conf=opt.save_conf,
             trace=not opt.no_trace,
             v5_metric=opt.v5_metric,
             hyp=hyp,
             embed_analyse=opt.embed_analyse)

    elif opt.task == 'speed':  # speed benchmarks
        for w in opt.weights:
            test(opt.data, w, opt.batch_size, opt.img_size, 0.25, 0.45, save_json=False, plots=False, v5_metric=opt.v5_metric)

    elif opt.task == 'study':  # run over a range of settings and save/plot
        # python test.py --task study --data coco.yaml --iou 0.65 --weights yolov7.pt
        x = list(range(256, 1536 + 128, 128))  # x axis (image sizes)
        for w in opt.weights:
            f = f'study_{Path(opt.data).stem}_{Path(w).stem}.txt'  # filename to save to
            y = []  # y axis
            for i in x:  # img-size
                print(f'\nRunning {f} point {i}...')
                r, _, t = test(opt.data, w, opt.batch_size, i, opt.conf_thres, opt.iou_thres, opt.save_json,
                               plots=False, v5_metric=opt.v5_metric)
                y.append(r + t)  # results and times
            np.savetxt(f, y, fmt='%10.4g')  # save
        os.system('zip -r study.zip study_*.txt')
        plot_study_txt(x=x)  # plot
"""

--weights ./yolov7/yolov7.pt --device 0 --batch-size 16 --data data/coco_2_tir.yaml --img-size 640 --conf 0.6 --verbose --save-txt --save-hybrid --norm-type single_image_percentile_0_1
test based on RGB coco model
--weights ./yolov7/yolov7.pt --device 0 --batch-size 64 --data data/coco_2_tir.yaml --img-size 640 --conf 0.25 --verbose --save-txt --norm-type single_image_percentile_0_1 --project test --task train

--weights ./yolov7/yolov7.pt --device 0 --batch-size 64 --data data/tir_od.yaml --img-size 640 --conf 0.25 --verbose --save-txt --norm-type single_image_percentile_0_1 --project test --task val
# Using pretrained model
--weights /mnt/Data/hanoch/runs/train/yolov7434/weights/epoch_099.pt --device 0 --batch-size 4 --data data/tir_od_test_set.yaml --img-size 640 --conf 0.25 --verbose --norm-type single_image_percentile_0_1 --project test --task test
#vbased on 7555 mAP=82.3
--weights /mnt/Data/hanoch/runs/train/yolov7563/weights/best.pt --device 0 --batch-size 16 --data data/tir_od_test_set.yaml --img-size 640 --conf 0.02 --verbose --norm-type single_image_percentile_0_1 --input-channels 1 --project test --task test --iou-thres 0.4

/home/hanoch/projects/tir_od/runs/train/yolov7563/weights


--weights /mnt/Data/hanoch/runs/train/yolov7575/weights/best.pt --device 0 --batch-size 16 --data data/tir_od_test_set.yaml --img-size 640 --conf 0.001 --verbose --norm-type single_image_percentile_0_1 --input-channels 1 --project test --task test --iou-thres 0.6
--weights /home/hanoch/projects/tir_od/runs/gpu02/yolov74/weights --device 0 --batch-size 16 --data data/tir_od_test_set.yaml --img-size 640 --conf 0.001 --verbose --norm-type single_image_percentile_0_1 --input-channels 1 --project test --task test --iou-thres 0.6

3class
/mnt/Data/hanoch/runs/train/yolov71058
--weights /mnt/Data/hanoch/runs/train/yolov71058/weights/best.pt --device 0 --batch-size 16 --data data/tir_od_center_roi_aug_list_train_cls.yaml --img-size 640 --conf 0.02 --verbose --norm-type single_image_percentile_0_1 --input-channels 1 --project test --task val --iou-thres 0.6

tir_tiff_w_center_roi_validation_set_train_cls_usa.txt

# per day/nigh SY/ML
--weights /mnt/Data/hanoch/runs/train/yolov7999/weights/best.pt --device 0 --batch-size 16 --data data/tir_od_test_set.yaml --img-size 640 --conf 0.001 --verbose --norm-type single_image_percentile_0_1 --input-channels 1 --project test --task test --iou-thres 0.6 --csv-metadata-path tir_od/tir_center_merged_seq_tiff_last_original_png.xlsx
P/R
--weights /mnt/Data/hanoch/runs/train/yolov7999/weights/best.pt --device 0 --batch-size 16 --data data/tir_od_test_set.yaml --img-size 640 --verbose --norm-type single_image_percentile_0_1 --input-channels 1 --project test --task test --iou-thres 0.6 --csv-metadata-path tir_od/tir_center_merged_seq_tiff_last_original_png.xlsx --conf 0.65
mAP:
--weights /mnt/Data/hanoch/runs/train/yolov7999/weights/best.pt --device 0 --batch-size 16 --data data/tir_od_test_set.yaml --img-size 640 --verbose --norm-type single_image_percentile_0_1 --input-channels 1 --project test --task test --iou-thres 0.6 --csv-metadata-path tir_od/tir_center_merged_seq_tiff_last_original_png.xlsx --conf 0.01


Fixed wether csv  P/R
--weights /mnt/Data/hanoch/runs/train/yolov7999/weights/best.pt --device 0 --batch-size 16 --data data/tir_od_test_set.yaml --img-size 640 --verbose --norm-type single_image_percentile_0_1 --input-channels 1 --project test --task test --csv-metadata-path tir_od/tir_tiff_seq_png_3_class_fixed_whether.xlsx --iou-thres 0.6  --conf 0.65

FOG
--weights /mnt/Data/hanoch/runs/train/yolov7999/weights/best.pt --device 0 --batch-size 16 --data data/tir_od_fog_set.yaml --img-size 640 --verbose --norm-type single_image_percentile_0_1 --input-channels 1 --project test --task test --csv-metadata-path tir_od/tir_tiff_seq_png_3_class_fixed_whether.xlsx --conf 0.65 --iou-thres 0.6

Locomotive

--weights /mnt/Data/hanoch/runs/train/yolov71107/weights/best.pt --device 0 --batch-size 16 --data data/tir_od_test_set_3_class_train.yaml --img-size 640 --verbose --norm-type single_image_percentile_0_1 --input-channels 1 --project test --task test --iou-thres 0.6  --conf 0.1 --embed-analyse
-------  Error analysis  ------------
1st run with conf_th=0.0001 then observe the desired threshold, re-run with the desired threshold abd observe images with bboxes given the deired threshold 
"""