Increment train, test, detect runs/ (#1322)

* Increment train, test, detect runs/

* Update ci-testing.yml

* inference/images to data/images

* move images

* runs/exp to runs/train/exp

* update 'results saved to %s' str

.github/workflows/ci-testing.yml

@@ -66,10 +66,10 @@ jobs:
           python train.py --img 256 --batch 8 --weights weights/${{ matrix.model }}.pt --cfg models/${{ matrix.model }}.yaml --epochs 1 --device $di
           # detect
           python detect.py --weights weights/${{ matrix.model }}.pt --device $di
-          python detect.py --weights runs/exp0/weights/last.pt --device $di
+          python detect.py --weights runs/train/exp0/weights/last.pt --device $di
           # test
           python test.py --img 256 --batch 8 --weights weights/${{ matrix.model }}.pt --device $di
-          python test.py --img 256 --batch 8 --weights runs/exp0/weights/last.pt --device $di
+          python test.py --img 256 --batch 8 --weights runs/train/exp0/weights/last.pt --device $di
 
           python models/yolo.py --cfg models/${{ matrix.model }}.yaml  # inspect
           python models/export.py --img 256 --batch 1 --weights weights/${{ matrix.model }}.pt  # export

.gitignore

@@ -26,8 +26,8 @@
 storage.googleapis.com
 runs/*
 data/*
-!data/samples/zidane.jpg
+!data/images/zidane.jpg
-!data/samples/bus.jpg
+!data/images/bus.jpg
 !data/coco.names
 !data/coco_paper.names
 !data/coco.data

Dockerfile

@@ -46,7 +46,7 @@ COPY . /usr/src/app
 # sudo docker commit 092b16b25c5b usr/resume && sudo docker run -it --gpus all --ipc=host -v "$(pwd)"/coco:/usr/src/coco --entrypoint=sh usr/resume
 
 # Send weights to GCP
-# python -c "from utils.general import *; strip_optimizer('runs/exp0_*/weights/best.pt', 'tmp.pt')" && gsutil cp tmp.pt gs://*.pt
+# python -c "from utils.general import *; strip_optimizer('runs/train/exp0_*/weights/best.pt', 'tmp.pt')" && gsutil cp tmp.pt gs://*.pt
 
 # Clean up
 # docker system prune -a --volumes

README.md

@@ -70,7 +70,7 @@ YOLOv5 may be run in any of the following up-to-date verified environments (with
 
 ## Inference
 
-detect.py runs inference on a variety of sources, downloading models automatically from the [latest YOLOv5 release](https://github.com/ultralytics/yolov5/releases) and saving results to `inference/output`.
+detect.py runs inference on a variety of sources, downloading models automatically from the [latest YOLOv5 release](https://github.com/ultralytics/yolov5/releases) and saving results to `runs/detect`.
 ```bash
 $ python detect.py --source 0  # webcam
                             file.jpg  # image 
@@ -82,20 +82,20 @@ $ python detect.py --source 0  # webcam
                             http://112.50.243.8/PLTV/88888888/224/3221225900/1.m3u8  # http stream
 ```
 
-To run inference on example images in `inference/images`:
+To run inference on example images in `data/images`:
 ```bash
-$ python detect.py --source inference/images --weights yolov5s.pt --conf 0.25
+$ python detect.py --source data/images --weights yolov5s.pt --conf 0.25
 
-Namespace(agnostic_nms=False, augment=False, classes=None, conf_thres=0.25, device='', img_size=640, iou_thres=0.45, output='inference/output', save_conf=False, save_txt=False, source='inference/images', update=False, view_img=False, weights='yolov5s.pt')
+Namespace(agnostic_nms=False, augment=False, classes=None, conf_thres=0.25, device='', img_size=640, iou_thres=0.45, output='runs/detect', save_conf=False, save_txt=False, source='data/images', update=False, view_img=False, weights='yolov5s.pt')
 Using CUDA device0 _CudaDeviceProperties(name='Tesla V100-SXM2-16GB', total_memory=16160MB)
 
 Downloading https://github.com/ultralytics/yolov5/releases/download/v3.0/yolov5s.pt to yolov5s.pt... 100%|██████████████| 14.5M/14.5M [00:00<00:00, 21.3MB/s]
 
 Fusing layers... 
 Model Summary: 140 layers, 7.45958e+06 parameters, 0 gradients
-image 1/2 yolov5/inference/images/bus.jpg: 640x480 4 persons, 1 buss, 1 skateboards, Done. (0.013s)
+image 1/2 data/images/bus.jpg: 640x480 4 persons, 1 buss, 1 skateboards, Done. (0.013s)
-image 2/2 yolov5/inference/images/zidane.jpg: 384x640 2 persons, 2 ties, Done. (0.013s)
+image 2/2 data/images/zidane.jpg: 384x640 2 persons, 2 ties, Done. (0.013s)
-Results saved to yolov5/inference/output
+Results saved to runs/detect/exp0
 Done. (0.124s)
 ```
 <img src="https://user-images.githubusercontent.com/26833433/97107365-685a8d80-16c7-11eb-8c2e-83aac701d8b9.jpeg" width="500">  

detect.py

@@ -1,6 +1,5 @@
 import argparse
 import os
-import shutil
 import time
 from pathlib import Path
 
@@ -11,23 +10,25 @@ from numpy import random
 
 from models.experimental import attempt_load
 from utils.datasets import LoadStreams, LoadImages
-from utils.general import (
+from utils.general import check_img_size, non_max_suppression, apply_classifier, scale_coords, xyxy2xywh, \
-    check_img_size, non_max_suppression, apply_classifier, scale_coords,
+    plot_one_box, strip_optimizer, set_logging, increment_dir
-    xyxy2xywh, plot_one_box, strip_optimizer, set_logging)
 from utils.torch_utils import select_device, load_classifier, time_synchronized
 
 
 def detect(save_img=False):
-    out, source, weights, view_img, save_txt, imgsz = \
+    save_dir, source, weights, view_img, save_txt, imgsz = \
-        opt.save_dir, opt.source, opt.weights, opt.view_img, opt.save_txt, opt.img_size
+        Path(opt.save_dir), opt.source, opt.weights, opt.view_img, opt.save_txt, opt.img_size
     webcam = source.isnumeric() or source.startswith(('rtsp://', 'rtmp://', 'http://')) or source.endswith('.txt')
 
+    # Directories
+    if save_dir == Path('runs/detect'):  # if default
+        os.makedirs('runs/detect', exist_ok=True)  # make base
+        save_dir = Path(increment_dir(save_dir / 'exp', opt.name))  # increment run
+    os.makedirs(save_dir / 'labels' if save_txt else save_dir, exist_ok=True)  # make new dir
+
     # Initialize
     set_logging()
     device = select_device(opt.device)
-    if os.path.exists(out):  # output dir
-        shutil.rmtree(out)  # delete dir
-    os.makedirs(out)  # make new dir
     half = device.type != 'cpu'  # half precision only supported on CUDA
 
     # Load model
@@ -83,12 +84,12 @@ def detect(save_img=False):
         # Process detections
         for i, det in enumerate(pred):  # detections per image
             if webcam:  # batch_size >= 1
-                p, s, im0 = path[i], '%g: ' % i, im0s[i].copy()
+                p, s, im0 = Path(path[i]), '%g: ' % i, im0s[i].copy()
             else:
-                p, s, im0 = path, '', im0s
+                p, s, im0 = Path(path), '', im0s
 
-            save_path = str(Path(out) / Path(p).name)
+            save_path = str(save_dir / p.name)
-            txt_path = str(Path(out) / Path(p).stem) + ('_%g' % dataset.frame if dataset.mode == 'video' else '')
+            txt_path = str(save_dir / 'labels' / p.stem) + ('_%g' % dataset.frame if dataset.mode == 'video' else '')
             s += '%gx%g ' % img.shape[2:]  # print string
             gn = torch.tensor(im0.shape)[[1, 0, 1, 0]]  # normalization gain whwh
             if det is not None and len(det):
@@ -104,7 +105,7 @@ def detect(save_img=False):
                 for *xyxy, conf, cls in reversed(det):
                     if save_txt:  # Write to file
                         xywh = (xyxy2xywh(torch.tensor(xyxy).view(1, 4)) / gn).view(-1).tolist()  # normalized xywh
-                        line = (cls, conf, *xywh) if opt.save_conf else (cls, *xywh)  # label format
+                        line = (cls, *xywh, conf) if opt.save_conf else (cls, *xywh)  # label format
                         with open(txt_path + '.txt', 'a') as f:
                             f.write(('%g ' * len(line) + '\n') % line)
 
@@ -139,7 +140,7 @@ def detect(save_img=False):
                     vid_writer.write(im0)
 
     if save_txt or save_img:
-        print('Results saved to %s' % Path(out))
+        print('Results saved to %s' % save_dir)
 
     print('Done. (%.3fs)' % (time.time() - t0))
 
@@ -147,15 +148,16 @@ def detect(save_img=False):
 if __name__ == '__main__':
     parser = argparse.ArgumentParser()
     parser.add_argument('--weights', nargs='+', type=str, default='yolov5s.pt', help='model.pt path(s)')
-    parser.add_argument('--source', type=str, default='inference/images', help='source')  # file/folder, 0 for webcam
+    parser.add_argument('--source', type=str, default='data/images', help='source')  # file/folder, 0 for webcam
     parser.add_argument('--img-size', type=int, default=640, help='inference size (pixels)')
     parser.add_argument('--conf-thres', type=float, default=0.25, help='object confidence threshold')
     parser.add_argument('--iou-thres', type=float, default=0.45, help='IOU threshold for NMS')
     parser.add_argument('--device', default='', help='cuda device, i.e. 0 or 0,1,2,3 or cpu')
     parser.add_argument('--view-img', action='store_true', help='display results')
-    parser.add_argument('--save-txt', action='store_true', help='save results to *.txt')
+    parser.add_argument('--save-txt', action='store_false', help='save results to *.txt')
     parser.add_argument('--save-conf', action='store_true', help='save confidences in --save-txt labels')
-    parser.add_argument('--save-dir', type=str, default='inference/output', help='directory to save results')
+    parser.add_argument('--save-dir', type=str, default='runs/detect', help='directory to save results')
+    parser.add_argument('--name', default='', help='name to append to --save-dir: i.e. runs/{N} -> runs/{N}_{name}')
     parser.add_argument('--classes', nargs='+', type=int, help='filter by class: --class 0, or --class 0 2 3')
     parser.add_argument('--agnostic-nms', action='store_true', help='class-agnostic NMS')
     parser.add_argument('--augment', action='store_true', help='augmented inference')

hubconf.py

@@ -113,6 +113,6 @@ if __name__ == '__main__':
     # Verify inference
     from PIL import Image
 
-    img = Image.open('inference/images/zidane.jpg')
+    img = Image.open('data/images/zidane.jpg')
     y = model(img)
     print(y[0].shape)

sotabench.py

@@ -1,307 +0,0 @@
-import argparse
-import glob
-import os
-import shutil
-from pathlib import Path
-
-import numpy as np
-import torch
-import yaml
-from sotabencheval.object_detection import COCOEvaluator
-from sotabencheval.utils import is_server
-from tqdm import tqdm
-
-from models.experimental import attempt_load
-from utils.datasets import create_dataloader
-from utils.general import (
-    coco80_to_coco91_class, check_dataset, check_file, check_img_size, compute_loss, non_max_suppression, scale_coords,
-    xyxy2xywh, clip_coords, set_logging)
-from utils.torch_utils import select_device, time_synchronized
-
-DATA_ROOT = './.data/vision/coco' if is_server() else '../coco'  # sotabench data dir
-
-
-def test(data,
-         weights=None,
-         batch_size=16,
-         imgsz=640,
-         conf_thres=0.001,
-         iou_thres=0.6,  # for NMS
-         save_json=False,
-         single_cls=False,
-         augment=False,
-         verbose=False,
-         model=None,
-         dataloader=None,
-         save_dir='',
-         merge=False,
-         save_txt=False):
-    # Initialize/load model and set device
-    training = model is not None
-    if training:  # called by train.py
-        device = next(model.parameters()).device  # get model device
-
-    else:  # called directly
-        set_logging()
-        device = select_device(opt.device, batch_size=batch_size)
-        merge, save_txt = opt.merge, opt.save_txt  # use Merge NMS, save *.txt labels
-        if save_txt:
-            out = Path('inference/output')
-            if os.path.exists(out):
-                shutil.rmtree(out)  # delete output folder
-            os.makedirs(out)  # make new output folder
-
-        # Remove previous
-        for f in glob.glob(str(Path(save_dir) / 'test_batch*.jpg')):
-            os.remove(f)
-
-        # Load model
-        model = attempt_load(weights, map_location=device)  # load FP32 model
-        imgsz = check_img_size(imgsz, s=model.stride.max())  # check img_size
-
-        # Multi-GPU disabled, incompatible with .half() https://github.com/ultralytics/yolov5/issues/99
-        # if device.type != 'cpu' and torch.cuda.device_count() > 1:
-        #     model = nn.DataParallel(model)
-
-    # Half
-    half = device.type != 'cpu'  # half precision only supported on CUDA
-    if half:
-        model.half()
-
-    # Configure
-    model.eval()
-    with open(data) as f:
-        data = yaml.load(f, Loader=yaml.FullLoader)  # model dict
-    check_dataset(data)  # check
-    nc = 1 if single_cls else int(data['nc'])  # number of classes
-    iouv = torch.linspace(0.5, 0.95, 10).to(device)  # iou vector for mAP@0.5:0.95
-    niou = iouv.numel()
-
-    # Dataloader
-    if not training:
-        img = torch.zeros((1, 3, imgsz, imgsz), device=device)  # init img
-        _ = model(img.half() if half else img) if device.type != 'cpu' else None  # run once
-        path = data['test'] if opt.task == 'test' else data['val']  # path to val/test images
-        dataloader = create_dataloader(path, imgsz, batch_size, model.stride.max(), opt,
-                                       hyp=None, augment=False, cache=True, pad=0.5, rect=True)[0]
-
-    seen = 0
-    names = model.names if hasattr(model, 'names') else model.module.names
-    coco91class = coco80_to_coco91_class()
-    s = ('%20s' + '%12s' * 6) % ('Class', 'Images', 'Targets', 'P', 'R', 'mAP@.5', 'mAP@.5:.95')
-    p, r, f1, mp, mr, map50, map, t0, t1 = 0., 0., 0., 0., 0., 0., 0., 0., 0.
-    loss = torch.zeros(3, device=device)
-    jdict, stats, ap, ap_class = [], [], [], []
-    evaluator = COCOEvaluator(root=DATA_ROOT, model_name=opt.weights.replace('.pt', ''))
-    for batch_i, (img, targets, paths, shapes) in enumerate(tqdm(dataloader, desc=s)):
-        img = img.to(device, non_blocking=True)
-        img = img.half() if half else img.float()  # uint8 to fp16/32
-        img /= 255.0  # 0 - 255 to 0.0 - 1.0
-        targets = targets.to(device)
-        nb, _, height, width = img.shape  # batch size, channels, height, width
-        whwh = torch.Tensor([width, height, width, height]).to(device)
-
-        # Disable gradients
-        with torch.no_grad():
-            # Run model
-            t = time_synchronized()
-            inf_out, train_out = model(img, augment=augment)  # inference and training outputs
-            t0 += time_synchronized() - t
-
-            # Compute loss
-            if training:  # if model has loss hyperparameters
-                loss += compute_loss([x.float() for x in train_out], targets, model)[1][:3]  # box, obj, cls
-
-            # Run NMS
-            t = time_synchronized()
-            output = non_max_suppression(inf_out, conf_thres=conf_thres, iou_thres=iou_thres, merge=merge)
-            t1 += time_synchronized() - t
-
-        # Statistics per image
-        for si, pred in enumerate(output):
-            labels = targets[targets[:, 0] == si, 1:]
-            nl = len(labels)
-            tcls = labels[:, 0].tolist() if nl else []  # target class
-            seen += 1
-
-            if pred is None:
-                if nl:
-                    stats.append((torch.zeros(0, niou, dtype=torch.bool), torch.Tensor(), torch.Tensor(), tcls))
-                continue
-
-            # Append to text file
-            if save_txt:
-                gn = torch.tensor(shapes[si][0])[[1, 0, 1, 0]]  # normalization gain whwh
-                x = pred.clone()
-                x[:, :4] = scale_coords(img[si].shape[1:], x[:, :4], shapes[si][0], shapes[si][1])  # to original
-                for *xyxy, conf, cls in x:
-                    xywh = (xyxy2xywh(torch.tensor(xyxy).view(1, 4)) / gn).view(-1).tolist()  # normalized xywh
-                    with open(str(out / Path(paths[si]).stem) + '.txt', 'a') as f:
-                        f.write(('%g ' * 5 + '\n') % (cls, *xywh))  # label format
-
-            # Clip boxes to image bounds
-            clip_coords(pred, (height, width))
-
-            # Append to pycocotools JSON dictionary
-            if save_json:
-                # [{"image_id": 42, "category_id": 18, "bbox": [258.15, 41.29, 348.26, 243.78], "score": 0.236}, ...
-                image_id = Path(paths[si]).stem
-                box = pred[:, :4].clone()  # xyxy
-                scale_coords(img[si].shape[1:], box, shapes[si][0], shapes[si][1])  # to original shape
-                box = xyxy2xywh(box)  # xywh
-                box[:, :2] -= box[:, 2:] / 2  # xy center to top-left corner
-                for p, b in zip(pred.tolist(), box.tolist()):
-                    result = {'image_id': int(image_id) if image_id.isnumeric() else image_id,
-                                  'category_id': coco91class[int(p[5])],
-                                  'bbox': [round(x, 3) for x in b],
-                                  'score': round(p[4], 5)}
-                    jdict.append(result)
-
-                    #evaluator.add([result])
-                    #if evaluator.cache_exists:
-                    #    break
-
-            # # Assign all predictions as incorrect
-            # correct = torch.zeros(pred.shape[0], niou, dtype=torch.bool, device=device)
-            # if nl:
-            #     detected = []  # target indices
-            #     tcls_tensor = labels[:, 0]
-            #
-            #     # target boxes
-            #     tbox = xywh2xyxy(labels[:, 1:5]) * whwh
-            #
-            #     # Per target class
-            #     for cls in torch.unique(tcls_tensor):
-            #         ti = (cls == tcls_tensor).nonzero(as_tuple=False).view(-1)  # prediction indices
-            #         pi = (cls == pred[:, 5]).nonzero(as_tuple=False).view(-1)  # target indices
-            #
-            #         # Search for detections
-            #         if pi.shape[0]:
-            #             # Prediction to target ious
-            #             ious, i = box_iou(pred[pi, :4], tbox[ti]).max(1)  # best ious, indices
-            #
-            #             # Append detections
-            #             detected_set = set()
-            #             for j in (ious > iouv[0]).nonzero(as_tuple=False):
-            #                 d = ti[i[j]]  # detected target
-            #                 if d.item() not in detected_set:
-            #                     detected_set.add(d.item())
-            #                     detected.append(d)
-            #                     correct[pi[j]] = ious[j] > iouv  # iou_thres is 1xn
-            #                     if len(detected) == nl:  # all targets already located in image
-            #                         break
-            #
-            # # Append statistics (correct, conf, pcls, tcls)
-            # stats.append((correct.cpu(), pred[:, 4].cpu(), pred[:, 5].cpu(), tcls))
-
-        # # Plot images
-        # if batch_i < 1:
-        #     f = Path(save_dir) / ('test_batch%g_gt.jpg' % batch_i)  # filename
-        #     plot_images(img, targets, paths, str(f), names)  # ground truth
-        #     f = Path(save_dir) / ('test_batch%g_pred.jpg' % batch_i)
-        #     plot_images(img, output_to_target(output, width, height), paths, str(f), names)  # predictions
-
-    evaluator.add(jdict)
-    evaluator.save()
-
-    # # Compute statistics
-    # stats = [np.concatenate(x, 0) for x in zip(*stats)]  # to numpy
-    # if len(stats) and stats[0].any():
-    #     p, r, ap, f1, ap_class = ap_per_class(*stats)
-    #     p, r, ap50, ap = p[:, 0], r[:, 0], ap[:, 0], ap.mean(1)  # [P, R, AP@0.5, AP@0.5:0.95]
-    #     mp, mr, map50, map = p.mean(), r.mean(), ap50.mean(), ap.mean()
-    #     nt = np.bincount(stats[3].astype(np.int64), minlength=nc)  # number of targets per class
-    # else:
-    #     nt = torch.zeros(1)
-    #
-    # # Print results
-    # pf = '%20s' + '%12.3g' * 6  # print format
-    # print(pf % ('all', seen, nt.sum(), mp, mr, map50, map))
-    #
-    # # Print results per class
-    # if verbose and nc > 1 and len(stats):
-    #     for i, c in enumerate(ap_class):
-    #         print(pf % (names[c], seen, nt[c], p[i], r[i], ap50[i], ap[i]))
-    #
-    # # Print speeds
-    # t = tuple(x / seen * 1E3 for x in (t0, t1, t0 + t1)) + (imgsz, imgsz, batch_size)  # tuple
-    # if not training:
-    #     print('Speed: %.1f/%.1f/%.1f ms inference/NMS/total per %gx%g image at batch-size %g' % t)
-    #
-    # # Save JSON
-    # if save_json and len(jdict):
-    #     f = 'detections_val2017_%s_results.json' % \
-    #         (weights.split(os.sep)[-1].replace('.pt', '') if isinstance(weights, str) else '')  # filename
-    #     print('\nCOCO mAP with pycocotools... saving %s...' % f)
-    #     with open(f, 'w') as file:
-    #         json.dump(jdict, file)
-    #
-    #     try:  # https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocoEvalDemo.ipynb
-    #         from pycocotools.coco import COCO
-    #         from pycocotools.cocoeval import COCOeval
-    #
-    #         imgIds = [int(Path(x).stem) for x in dataloader.dataset.img_files]
-    #         cocoGt = COCO(glob.glob('../coco/annotations/instances_val*.json')[0])  # initialize COCO ground truth api
-    #         cocoDt = cocoGt.loadRes(f)  # initialize COCO pred api
-    #         cocoEval = COCOeval(cocoGt, cocoDt, 'bbox')
-    #         cocoEval.params.imgIds = imgIds  # image IDs to evaluate
-    #         cocoEval.evaluate()
-    #         cocoEval.accumulate()
-    #         cocoEval.summarize()
-    #         map, map50 = cocoEval.stats[:2]  # update results (mAP@0.5:0.95, mAP@0.5)
-    #     except Exception as e:
-    #         print('ERROR: pycocotools unable to run: %s' % e)
-    #
-    # # Return results
-    # model.float()  # for training
-    # maps = np.zeros(nc) + map
-    # for i, c in enumerate(ap_class):
-    #     maps[c] = ap[i]
-    # return (mp, mr, map50, map, *(loss.cpu() / len(dataloader)).tolist()), maps, t
-
-
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser(prog='test.py')
-    parser.add_argument('--weights', nargs='+', type=str, default='yolov5s.pt', help='model.pt path(s)')
-    parser.add_argument('--data', type=str, default='data/coco.yaml', help='*.data path')
-    parser.add_argument('--batch-size', type=int, default=32, help='size of each image batch')
-    parser.add_argument('--img-size', type=int, default=640, help='inference size (pixels)')
-    parser.add_argument('--conf-thres', type=float, default=0.001, help='object confidence threshold')
-    parser.add_argument('--iou-thres', type=float, default=0.65, help='IOU threshold for NMS')
-    parser.add_argument('--save-json', action='store_true', help='save a cocoapi-compatible JSON results file')
-    parser.add_argument('--task', default='val', help="'val', 'test', 'study'")
-    parser.add_argument('--device', default='', help='cuda device, i.e. 0 or 0,1,2,3 or cpu')
-    parser.add_argument('--single-cls', action='store_true', help='treat as single-class dataset')
-    parser.add_argument('--augment', action='store_true', help='augmented inference')
-    parser.add_argument('--merge', action='store_true', help='use Merge NMS')
-    parser.add_argument('--verbose', action='store_true', help='report mAP by class')
-    parser.add_argument('--save-txt', action='store_true', help='save results to *.txt')
-    opt = parser.parse_args()
-    opt.save_json |= opt.data.endswith('coco.yaml')
-    opt.data = check_file(opt.data)  # check file
-    print(opt)
-
-    if opt.task in ['val', 'test']:  # run normally
-        test(opt.data,
-             opt.weights,
-             opt.batch_size,
-             opt.img_size,
-             opt.conf_thres,
-             opt.iou_thres,
-             opt.save_json,
-             opt.single_cls,
-             opt.augment,
-             opt.verbose)
-
-    elif opt.task == 'study':  # run over a range of settings and save/plot
-        for weights in ['yolov5s.pt', 'yolov5m.pt', 'yolov5l.pt', 'yolov5x.pt']:
-            f = 'study_%s_%s.txt' % (Path(opt.data).stem, Path(weights).stem)  # filename to save to
-            x = list(range(320, 800, 64))  # x axis
-            y = []  # y axis
-            for i in x:  # img-size
-                print('\nRunning %s point %s...' % (f, i))
-                r, _, t = test(opt.data, weights, opt.batch_size, i, opt.conf_thres, opt.iou_thres, opt.save_json)
-                y.append(r + t)  # results and times
-            np.savetxt(f, y, fmt='%10.4g')  # save
-        os.system('zip -r study.zip study_*.txt')
-        # utils.general.plot_study_txt(f, x)  # plot

test.py

@@ -2,7 +2,6 @@ import argparse
 import glob
 import json
 import os
-import shutil
 from pathlib import Path
 
 import numpy as np
@@ -12,9 +11,9 @@ from tqdm import tqdm
 
 from models.experimental import attempt_load
 from utils.datasets import create_dataloader
-from utils.general import (
+from utils.general import coco80_to_coco91_class, check_dataset, check_file, check_img_size, compute_loss, \
-    coco80_to_coco91_class, check_dataset, check_file, check_img_size, compute_loss, non_max_suppression, scale_coords,
+    non_max_suppression, scale_coords, xyxy2xywh, clip_coords, plot_images, xywh2xyxy, box_iou, output_to_target, \
-    xyxy2xywh, clip_coords, plot_images, xywh2xyxy, box_iou, output_to_target, ap_per_class, set_logging)
+    ap_per_class, set_logging, increment_dir
 from utils.torch_utils import select_device, time_synchronized
 
 
@@ -46,16 +45,11 @@ def test(data,
         device = select_device(opt.device, batch_size=batch_size)
         save_txt = opt.save_txt  # save *.txt labels
 
-        # Remove previous
+        # Directories
-        if os.path.exists(save_dir):
+        if save_dir == Path('runs/test'):  # if default
-            shutil.rmtree(save_dir)  # delete dir
+            os.makedirs('runs/test', exist_ok=True)  # make base
-        os.makedirs(save_dir)  # make new dir
+            save_dir = Path(increment_dir(save_dir / 'exp', opt.name))  # increment run
-
+        os.makedirs(save_dir / 'labels' if save_txt else save_dir, exist_ok=True)  # make new dir
-        if save_txt:
-            out = save_dir / 'autolabels'
-            if os.path.exists(out):
-                shutil.rmtree(out)  # delete dir
-            os.makedirs(out)  # make new dir
 
         # Load model
         model = attempt_load(weights, map_location=device)  # load FP32 model
@@ -144,8 +138,8 @@ def test(data,
                 x[:, :4] = scale_coords(img[si].shape[1:], x[:, :4], shapes[si][0], shapes[si][1])  # to original
                 for *xyxy, conf, cls in x:
                     xywh = (xyxy2xywh(torch.tensor(xyxy).view(1, 4)) / gn).view(-1).tolist()  # normalized xywh
-                    line = (cls, conf, *xywh) if save_conf else (cls, *xywh)  # label format
+                    line = (cls, *xywh, conf) if save_conf else (cls, *xywh)  # label format
-                    with open(str(out / Path(paths[si]).stem) + '.txt', 'a') as f:
+                    with open(str(save_dir / 'labels' / Path(paths[si]).stem) + '.txt', 'a') as f:
                         f.write(('%g ' * len(line) + '\n') % line)
 
             # W&B logging
@@ -268,6 +262,7 @@ def test(data,
             print('ERROR: pycocotools unable to run: %s' % e)
 
     # Return results
+    print('Results saved to %s' % save_dir)
     model.float()  # for training
     maps = np.zeros(nc) + map
     for i, c in enumerate(ap_class):
@@ -292,6 +287,7 @@ if __name__ == '__main__':
     parser.add_argument('--save-txt', action='store_true', help='save results to *.txt')
     parser.add_argument('--save-conf', action='store_true', help='save confidences in --save-txt labels')
     parser.add_argument('--save-dir', type=str, default='runs/test', help='directory to save results')
+    parser.add_argument('--name', default='', help='name to append to --save-dir: i.e. runs/{N} -> runs/{N}_{name}')
     opt = parser.parse_args()
     opt.save_json |= opt.data.endswith('coco.yaml')
     opt.data = check_file(opt.data)  # check file
@@ -313,8 +309,6 @@ if __name__ == '__main__':
              save_conf=opt.save_conf,
              )
 
-        print('Results saved to %s' % opt.save_dir)
-
     elif opt.task == 'study':  # run over a range of settings and save/plot
         for weights in ['yolov5s.pt', 'yolov5m.pt', 'yolov5l.pt', 'yolov5x.pt']:
             f = 'study_%s_%s.txt' % (Path(opt.data).stem, Path(weights).stem)  # filename to save to

train.py

@@ -1,5 +1,6 @@
 import argparse
 import logging
+import math
 import os
 import random
 import shutil
@@ -7,7 +8,6 @@ import time
 from pathlib import Path
 from warnings import warn
 
-import math
 import numpy as np
 import torch.distributed as dist
 import torch.nn as nn
@@ -404,14 +404,14 @@ if __name__ == '__main__':
     parser.add_argument('--bucket', type=str, default='', help='gsutil bucket')
     parser.add_argument('--cache-images', action='store_true', help='cache images for faster training')
     parser.add_argument('--image-weights', action='store_true', help='use weighted image selection for training')
-    parser.add_argument('--name', default='', help='renames experiment folder exp{N} to exp{N}_{name} if supplied')
     parser.add_argument('--device', default='', help='cuda device, i.e. 0 or 0,1,2,3 or cpu')
     parser.add_argument('--multi-scale', action='store_true', help='vary img-size +/- 50%%')
     parser.add_argument('--single-cls', action='store_true', help='train as single-class dataset')
     parser.add_argument('--adam', action='store_true', help='use torch.optim.Adam() optimizer')
     parser.add_argument('--sync-bn', action='store_true', help='use SyncBatchNorm, only available in DDP mode')
     parser.add_argument('--local_rank', type=int, default=-1, help='DDP parameter, do not modify')
-    parser.add_argument('--logdir', type=str, default='runs/', help='logging directory')
+    parser.add_argument('--logdir', type=str, default='runs/train', help='logging directory')
+    parser.add_argument('--name', default='', help='name to append to --save-dir: i.e. runs/{N} -> runs/{N}_{name}')
     parser.add_argument('--log-imgs', type=int, default=10, help='number of images for W&B logging, max 100')
     parser.add_argument('--workers', type=int, default=8, help='maximum number of dataloader workers')
 
@@ -428,7 +428,7 @@ if __name__ == '__main__':
     # Resume
     if opt.resume:  # resume an interrupted run
         ckpt = opt.resume if isinstance(opt.resume, str) else get_latest_run()  # specified or most recent path
-        log_dir = Path(ckpt).parent.parent  # runs/exp0
+        log_dir = Path(ckpt).parent.parent  # runs/train/exp0
         assert os.path.isfile(ckpt), 'ERROR: --resume checkpoint does not exist'
         with open(log_dir / 'opt.yaml') as f:
             opt = argparse.Namespace(**yaml.load(f, Loader=yaml.FullLoader))  # replace
@@ -467,14 +467,13 @@ if __name__ == '__main__':
         if opt.global_rank in [-1, 0]:
             # Tensorboard
             logger.info(f'Start Tensorboard with "tensorboard --logdir {opt.logdir}", view at http://localhost:6006/')
-            tb_writer = SummaryWriter(log_dir=log_dir)  # runs/exp0
+            tb_writer = SummaryWriter(log_dir=log_dir)  # runs/train/exp0
 
             # W&B
             try:
                 import wandb
 
                 assert os.environ.get('WANDB_DISABLED') != 'true'
-                logger.info("Weights & Biases logging enabled, to disable set os.environ['WANDB_DISABLED'] = 'true'")
             except (ImportError, AssertionError):
                 opt.log_imgs = 0
                 logger.info("Install Weights & Biases for experiment logging via 'pip install wandb' (recommended)")

tutorial.ipynb

@@ -596,22 +596,22 @@
         }
       },
       "source": [
-        "!python detect.py --weights yolov5s.pt --img 640 --conf 0.25 --source inference/images/\n",
+        "!python detect.py --weights yolov5s.pt --img 640 --conf 0.25 --source data/images/\n",
-        "Image(filename='inference/output/zidane.jpg', width=600)"
+        "Image(filename='runs/detect/exp0/zidane.jpg', width=600)"
       ],
       "execution_count": null,
       "outputs": [
         {
           "output_type": "stream",
           "text": [
-            "Namespace(agnostic_nms=False, augment=False, classes=None, conf_thres=0.25, device='', img_size=640, iou_thres=0.45, save_conf=False, save_dir='inference/output', save_txt=False, source='inference/images/', update=False, view_img=False, weights=['yolov5s.pt'])\n",
+            "Namespace(agnostic_nms=False, augment=False, classes=None, conf_thres=0.25, device='', img_size=640, iou_thres=0.45, save_conf=False, save_dir='runs/detect', save_txt=False, source='data/images/', update=False, view_img=False, weights=['yolov5s.pt'])\n",
             "Using CUDA device0 _CudaDeviceProperties(name='Tesla V100-SXM2-16GB', total_memory=16130MB)\n",
             "\n",
             "Fusing layers... \n",
             "Model Summary: 140 layers, 7.45958e+06 parameters, 0 gradients\n",
-            "image 1/2 /content/yolov5/inference/images/bus.jpg: 640x480 4 persons, 1 buss, 1 skateboards, Done. (0.012s)\n",
+            "image 1/2 /content/yolov5/data/images/bus.jpg: 640x480 4 persons, 1 buss, 1 skateboards, Done. (0.012s)\n",
-            "image 2/2 /content/yolov5/inference/images/zidane.jpg: 384x640 2 persons, 2 ties, Done. (0.012s)\n",
+            "image 2/2 /content/yolov5/data/images/zidane.jpg: 384x640 2 persons, 2 ties, Done. (0.012s)\n",
-            "Results saved to inference/output\n",
+            "Results saved to runs/detect/exp0\n",
             "Done. (0.113s)\n"
           ],
           "name": "stdout"
@@ -640,7 +640,7 @@
         "id": "4qbaa3iEcrcE"
       },
       "source": [
-        "Results are saved to `inference/output`. A full list of available inference sources:\n",
+        "Results are saved to `runs/detect`. A full list of available inference sources:\n",
         "<img src=\"https://user-images.githubusercontent.com/26833433/98274798-2b7a7a80-1f94-11eb-91a4-70c73593e26b.jpg\" width=\"900\"> "
       ]
     },
@@ -887,7 +887,7 @@
       "source": [
         "Train a YOLOv5s model on [COCO128](https://www.kaggle.com/ultralytics/coco128) with dataset `--data coco128.yaml`, starting from pretrained `--weights yolov5s.pt`, or from randomly initialized `--weights '' --cfg yolov5s.yaml`. Models are downloaded automatically from the [latest YOLOv5 release](https://github.com/ultralytics/yolov5/releases), and **COCO, COCO128, and VOC datasets are downloaded automatically** on first use.\n",
         "\n",
-        "All training results are saved to `runs/exp0` for the first experiment, then `runs/exp1`, `runs/exp2` etc. for subsequent experiments.\n"
+        "All training results are saved to `runs/train/exp0` for the first experiment, then `runs/exp1`, `runs/exp2` etc. for subsequent experiments.\n"
       ]
     },
     {
@@ -969,7 +969,7 @@
             "Analyzing anchors... anchors/target = 4.26, Best Possible Recall (BPR) = 0.9946\n",
             "Image sizes 640 train, 640 test\n",
             "Using 2 dataloader workers\n",
-            "Logging results to runs/exp0\n",
+            "Logging results to runs/train/exp0\n",
             "Starting training for 3 epochs...\n",
             "\n",
             "     Epoch   gpu_mem       box       obj       cls     total   targets  img_size\n",
@@ -986,8 +986,8 @@
             "       2/2     3.17G   0.04445   0.06545   0.01666    0.1266       149       640: 100% 8/8 [00:01<00:00,  4.33it/s]\n",
             "               Class      Images     Targets           P           R      mAP@.5  mAP@.5:.95: 100% 8/8 [00:02<00:00,  2.78it/s]\n",
             "                 all         128         929       0.395       0.766       0.701       0.455\n",
-            "Optimizer stripped from runs/exp0/weights/last.pt, 15.2MB\n",
+            "Optimizer stripped from runs/train/exp0/weights/last.pt, 15.2MB\n",
-            "Optimizer stripped from runs/exp0/weights/best.pt, 15.2MB\n",
+            "Optimizer stripped from runs/train/exp0/weights/best.pt, 15.2MB\n",
             "3 epochs completed in 0.005 hours.\n",
             "\n"
           ],
@@ -1030,7 +1030,7 @@
       "source": [
         "## Local Logging\n",
         "\n",
-        "All results are logged by default to the `runs/exp0` directory, with a new directory created for each new training as `runs/exp1`, `runs/exp2`, etc. View train and test jpgs to see mosaics, labels/predictions and augmentation effects. Note a **Mosaic Dataloader** is used for training (shown below), a new concept developed by Ultralytics and first featured in [YOLOv4](https://arxiv.org/abs/2004.10934)."
+        "All results are logged by default to the `runs/train/exp0` directory, with a new directory created for each new training as `runs/exp1`, `runs/exp2`, etc. View train and test jpgs to see mosaics, labels/predictions and augmentation effects. Note a **Mosaic Dataloader** is used for training (shown below), a new concept developed by Ultralytics and first featured in [YOLOv4](https://arxiv.org/abs/2004.10934)."
       ]
     },
     {
@@ -1039,9 +1039,9 @@
         "id": "riPdhraOTCO0"
       },
       "source": [
-        "Image(filename='runs/exp0/train_batch0.jpg', width=800)  # train batch 0 mosaics and labels\n",
+        "Image(filename='runs/train/exp0/train_batch0.jpg', width=800)  # train batch 0 mosaics and labels\n",
-        "Image(filename='runs/exp0/test_batch0_gt.jpg', width=800)  # test batch 0 ground truth\n",
+        "Image(filename='runs/train/exp0/test_batch0_gt.jpg', width=800)  # test batch 0 ground truth\n",
-        "Image(filename='runs/exp0/test_batch0_pred.jpg', width=800)  # test batch 0 predictions"
+        "Image(filename='runs/train/exp0/test_batch0_pred.jpg', width=800)  # test batch 0 predictions"
       ],
       "execution_count": null,
       "outputs": []
@@ -1078,7 +1078,7 @@
       },
       "source": [
         "from utils.utils import plot_results \n",
-        "plot_results(save_dir='runs/exp0')  # plot results.txt as results.png\n",
+        "plot_results(save_dir='runs/train/exp0')  # plot results.txt as results.png\n",
         "Image(filename='results.png', width=800) "
       ],
       "execution_count": null,
@@ -1170,9 +1170,9 @@
         "  for di in 0 cpu # inference devices\n",
         "  do\n",
         "    python detect.py --weights $x.pt --device $di  # detect official\n",
-        "    python detect.py --weights runs/exp0/weights/last.pt --device $di  # detect custom\n",
+        "    python detect.py --weights runs/train/exp0/weights/last.pt --device $di  # detect custom\n",
         "    python test.py --weights $x.pt --device $di # test official\n",
-        "    python test.py --weights runs/exp0/weights/last.pt --device $di # test custom\n",
+        "    python test.py --weights runs/train/exp0/weights/last.pt --device $di # test custom\n",
         "  done\n",
         "  python models/yolo.py --cfg $x.yaml # inspect\n",
         "  python models/export.py --weights $x.pt --img 640 --batch 1 # export\n",

utils/general.py

@@ -955,9 +955,15 @@ def increment_dir(dir, comment=''):
     # Increments a directory runs/exp1 --> runs/exp2_comment
     n = 0  # number
     dir = str(Path(dir))  # os-agnostic
+    if os.path.isdir(dir):
+        stem = ''
+        dir += os.sep  # removed by Path
+    else:
+        stem = Path(dir).stem
+
     dirs = sorted(glob.glob(dir + '*'))  # directories
     if dirs:
-        matches = [re.search(r"exp(\d+)", d) for d in dirs]
+        matches = [re.search(r"%s(\d+)" % stem, d) for d in dirs]
         idxs = [int(m.groups()[0]) for m in matches if m]
         if idxs:
             n = max(idxs) + 1  # increment
@@ -1262,7 +1268,7 @@ def plot_results_overlay(start=0, stop=0):  # from utils.general import *; plot_
 
 
 def plot_results(start=0, stop=0, bucket='', id=(), labels=(), save_dir=''):
-    # from utils.general import *; plot_results(save_dir='runs/exp0')
+    # from utils.general import *; plot_results(save_dir='runs/train/exp0')
     # Plot training 'results*.txt' as seen in https://github.com/ultralytics/yolov5#reproduce-our-training
     fig, ax = plt.subplots(2, 5, figsize=(12, 6))
     ax = ax.ravel()