mirror of https://github.com/JosephKJ/OWOD.git
121 lines
7.9 KiB
Python
121 lines
7.9 KiB
Python
# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
|
|
import contextlib
|
|
import copy
|
|
import io
|
|
import json
|
|
import numpy as np
|
|
import os
|
|
import tempfile
|
|
import unittest
|
|
from pycocotools.coco import COCO
|
|
from pycocotools.cocoeval import COCOeval
|
|
|
|
from detectron2.evaluation.fast_eval_api import COCOeval_opt
|
|
|
|
|
|
class TestCOCOeval(unittest.TestCase):
|
|
def test(self):
|
|
# A small set of images/categories from COCO val
|
|
# fmt: off
|
|
detections = [{"image_id": 139, "category_id": 1, "bbox": [417.3332824707031, 159.27003479003906, 47.66064453125, 143.00193786621094], "score": 0.9949821829795837, "segmentation": {"size": [426, 640], "counts": "Tc`52W=3N0N4aNN^E7]:4XE1g:8kDMT;U100000001O1gE[Nk8h1dFiNY9Z1aFkN]9g2J3NdN`FlN`9S1cFRN07]9g1bFoM6;X9c1cFoM=8R9g1bFQN>3U9Y30O01OO1O001N2O1N1O4L4L5UNoE3V:CVF6Q:@YF9l9@ZF<k9[O`F=];HYnX2"}}, {"image_id": 139, "category_id": 1, "bbox": [383.5909118652344, 172.0777587890625, 17.959075927734375, 36.94813537597656], "score": 0.7685421705245972, "segmentation": {"size": [426, 640], "counts": "lZP5m0Z<300O100O100000001O00]OlC0T<OnCOT<OnCNX<JnC2bQT3"}}, {"image_id": 139, "category_id": 1, "bbox": [457.8359069824219, 158.88027954101562, 9.89764404296875, 8.771820068359375], "score": 0.07092753797769547, "segmentation": {"size": [426, 640], "counts": "bSo54T=2N2O1001O006ImiW2"}}] # noqa
|
|
gt_annotations = {"categories": [{"supercategory": "person", "id": 1, "name": "person"}, {"supercategory": "furniture", "id": 65, "name": "bed"}], "images": [{"license": 4, "file_name": "000000000285.jpg", "coco_url": "http://images.cocodataset.org/val2017/000000000285.jpg", "height": 640, "width": 586, "date_captured": "2013-11-18 13:09:47", "flickr_url": "http://farm8.staticflickr.com/7434/9138147604_c6225224b8_z.jpg", "id": 285}, {"license": 2, "file_name": "000000000139.jpg", "coco_url": "http://images.cocodataset.org/val2017/000000000139.jpg", "height": 426, "width": 640, "date_captured": "2013-11-21 01:34:01", "flickr_url": "http://farm9.staticflickr.com/8035/8024364858_9c41dc1666_z.jpg", "id": 139}], "annotations": [{"segmentation": [[428.19, 219.47, 430.94, 209.57, 430.39, 210.12, 421.32, 216.17, 412.8, 217.27, 413.9, 214.24, 422.42, 211.22, 429.29, 201.6, 430.67, 181.8, 430.12, 175.2, 427.09, 168.06, 426.27, 164.21, 430.94, 159.26, 440.29, 157.61, 446.06, 163.93, 448.53, 168.06, 448.53, 173.01, 449.08, 174.93, 454.03, 185.1, 455.41, 188.4, 458.43, 195.0, 460.08, 210.94, 462.28, 226.61, 460.91, 233.76, 454.31, 234.04, 460.08, 256.85, 462.56, 268.13, 465.58, 290.67, 465.85, 293.14, 463.38, 295.62, 452.66, 295.34, 448.26, 294.52, 443.59, 282.7, 446.06, 235.14, 446.34, 230.19, 438.09, 232.39, 438.09, 221.67, 434.24, 221.12, 427.09, 219.74]], "area": 2913.1103999999987, "iscrowd": 0, "image_id": 139, "bbox": [412.8, 157.61, 53.05, 138.01], "category_id": 1, "id": 230831}, {"segmentation": [[384.98, 206.58, 384.43, 199.98, 385.25, 193.66, 385.25, 190.08, 387.18, 185.13, 387.18, 182.93, 386.08, 181.01, 385.25, 178.81, 385.25, 175.79, 388.0, 172.76, 394.88, 172.21, 398.72, 173.31, 399.27, 176.06, 399.55, 183.48, 397.9, 185.68, 395.15, 188.98, 396.8, 193.38, 398.45, 194.48, 399.0, 205.75, 395.43, 207.95, 388.83, 206.03]], "area": 435.1449499999997, "iscrowd": 0, "image_id": 139, "bbox": [384.43, 172.21, 15.12, 35.74], "category_id": 1, "id": 233201}]} # noqa
|
|
# fmt: on
|
|
|
|
# Test a small dataset for typical COCO format
|
|
experiments = {"full": (detections, gt_annotations, {})}
|
|
|
|
# Test what happens if the list of detections or ground truth annotations is empty
|
|
experiments["empty_dt"] = ([], gt_annotations, {})
|
|
gt = copy.deepcopy(gt_annotations)
|
|
gt["annotations"] = []
|
|
experiments["empty_gt"] = (detections, gt, {})
|
|
|
|
# Test changing parameter settings
|
|
experiments["no_categories"] = (detections, gt_annotations, {"useCats": 0})
|
|
experiments["no_ious"] = (detections, gt_annotations, {"iouThrs": []})
|
|
experiments["no_rec_thrs"] = (detections, gt_annotations, {"recThrs": []})
|
|
experiments["no_max_dets"] = (detections, gt_annotations, {"maxDets": []})
|
|
experiments["one_max_det"] = (detections, gt_annotations, {"maxDets": [1]})
|
|
experiments["no_area"] = (detections, gt_annotations, {"areaRng": [], "areaRngLbl": []})
|
|
|
|
# Test what happens if one omits different fields from the annotation structure
|
|
annotation_fields = [
|
|
"id",
|
|
"image_id",
|
|
"category_id",
|
|
"score",
|
|
"area",
|
|
"iscrowd",
|
|
"ignore",
|
|
"bbox",
|
|
"segmentation",
|
|
]
|
|
for a in annotation_fields:
|
|
gt = copy.deepcopy(gt_annotations)
|
|
for g in gt["annotations"]:
|
|
if a in g:
|
|
del g[a]
|
|
dt = copy.deepcopy(detections)
|
|
for d in dt:
|
|
if a in d:
|
|
del d[a]
|
|
experiments["omit_gt_" + a] = (detections, gt, {})
|
|
experiments["omit_dt_" + a] = (dt, gt_annotations, {})
|
|
|
|
# Compare precision/recall for original COCO PythonAPI to custom optimized one
|
|
for name, (dt, gt, params) in experiments.items():
|
|
# Dump to json.
|
|
try:
|
|
with tempfile.TemporaryDirectory() as tmpdir:
|
|
json_file_name = os.path.join(tmpdir, "gt_" + name + ".json")
|
|
with open(json_file_name, "w") as f:
|
|
json.dump(gt, f)
|
|
with contextlib.redirect_stdout(io.StringIO()):
|
|
coco_api = COCO(json_file_name)
|
|
except Exception:
|
|
pass
|
|
|
|
for iou_type in ["bbox", "segm", "keypoints"]:
|
|
# Run original COCOeval PythonAPI
|
|
api_exception = None
|
|
try:
|
|
with contextlib.redirect_stdout(io.StringIO()):
|
|
coco_dt = coco_api.loadRes(dt)
|
|
coco_eval = COCOeval(coco_api, coco_dt, iou_type)
|
|
for p, v in params.items():
|
|
setattr(coco_eval.params, p, v)
|
|
coco_eval.evaluate()
|
|
coco_eval.accumulate()
|
|
coco_eval.summarize()
|
|
except Exception as ex:
|
|
api_exception = ex
|
|
|
|
# Run optimized COCOeval_opt API
|
|
opt_exception = None
|
|
try:
|
|
with contextlib.redirect_stdout(io.StringIO()):
|
|
coco_dt = coco_api.loadRes(dt)
|
|
coco_eval_opt = COCOeval_opt(coco_api, coco_dt, iou_type)
|
|
for p, v in params.items():
|
|
setattr(coco_eval_opt.params, p, v)
|
|
coco_eval_opt.evaluate()
|
|
coco_eval_opt.accumulate()
|
|
coco_eval_opt.summarize()
|
|
except Exception as ex:
|
|
opt_exception = ex
|
|
|
|
if api_exception is not None and opt_exception is not None:
|
|
# Original API and optimized API should throw the same exception if annotation
|
|
# format is bad
|
|
api_error = "" if api_exception is None else type(api_exception).__name__
|
|
opt_error = "" if opt_exception is None else type(opt_exception).__name__
|
|
msg = "%s: comparing COCO APIs, '%s' != '%s'" % (name, api_error, opt_error)
|
|
self.assertTrue(api_error == opt_error, msg=msg)
|
|
else:
|
|
# Original API and optimized API should produce the same precision/recalls
|
|
for k in ["precision", "recall"]:
|
|
diff = np.abs(coco_eval.eval[k] - coco_eval_opt.eval[k])
|
|
abs_diff = np.max(diff) if diff.size > 0 else 0.0
|
|
msg = "%s: comparing COCO APIs, %s differs by %f" % (name, k, abs_diff)
|
|
self.assertTrue(abs_diff < 1e-4, msg=msg)
|