mirrors
/
OWOD
mirror of https://github.com/JosephKJ/OWOD.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

- Changes to data-modelling 

- Evalualting the effect of training with known unknowns,
- Plotting energy and tsne
Nov3
Joseph 2020-11-03 16:59:41 +05:30
parent c7be064638
commit 73ade98fcf
12 changed files with 272 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
configs/Base-RCNN-C4-OWOD.yaml
View File

@ -24,7 +24,7 @@ SOLVER:
MAX_ITER: 90000
VERSION: 2
OWOD:
ENABLE_THRESHOLD_AUTOLABEL_UNK: True
ENABLE_THRESHOLD_AUTOLABEL_UNK: False
NUM_UNK_PER_IMAGE: 1
ENABLE_UNCERTAINITY_AUTOLABEL_UNK: False
ENABLE_CLUSTERING: True

11
configs/OWOD/t1.yaml
View File

@ -1,18 +1,19 @@
_BASE_: "../Base-RCNN-C4-OWOD.yaml"
MODEL:
# WEIGHTS: "detectron2://ImageNetPretrained/MSRA/R-50.pkl"
WEIGHTS: "/home/fk1/workspace/OWOD/output/t1_20_class/model_final.pth"
WEIGHTS: "detectron2://ImageNetPretrained/MSRA/R-50.pkl"
# WEIGHTS: "/home/fk1/workspace/OWOD/output/expr_training_with_unk_with_clustering_Z_DIMENSION_256/model_final.pth"
# WEIGHTS: "/home/fk1/workspace/OWOD/output/t1/model_final.pth"
DATASETS:
TRAIN: ('voc_2007_trainval', 'voc_2012_trainval')
TEST: ('voc_2007_test', )
TRAIN: ('voc_2007_trainval', 'voc_2012_trainval', 't2_test_unk')
TEST: ('voc_2007_test_unk', )
# TEST: ('t2_all_test_unk', )
# TEST: ('voc_2007_test','t2_test_unk', 't3_test_unk', 't4_test_unk')
SOLVER:
STEPS: (12000, 16000)
MAX_ITER: 18000
WARMUP_ITERS: 100
OUTPUT_DIR: "./output/t1_20_class_test"
OUTPUT_DIR: "./output/expr_training_with_unk_with_clustering_cdist_10"
OWOD:
PREV_INTRODUCED_CLS: 0
CUR_INTRODUCED_CLS: 20

1
detectron2/data/datasets/builtin.py
View File

@ -214,6 +214,7 @@ def register_all_pascal_voc(root):
("voc_2012_trainval", "VOC2012", "trainval"),
("voc_2012_train", "VOC2012", "train"),
("voc_2012_val", "VOC2012", "val"),
("voc_2007_test_unk", "VOC2007", "test_unk"),
]
for name, dirname, split in SPLITS:
year = 2007 if "2007" in name else 2012

10
detectron2/data/datasets/pascal_voc.py
View File

@ -42,6 +42,11 @@ def load_voc_instances(dirname: str, split: str, class_names: Union[List[str], T
annotation_dirname = PathManager.get_local_path(os.path.join(dirname, "Annotations/"))
dicts = []
for fileid in fileids:
has_unk = False
if 'unk' in fileid:
has_unk = True
fileid = fileid.replace('_unk','')
anno_file = os.path.join(annotation_dirname, fileid + ".xml")
jpeg_file = os.path.join(dirname, "JPEGImages", fileid + ".jpg")
@ -58,6 +63,11 @@ def load_voc_instances(dirname: str, split: str, class_names: Union[List[str], T
for obj in tree.findall("object"):
cls = obj.find("name").text
if has_unk:
if cls not in class_names:
cls = 'unknown'
# else:
# continue
# We include "difficult" samples in training.
# Based on limited experiments, they don't hurt accuracy.
# difficult = int(obj.find("difficult").text)

41
detectron2/data/datasets/voc_style_coco.py
View File

@ -19,6 +19,12 @@ VOC_CLASS_NAMES = [
"pottedplant", "sheep", "sofa", "train", "tvmonitor"
]
VOC_CLASS_NAMES_COCOFIED = [
"airplane", "bicycle", "bird", "boat", "bottle", "bus", "car", "cat",
"chair", "cow", "dining table", "dog", "horse", "motorcycle", "person",
"potted plant", "sheep", "couch", "train", "tv"
]
T2_CLASS_NAMES = [
"truck", "traffic light", "fire hydrant", "stop sign", "parking meter",
"bench", "elephant", "bear", "zebra", "giraffe",
@ -42,9 +48,11 @@ T4_CLASS_NAMES = [
UNK_CLASS = ["unknown"]
INCR_CLASS_NAMES = itertools.chain(VOC_CLASS_NAMES, T2_CLASS_NAMES, T3_CLASS_NAMES, T4_CLASS_NAMES, UNK_CLASS)
# INCR_CLASS_NAMES = itertools.chain(VOC_CLASS_NAMES, T2_CLASS_NAMES, T3_CLASS_NAMES, T4_CLASS_NAMES, UNK_CLASS)
INCR_CLASS_NAMES = itertools.chain(VOC_CLASS_NAMES, UNK_CLASS)
INCR_CLASS_NAMES = tuple(INCR_CLASS_NAMES)
INCR_CLASS_NAMES_2 = tuple(itertools.chain(VOC_CLASS_NAMES_COCOFIED, UNK_CLASS))
def load_voc_coco_instances(dirname: str, split: str, class_names: Union[List[str], Tuple[str, ...]]):
"""
@ -66,6 +74,8 @@ def load_voc_coco_instances(dirname: str, split: str, class_names: Union[List[st
elif 't4' in split:
known_class_list = T4_CLASS_NAMES
unknown_class_list = tuple(itertools.chain(T2_CLASS_NAMES, T3_CLASS_NAMES, T4_CLASS_NAMES))
# Needs to read many small annotation files. Makes sense at local
annotation_dirname = PathManager.get_local_path(os.path.join(dirname, "Annotations/"))
dicts = []
@ -87,21 +97,19 @@ def load_voc_coco_instances(dirname: str, split: str, class_names: Union[List[st
for obj in tree.findall("object"):
cls_name = obj.find("name").text
if cls_name not in known_class_list:
continue
cls = cls_name
# if 'unk' in split:g
if cls_name in unknown_class_list:
cls = "unknown"
# if cls_name not in known_class_list:
# continue
#
# if 'unk' in split:
# cls = "unknown"
# else:
# cls = cls_name
print(fileid + "--> " + str(len(fileid)))
print(type(fileid))
print('\n\n')
if len(fileid) > 10:
cls = "unknown"
else:
cls = cls_name
# We include "difficult" samples in training.
# Based on limited experiments, they don't hurt accuracy.
# difficult = int(obj.find("difficult").text)
@ -115,9 +123,14 @@ def load_voc_coco_instances(dirname: str, split: str, class_names: Union[List[st
# In coordinate space this is represented by (xmin=0, xmax=W)
bbox[0] -= 1.0
bbox[1] -= 1.0
instances.append(
{"category_id": class_names.index(cls), "bbox": bbox, "bbox_mode": BoxMode.XYXY_ABS}
)
try:
instances.append(
{"category_id": INCR_CLASS_NAMES_2.index(cls), "bbox": bbox, "bbox_mode": BoxMode.XYXY_ABS}
)
except:
print(cls)
print(class_names)
print(unknown_class_list)
r["annotations"] = instances
dicts.append(r)
return dicts

53
detectron2/evaluation/pascal_voc_evaluation.py
View File

@ -4,6 +4,7 @@
import logging
import numpy as np
import os
import sys
import tempfile
import xml.etree.ElementTree as ET
from collections import OrderedDict, defaultdict
@ -16,6 +17,7 @@ from detectron2.utils import comm
from .evaluator import DatasetEvaluator
np.set_printoptions(threshold=sys.maxsize)
class PascalVOCDetectionEvaluator(DatasetEvaluator):
"""
@ -96,7 +98,7 @@ class PascalVOCDetectionEvaluator(DatasetEvaluator):
for cls_id, cls_name in enumerate(self._class_names):
lines = predictions.get(cls_id, [""])
self._logger.info(cls_name + " has " + str(len(lines)) + " predictions.")
with open(res_file_template.format(cls_name), "w") as f:
f.write("\n".join(lines))
@ -108,6 +110,7 @@ class PascalVOCDetectionEvaluator(DatasetEvaluator):
cls_name,
ovthresh=thresh / 100.0,
use_07_metric=self._is_2007,
known_classes=self._class_names,
)
aps[thresh].append(ap * 100)
# recs[thresh].append(rec * 100)
@ -149,14 +152,25 @@ class PascalVOCDetectionEvaluator(DatasetEvaluator):
@lru_cache(maxsize=None)
def parse_rec(filename):
def parse_rec(filename, known_classes):
"""Parse a PASCAL VOC xml file."""
has_unk = False
if 'unk' in filename:
has_unk = True
filename = filename.replace('_unk', '')
with PathManager.open(filename) as f:
tree = ET.parse(f)
objects = []
for obj in tree.findall("object"):
cls = obj.find("name").text
if has_unk:
if cls not in known_classes:
cls = 'unknown'
# else:
# continue
obj_struct = {}
obj_struct["name"] = obj.find("name").text
obj_struct["name"] = cls
# obj_struct["pose"] = obj.find("pose").text
# obj_struct["truncated"] = int(obj.find("truncated").text)
obj_struct["difficult"] = int(obj.find("difficult").text)
@ -204,7 +218,7 @@ def voc_ap(rec, prec, use_07_metric=False):
return ap
def voc_eval(detpath, annopath, imagesetfile, classname, ovthresh=0.5, use_07_metric=False):
def voc_eval(detpath, annopath, imagesetfile, classname, ovthresh=0.5, use_07_metric=False, known_classes=None):
"""rec, prec, ap = voc_eval(detpath,
annopath,
imagesetfile,
@ -237,12 +251,13 @@ def voc_eval(detpath, annopath, imagesetfile, classname, ovthresh=0.5, use_07_me
# load annots
recs = {}
for imagename in imagenames:
recs[imagename] = parse_rec(annopath.format(imagename))
recs[imagename.replace('_unk', '')] = parse_rec(annopath.format(imagename), tuple(known_classes))
# extract gt objects for this class
class_recs = {}
npos = 0
for imagename in imagenames:
imagename = imagename.replace('_unk', '')
R = [obj for obj in recs[imagename] if obj["name"] == classname]
bbox = np.array([x["bbox"] for x in R])
difficult = np.array([x["difficult"] for x in R]).astype(np.bool)
@ -309,12 +324,40 @@ def voc_eval(detpath, annopath, imagesetfile, classname, ovthresh=0.5, use_07_me
fp[d] = 1.0
# compute precision recall
# if ovthresh == 0.5:
# if classname == 'unknown' or classname == 'aeroplane':
# print('\n image_ids: ')
# print(image_ids)
# print('\n FP: ')
# print(fp)
# print('\n TP: ')
# print(tp)
fp = np.cumsum(fp)
tp = np.cumsum(tp)
# if ovthresh == 0.5:
# if classname == 'unknown' or classname == 'aeroplane':
# print('\n\n FP after cumsum: ')
# print(fp)
# print('\n TP after cumsum: ')
# print(tp)
rec = tp / float(npos)
# avoid divide by zero in case the first detection matches a difficult
# ground truth
prec = tp / np.maximum(tp + fp, np.finfo(np.float64).eps)
ap = voc_ap(rec, prec, use_07_metric)
# print('\nclassname:' + classname)
# print('\n Prec:')
# print(prec)
# print('\n Recall:')
# print(rec)
# print('\n AP:')
# print(ap)
# print('\n \n')
return rec, prec, ap

7
detectron2/modeling/roi_heads/fast_rcnn.py
View File

@ -3,6 +3,7 @@ import logging
from typing import Dict, Union
import torch
import math
import shortuuid
from fvcore.nn import giou_loss, smooth_l1_loss
from torch import nn
from torch.nn import functional as F
@ -234,8 +235,14 @@ class FastRCNNOutputs:
else:
self._log_accuracy()
self.pred_class_logits[:, self.invalid_class_range] = -10e10
# self.log_logits(self.pred_class_logits, self.gt_classes)
return F.cross_entropy(self.pred_class_logits, self.gt_classes, reduction="mean")
def log_logits(self, logits, cls):
data = (logits, cls)
location = '/home/fk1/workspace/OWOD/output/logits/' + shortuuid.uuid() + '.pkl'
torch.save(data, location)
def box_reg_loss(self):
"""
Compute the smooth L1 loss for box regression.

8
detectron2/modeling/roi_heads/roi_heads.py
View File

@ -4,6 +4,7 @@ import logging
import numpy as np
import heapq
import operator
import shortuuid
from typing import Dict, List, Optional, Tuple, Union
import torch
from torch import nn
@ -436,6 +437,12 @@ class Res5ROIHeads(ROIHeads):
x = self.pooler(features, boxes)
return self.res5(x)
def log_features(self, features, proposals):
gt_classes = torch.cat([p.gt_classes for p in proposals])
data = (features, gt_classes)
location = '/home/fk1/workspace/OWOD/output/features/' + shortuuid.uuid() + '.pkl'
torch.save(data, location)
def forward(self, images, features, proposals, targets=None):
"""
See :meth:`ROIHeads.forward`.
@ -455,6 +462,7 @@ class Res5ROIHeads(ROIHeads):
predictions = self.box_predictor(input_features)
if self.training:
# self.log_features(input_features, proposals)
if self.enable_clustering:
self.box_predictor.update_feature_store(input_features, proposals)
del features

15
detectron2/utils/store.py
View File

@ -1,6 +1,6 @@
import random
from collections import deque
import numpy as np
class Store:
def __init__(self, total_num_classes, items_per_class, shuffle=False):
@ -55,13 +55,8 @@ if __name__ == "__main__":
# print(store.retrieve(3))
# print(store.retrieve(9))
print(store.retrieve(-1))
print(len(store))
store.reset()
print(len(store))
# print(len(store))
# store.reset()
# print(len(store))
means = [None for i in range(10)]
print(means)
means[0] = 100
print(means)
print(means[1])
print(means[0])
print(store)

33
tools/plot_energy.py 100644
View File

@ -0,0 +1,33 @@
import os
import torch
import pickle
source_dir = '/home/fk1/workspace/OWOD/output/logits'
files = os.listdir(source_dir)
unk = []
known = []
for file in files:
path = os.path.join(source_dir, file)
logits, classes = torch.load(path)
lse = torch.logsumexp(logits[:,:-2], dim=1)
for i, cls in enumerate(classes):
if cls == 21:
continue
if cls == 20:
unk.append(lse[i].detach().cpu().tolist())
else:
known.append(lse[i].detach().cpu().tolist())
print(known)
print('\n\n')
print(unk)
# dir = '/home/fk1/workspace/OWOD/output'
#
# with open(os.path.join(dir, 'unk.pkl'), 'wb') as f:
# pickle.dump(unk, f)
#
# with open(os.path.join(dir, 'known.pkl'), 'wb') as f:
# pickle.dump(unk, f)

70
tools/plot_tsne.py 100644
View File

@ -0,0 +1,70 @@
import os
import torch
import matplotlib.pyplot as plt
from matplotlib.ticker import NullFormatter
from sklearn import manifold, datasets
from time import time
from collections import deque
import seaborn as sns
import numpy as np
from detectron2.utils.store import Store
def plot_tsne(X, label, total_num_classes):
n_components = 2
(fig, subplots) = plt.subplots(1, 5, figsize=(15, 8))
perplexities = [5, 30, 50, 100, 150]
for i, perplexity in enumerate(perplexities):
ax = subplots[i]
t0 = time()
tsne = manifold.TSNE(n_components=n_components, init='random',
random_state=0, perplexity=perplexity)
Y = tsne.fit_transform(X)
t1 = time()
print("circles, perplexity=%d in %.2g sec" % (perplexity, t1 - t0))
ax.set_title("Perplexity=%d" % perplexity)
#
# sc = ax.scatter(Y[:, 0], Y[:, 1], c=label, cmap="plasma")
sns.scatterplot(x=Y[:, 0], y=Y[:, 1], hue=label, ax=ax, legend='full', palette='colorblind')
ax.xaxis.set_major_formatter(NullFormatter())
ax.yaxis.set_major_formatter(NullFormatter())
ax.axis('tight')
# plt.legend(handles=sc.legend_elements()[0], labels=range(total_num_classes))
# plt.legend(handles=sc.legend_elements()[0], labels=['0', '1'])
# plt.show()
plt.savefig('tsne.png')
maxlen_queue = 100
total_num_classes = 22
queues = [deque(maxlen=maxlen_queue) for _ in range(total_num_classes)]
source_dir = '/home/fk1/workspace/OWOD/output/features'
files = os.listdir(source_dir)
for i, file in enumerate(files):
path = os.path.join(source_dir, file)
features, classes = torch.load(path)
for f, c in zip(features, classes):
queues[c.detach().cpu().numpy()].append(f.detach().cpu().numpy())
# if i == 2:
# break
x = []
y = []
for i, queue in enumerate(queues):
for item in queue:
x.append(item)
y.append(i)
print('Going to plot')
plot_tsne(x, y, total_num_classes)

56
tools/tsne.py 100644
View File

@ -0,0 +1,56 @@
# Author: Narine Kokhlikyan <narine@slice.com>
# License: BSD
import matplotlib.pyplot as plt
from matplotlib.ticker import NullFormatter
from sklearn import manifold, datasets
from time import time
import numpy as np
def plot_tsne(X, label):
n_components = 2
(fig, subplots) = plt.subplots(1, 5, figsize=(15, 8))
perplexities = [5, 30, 50, 100, 150]
for i, perplexity in enumerate(perplexities):
ax = subplots[i]
t0 = time()
tsne = manifold.TSNE(n_components=n_components, init='random',
random_state=0, perplexity=perplexity)
Y = tsne.fit_transform(X)
t1 = time()
print("circles, perplexity=%d in %.2g sec" % (perplexity, t1 - t0))
ax.set_title("Perplexity=%d" % perplexity)
sc = ax.scatter(Y[:, 0], Y[:, 1], c=label, cmap="Set1")
ax.xaxis.set_major_formatter(NullFormatter())
ax.yaxis.set_major_formatter(NullFormatter())
ax.axis('tight')
plt.legend(handles=sc.legend_elements()[0], labels=['0', '1'])
# plt.show()
plt.savefig('tsne.png')
n_samples = 300
# X, y = datasets.make_circles(n_samples=n_samples, factor=.5, noise=.05)
# plot_tsne(X, y)
num_samples_from_prior = 10
num_tasks = 10
X = []
color = []
label = []
for i in range(num_tasks):
for p in range(num_samples_from_prior):
prior = np.random.rand(1000)
X.append(prior)
color.append('C'+str(i))
label.append(i%2)
plot_tsne(np.array(X), np.array(color), label)