From 81516e554360b407ecb650bc8328451860cfba0e Mon Sep 17 00:00:00 2001 From: Joseph Date: Mon, 9 Nov 2020 10:34:45 +0530 Subject: [PATCH] - Saving feature store - Implemented Absolute OSE error and initial WI --- configs/Base-RCNN-C4-OWOD.yaml | 1 + configs/OWOD/t1/t1_test.yaml | 18 +- configs/OWOD/t1/t1_train.yaml | 5 +- configs/OWOD/{t2.yaml => t2/t2_train.yaml} | 11 +- datasets/OWOD_imagesets/t1_known_test.txt | 4952 +++++++++++++++++ detectron2/config/defaults.py | 1 + detectron2/data/datasets/builtin.py | 1 + detectron2/engine/defaults.py | 3 + .../evaluation/pascal_voc_evaluation.py | 210 +- detectron2/modeling/roi_heads/fast_rcnn.py | 168 +- 10 files changed, 5262 insertions(+), 108 deletions(-) rename configs/OWOD/{t2.yaml => t2/t2_train.yaml} (54%) create mode 100644 datasets/OWOD_imagesets/t1_known_test.txt diff --git a/configs/Base-RCNN-C4-OWOD.yaml b/configs/Base-RCNN-C4-OWOD.yaml index 2ba5110..ba178e4 100644 --- a/configs/Base-RCNN-C4-OWOD.yaml +++ b/configs/Base-RCNN-C4-OWOD.yaml @@ -28,6 +28,7 @@ OWOD: NUM_UNK_PER_IMAGE: 1 ENABLE_UNCERTAINITY_AUTOLABEL_UNK: False ENABLE_CLUSTERING: True + FEATURE_STORE_SAVE_PATH: 'feature_store' CLUSTERING: ITEMS_PER_CLASS: 20 START_ITER: 1000 diff --git a/configs/OWOD/t1/t1_test.yaml b/configs/OWOD/t1/t1_test.yaml index 2489903..ddb69cf 100644 --- a/configs/OWOD/t1/t1_test.yaml +++ b/configs/OWOD/t1/t1_test.yaml @@ -1,10 +1,18 @@ _BASE_: "../../Base-RCNN-C4-OWOD.yaml" MODEL: # WEIGHTS: "detectron2://ImageNetPretrained/MSRA/R-50.pkl" - WEIGHTS: "/home/fk1/workspace/OWOD/output/t1_ENABLE_CLUSTERING/model_final.pth" + WEIGHTS: "/home/fk1/workspace/OWOD/output/t1_std_frcnn/model_final.pth" +# WEIGHTS: "/home/fk1/workspace/OWOD/output/t1_THRESHOLD_AUTOLABEL_UNK/model_final.pth" +# WEIGHTS: "/home/fk1/workspace/OWOD/output/t1_ENABLE_CLUSTERING/model_final.pth" + ROI_HEADS: + POSITIVE_FRACTION: 0.25 + NMS_THRESH_TEST: 0.5 + SCORE_THRESH_TEST: 0.05 +TEST: + DETECTIONS_PER_IMAGE: 50 DATASETS: TRAIN: ('t1_voc_coco_2007_train', ) # t1_voc_coco_2007_train, t1_voc_coco_2007_ft - TEST: ('voc_coco_2007_test', ) # voc_coco_2007_test + TEST: ('voc_coco_2007_test', 't1_voc_coco_2007_known_test') # voc_coco_2007_test SOLVER: STEPS: (12000, 16000) MAX_ITER: 18000 @@ -12,4 +20,8 @@ SOLVER: OUTPUT_DIR: "./output/t1_clustering_new_4" OWOD: PREV_INTRODUCED_CLS: 0 - CUR_INTRODUCED_CLS: 20 \ No newline at end of file + CUR_INTRODUCED_CLS: 20 + +# POSITIVE_FRACTION: 0.25 +# NMS_THRESH_TEST: 0.5 +# SCORE_THRESH_TEST: 0.05 \ No newline at end of file diff --git a/configs/OWOD/t1/t1_train.yaml b/configs/OWOD/t1/t1_train.yaml index 3e65a34..552f6fe 100644 --- a/configs/OWOD/t1/t1_train.yaml +++ b/configs/OWOD/t1/t1_train.yaml @@ -2,14 +2,15 @@ _BASE_: "../../Base-RCNN-C4-OWOD.yaml" MODEL: WEIGHTS: "detectron2://ImageNetPretrained/MSRA/R-50.pkl" # WEIGHTS: "/home/fk1/workspace/OWOD/output/t1/model_final.pth" +# WEIGHTS: "/home/fk1/workspace/OWOD/output/t1_std_frcnn/model_final.pth" DATASETS: TRAIN: ('t1_voc_coco_2007_train', ) # t1_voc_coco_2007_train, t1_voc_coco_2007_ft - TEST: (, ) # voc_coco_2007_test + TEST: ('voc_coco_2007_test', ) # voc_coco_2007_test, t1_voc_coco_2007_test, t1_voc_coco_2007_val SOLVER: STEPS: (12000, 16000) MAX_ITER: 18000 WARMUP_ITERS: 100 -OUTPUT_DIR: "./output/t1_expr" +OUTPUT_DIR: "./output/t1_clustering_with_save" OWOD: PREV_INTRODUCED_CLS: 0 CUR_INTRODUCED_CLS: 20 \ No newline at end of file diff --git a/configs/OWOD/t2.yaml b/configs/OWOD/t2/t2_train.yaml similarity index 54% rename from configs/OWOD/t2.yaml rename to configs/OWOD/t2/t2_train.yaml index 1a23333..76b2312 100644 --- a/configs/OWOD/t2.yaml +++ b/configs/OWOD/t2/t2_train.yaml @@ -1,7 +1,8 @@ _BASE_: "../Base-RCNN-C4-OWOD.yaml" MODEL: # WEIGHTS: "detectron2://ImageNetPretrained/MSRA/R-50.pkl" - WEIGHTS: "/home/fk1/workspace/OWOD/output/t1_ENABLE_CLUSTERING/model_final.pth" +# WEIGHTS: "/home/fk1/workspace/OWOD/output/t1_ENABLE_CLUSTERING/model_final.pth" + WEIGHTS: "/home/fk1/workspace/OWOD/output/t2/model_final.pth" DATASETS: TRAIN: ('t2_voc_coco_2007_train', ) # t1_voc_coco_2007_train, t1_voc_coco_2007_ft TEST: ('voc_coco_2007_test', ) @@ -12,4 +13,10 @@ SOLVER: OUTPUT_DIR: "./output/t2" OWOD: PREV_INTRODUCED_CLS: 20 - CUR_INTRODUCED_CLS: 20 \ No newline at end of file + CUR_INTRODUCED_CLS: 20 + CLUSTERING: + ITEMS_PER_CLASS: 20 + START_ITER: 20000 + UPDATE_MU_ITER: 3000 + MOMENTUM: 0.99 + Z_DIMENSION: 128 \ No newline at end of file diff --git a/datasets/OWOD_imagesets/t1_known_test.txt b/datasets/OWOD_imagesets/t1_known_test.txt new file mode 100644 index 0000000..30177c1 --- /dev/null +++ b/datasets/OWOD_imagesets/t1_known_test.txt @@ -0,0 +1,4952 @@ +000001 +000002 +000003 +000004 +000006 +000008 +000010 +000011 +000013 +000014 +000015 +000018 +000022 +000025 +000027 +000028 +000029 +000031 +000037 +000038 +000040 +000043 +000045 +000049 +000053 +000054 +000055 +000056 +000057 +000058 +000059 +000062 +000067 +000068 +000069 +000070 +000071 +000074 +000075 +000076 +000079 +000080 +000082 +000084 +000085 +000086 +000087 +000088 +000090 +000092 +000094 +000096 +000097 +000098 +000100 +000103 +000105 +000106 +000108 +000111 +000114 +000115 +000116 +000119 +000124 +000126 +000127 +000128 +000135 +000136 +000137 +000139 +000144 +000145 +000148 +000149 +000151 +000152 +000155 +000157 +000160 +000166 +000167 +000168 +000172 +000175 +000176 +000178 +000179 +000181 +000182 +000183 +000185 +000186 +000188 +000191 +000195 +000196 +000197 +000199 +000201 +000202 +000204 +000205 +000206 +000212 +000213 +000216 +000217 +000223 +000226 +000227 +000230 +000231 +000234 +000237 +000238 +000239 +000240 +000243 +000247 +000248 +000252 +000253 +000254 +000255 +000258 +000260 +000261 +000264 +000265 +000267 +000271 +000272 +000273 +000274 +000277 +000279 +000280 +000281 +000283 +000284 +000286 +000287 +000290 +000291 +000292 +000293 +000295 +000297 +000299 +000300 +000301 +000309 +000310 +000313 +000314 +000315 +000316 +000319 +000324 +000326 +000327 +000330 +000333 +000335 +000339 +000341 +000342 +000345 +000346 +000348 +000350 +000351 +000353 +000356 +000357 +000358 +000360 +000361 +000362 +000364 +000365 +000366 +000368 +000369 +000371 +000375 +000376 +000377 +000378 +000383 +000384 +000385 +000386 +000388 +000389 +000390 +000392 +000393 +000397 +000398 +000399 +000401 +000402 +000405 +000409 +000410 +000412 +000413 +000414 +000415 +000418 +000421 +000422 +000423 +000425 +000426 +000429 +000432 +000434 +000436 +000437 +000440 +000441 +000442 +000444 +000445 +000447 +000449 +000451 +000452 +000453 +000455 +000456 +000457 +000458 +000465 +000466 +000467 +000471 +000472 +000473 +000475 +000478 +000479 +000481 +000485 +000487 +000488 +000490 +000493 +000495 +000497 +000502 +000504 +000505 +000506 +000507 +000510 +000511 +000512 +000517 +000521 +000527 +000529 +000532 +000533 +000534 +000536 +000538 +000539 +000542 +000546 +000547 +000548 +000551 +000553 +000556 +000557 +000558 +000560 +000561 +000562 +000566 +000567 +000568 +000569 +000570 +000571 +000572 +000573 +000574 +000575 +000576 +000578 +000580 +000584 +000585 +000586 +000587 +000593 +000594 +000595 +000596 +000600 +000602 +000603 +000604 +000606 +000607 +000611 +000614 +000615 +000616 +000617 +000618 +000621 +000623 +000624 +000627 +000629 +000630 +000631 +000634 +000636 +000638 +000639 +000640 +000641 +000642 +000643 +000644 +000646 +000649 +000650 +000651 +000652 +000655 +000658 +000659 +000662 +000664 +000665 +000666 +000668 +000669 +000670 +000673 +000674 +000678 +000679 +000681 +000683 +000687 +000691 +000692 +000693 +000696 +000697 +000698 +000701 +000703 +000704 +000706 +000708 +000715 +000716 +000718 +000719 +000721 +000722 +000723 +000724 +000725 +000727 +000732 +000734 +000735 +000736 +000737 +000741 +000743 +000744 +000745 +000747 +000749 +000751 +000757 +000758 +000759 +000762 +000765 +000766 +000769 +000773 +000775 +000778 +000779 +000781 +000783 +000784 +000785 +000788 +000789 +000790 +000792 +000795 +000798 +000801 +000803 +000807 +000809 +000811 +000813 +000817 +000819 +000821 +000824 +000825 +000833 +000835 +000836 +000837 +000838 +000839 +000840 +000841 +000844 +000846 +000852 +000853 +000856 +000858 +000861 +000864 +000866 +000869 +000870 +000873 +000875 +000877 +000881 +000883 +000884 +000886 +000890 +000891 +000893 +000894 +000897 +000901 +000905 +000907 +000909 +000910 +000913 +000914 +000916 +000922 +000924 +000925 +000927 +000928 +000930 +000932 +000933 +000938 +000939 +000940 +000941 +000942 +000944 +000945 +000952 +000953 +000955 +000956 +000957 +000959 +000960 +000961 +000963 +000968 +000969 +000970 +000974 +000975 +000976 +000978 +000979 +000981 +000983 +000984 +000985 +000986 +000988 +000990 +000992 +000994 +000995 +000998 +001000 +001003 +001005 +001006 +001007 +001013 +001016 +001019 +001020 +001021 +001022 +001023 +001025 +001026 +001029 +001030 +001031 +001032 +001033 +001034 +001035 +001037 +001038 +001039 +001040 +001044 +001046 +001047 +001048 +001049 +001051 +001054 +001055 +001058 +001059 +001063 +001065 +001067 +001070 +001075 +001076 +001080 +001081 +001085 +001086 +001087 +001088 +001089 +001090 +001094 +001095 +001096 +001098 +001099 +001100 +001103 +001105 +001108 +001111 +001114 +001115 +001116 +001117 +001118 +001120 +001122 +001123 +001126 +001128 +001131 +001132 +001133 +001134 +001135 +001138 +001139 +001141 +001146 +001150 +001153 +001155 +001157 +001159 +001162 +001163 +001165 +001167 +001169 +001173 +001177 +001178 +001179 +001180 +001181 +001183 +001188 +001189 +001190 +001193 +001195 +001196 +001197 +001198 +001202 +001208 +001210 +001213 +001216 +001217 +001218 +001219 +001220 +001222 +001223 +001227 +001228 +001232 +001235 +001238 +001242 +001243 +001244 +001245 +001246 +001249 +001251 +001252 +001253 +001255 +001256 +001257 +001261 +001262 +001264 +001267 +001271 +001275 +001276 +001278 +001280 +001282 +001283 +001285 +001291 +001295 +001296 +001297 +001300 +001301 +001302 +001303 +001305 +001306 +001307 +001308 +001313 +001317 +001318 +001319 +001320 +001321 +001322 +001328 +001329 +001331 +001335 +001336 +001338 +001339 +001340 +001342 +001344 +001347 +001349 +001351 +001353 +001354 +001355 +001356 +001357 +001358 +001359 +001363 +001366 +001367 +001368 +001369 +001370 +001372 +001373 +001374 +001376 +001377 +001379 +001380 +001381 +001382 +001389 +001391 +001392 +001394 +001396 +001398 +001399 +001401 +001403 +001407 +001410 +001411 +001412 +001415 +001416 +001417 +001419 +001422 +001423 +001424 +001425 +001428 +001429 +001431 +001433 +001435 +001437 +001438 +001440 +001446 +001447 +001448 +001449 +001452 +001454 +001456 +001458 +001459 +001461 +001462 +001469 +001471 +001473 +001474 +001476 +001477 +001478 +001482 +001487 +001489 +001491 +001495 +001496 +001500 +001502 +001503 +001505 +001506 +001507 +001508 +001511 +001513 +001516 +001518 +001519 +001520 +001525 +001527 +001530 +001533 +001534 +001535 +001538 +001540 +001542 +001546 +001547 +001549 +001550 +001551 +001552 +001558 +001560 +001562 +001564 +001566 +001567 +001568 +001569 +001570 +001572 +001573 +001574 +001575 +001578 +001581 +001583 +001584 +001585 +001587 +001589 +001591 +001592 +001596 +001599 +001600 +001601 +001602 +001605 +001606 +001609 +001613 +001615 +001616 +001619 +001620 +001621 +001623 +001624 +001625 +001626 +001629 +001631 +001634 +001635 +001637 +001639 +001641 +001644 +001645 +001646 +001648 +001652 +001655 +001656 +001657 +001658 +001659 +001660 +001663 +001664 +001665 +001666 +001667 +001668 +001670 +001671 +001672 +001674 +001679 +001681 +001687 +001692 +001694 +001695 +001696 +001697 +001698 +001700 +001701 +001702 +001703 +001704 +001705 +001706 +001709 +001710 +001712 +001715 +001716 +001719 +001720 +001722 +001728 +001731 +001735 +001736 +001737 +001740 +001742 +001743 +001744 +001745 +001748 +001751 +001753 +001757 +001760 +001762 +001763 +001764 +001767 +001769 +001770 +001773 +001774 +001776 +001779 +001781 +001783 +001786 +001788 +001790 +001791 +001792 +001794 +001796 +001798 +001802 +001803 +001804 +001805 +001808 +001811 +001812 +001813 +001814 +001815 +001817 +001819 +001820 +001822 +001823 +001824 +001826 +001829 +001831 +001835 +001838 +001839 +001844 +001846 +001848 +001850 +001851 +001852 +001856 +001857 +001859 +001863 +001865 +001866 +001867 +001868 +001869 +001871 +001873 +001874 +001876 +001879 +001880 +001883 +001884 +001885 +001886 +001889 +001890 +001891 +001893 +001895 +001897 +001900 +001905 +001908 +001909 +001910 +001912 +001913 +001914 +001916 +001917 +001919 +001921 +001923 +001924 +001925 +001926 +001929 +001935 +001939 +001942 +001943 +001946 +001947 +001949 +001951 +001953 +001955 +001956 +001957 +001959 +001961 +001965 +001966 +001967 +001968 +001969 +001973 +001974 +001975 +001979 +001983 +001984 +001986 +001987 +001988 +001990 +001991 +001992 +001993 +001994 +001996 +001997 +001998 +002003 +002005 +002007 +002008 +002009 +002010 +002013 +002014 +002016 +002017 +002018 +002026 +002028 +002029 +002031 +002032 +002033 +002035 +002038 +002040 +002041 +002044 +002046 +002048 +002050 +002052 +002053 +002057 +002059 +002060 +002062 +002065 +002066 +002071 +002072 +002073 +002074 +002075 +002076 +002077 +002078 +002079 +002080 +002081 +002084 +002085 +002087 +002089 +002092 +002093 +002097 +002100 +002103 +002105 +002106 +002107 +002110 +002111 +002113 +002115 +002118 +002119 +002121 +002122 +002123 +002127 +002128 +002130 +002131 +002133 +002137 +002138 +002141 +002143 +002144 +002147 +002148 +002149 +002150 +002154 +002157 +002159 +002160 +002161 +002162 +002164 +002167 +002168 +002173 +002175 +002177 +002185 +002188 +002189 +002195 +002198 +002200 +002203 +002204 +002205 +002206 +002207 +002210 +002211 +002216 +002217 +002222 +002223 +002225 +002227 +002229 +002230 +002231 +002232 +002235 +002236 +002239 +002240 +002242 +002243 +002245 +002246 +002250 +002252 +002254 +002258 +002262 +002264 +002269 +002271 +002274 +002275 +002282 +002283 +002286 +002289 +002292 +002294 +002295 +002296 +002297 +002298 +002299 +002301 +002303 +002304 +002309 +002312 +002313 +002314 +002316 +002317 +002319 +002322 +002325 +002326 +002327 +002331 +002336 +002338 +002339 +002341 +002344 +002346 +002349 +002351 +002353 +002356 +002357 +002358 +002360 +002363 +002365 +002370 +002379 +002380 +002381 +002383 +002386 +002388 +002389 +002390 +002394 +002395 +002396 +002397 +002398 +002399 +002400 +002402 +002406 +002408 +002409 +002412 +002414 +002416 +002418 +002421 +002422 +002424 +002426 +002428 +002429 +002430 +002431 +002432 +002434 +002438 +002440 +002446 +002447 +002449 +002451 +002453 +002455 +002457 +002463 +002464 +002467 +002469 +002473 +002474 +002475 +002482 +002484 +002485 +002486 +002487 +002488 +002489 +002495 +002498 +002499 +002503 +002506 +002507 +002509 +002510 +002511 +002515 +002516 +002517 +002521 +002522 +002526 +002527 +002528 +002530 +002531 +002532 +002535 +002536 +002538 +002541 +002543 +002548 +002550 +002551 +002552 +002553 +002556 +002557 +002560 +002562 +002568 +002570 +002573 +002574 +002575 +002576 +002577 +002580 +002581 +002582 +002583 +002587 +002588 +002591 +002592 +002596 +002597 +002601 +002602 +002604 +002607 +002608 +002610 +002612 +002614 +002616 +002617 +002619 +002620 +002622 +002623 +002624 +002626 +002628 +002629 +002630 +002631 +002638 +002639 +002640 +002642 +002644 +002650 +002651 +002652 +002654 +002655 +002656 +002660 +002661 +002663 +002665 +002671 +002672 +002673 +002674 +002676 +002679 +002681 +002685 +002686 +002687 +002688 +002692 +002694 +002698 +002700 +002701 +002703 +002705 +002707 +002708 +002711 +002712 +002716 +002719 +002720 +002724 +002725 +002726 +002728 +002729 +002731 +002733 +002736 +002739 +002740 +002742 +002743 +002746 +002748 +002750 +002752 +002753 +002754 +002756 +002758 +002761 +002764 +002768 +002769 +002770 +002771 +002773 +002777 +002780 +002781 +002787 +002788 +002789 +002790 +002792 +002793 +002797 +002799 +002802 +002805 +002806 +002808 +002809 +002811 +002813 +002814 +002818 +002819 +002821 +002822 +002823 +002824 +002825 +002828 +002829 +002830 +002831 +002832 +002837 +002839 +002840 +002843 +002846 +002849 +002850 +002851 +002852 +002853 +002856 +002857 +002860 +002861 +002862 +002863 +002865 +002871 +002872 +002874 +002876 +002877 +002878 +002882 +002883 +002885 +002887 +002888 +002890 +002892 +002894 +002895 +002897 +002898 +002900 +002902 +002903 +002904 +002905 +002907 +002908 +002909 +002911 +002918 +002920 +002921 +002922 +002923 +002925 +002926 +002927 +002928 +002929 +002930 +002936 +002945 +002948 +002949 +002950 +002951 +002955 +002959 +002961 +002964 +002968 +002970 +002971 +002972 +002973 +002974 +002979 +002980 +002981 +002982 +002983 +002985 +002991 +002993 +002996 +002997 +002998 +002999 +003001 +003006 +003010 +003012 +003014 +003016 +003018 +003019 +003020 +003022 +003025 +003026 +003029 +003030 +003033 +003035 +003036 +003037 +003040 +003041 +003043 +003046 +003048 +003049 +003050 +003052 +003055 +003059 +003060 +003062 +003067 +003068 +003069 +003070 +003071 +003073 +003075 +003076 +003079 +003080 +003081 +003084 +003087 +003091 +003095 +003096 +003097 +003099 +003101 +003104 +003109 +003111 +003113 +003114 +003115 +003119 +003123 +003125 +003128 +003130 +003131 +003132 +003136 +003139 +003141 +003143 +003144 +003148 +003151 +003152 +003153 +003156 +003158 +003160 +003166 +003167 +003168 +003171 +003172 +003173 +003174 +003179 +003180 +003182 +003187 +003190 +003191 +003192 +003193 +003196 +003197 +003198 +003201 +003203 +003206 +003208 +003209 +003212 +003215 +003217 +003220 +003221 +003222 +003224 +003225 +003226 +003227 +003230 +003232 +003234 +003235 +003237 +003238 +003241 +003245 +003246 +003248 +003249 +003251 +003252 +003257 +003263 +003264 +003265 +003266 +003267 +003268 +003275 +003276 +003277 +003278 +003281 +003283 +003286 +003287 +003288 +003289 +003291 +003295 +003297 +003298 +003302 +003304 +003305 +003306 +003309 +003310 +003312 +003314 +003315 +003317 +003318 +003319 +003321 +003322 +003323 +003324 +003326 +003328 +003329 +003332 +003333 +003334 +003340 +003341 +003342 +003345 +003346 +003347 +003348 +003352 +003353 +003357 +003358 +003361 +003364 +003366 +003368 +003371 +003372 +003375 +003378 +003381 +003383 +003384 +003385 +003387 +003388 +003389 +003393 +003394 +003399 +003400 +003402 +003405 +003409 +003411 +003414 +003418 +003423 +003426 +003427 +003428 +003431 +003432 +003434 +003437 +003438 +003440 +003442 +003445 +003446 +003447 +003448 +003454 +003456 +003457 +003459 +003460 +003463 +003467 +003471 +003472 +003473 +003474 +003475 +003476 +003478 +003479 +003480 +003481 +003482 +003483 +003485 +003486 +003488 +003490 +003494 +003495 +003498 +003501 +003502 +003503 +003504 +003505 +003507 +003512 +003513 +003514 +003515 +003517 +003520 +003523 +003526 +003527 +003531 +003532 +003533 +003534 +003535 +003538 +003540 +003541 +003542 +003543 +003544 +003545 +003547 +003552 +003553 +003557 +003558 +003559 +003560 +003561 +003562 +003563 +003568 +003569 +003570 +003571 +003572 +003573 +003574 +003578 +003579 +003581 +003582 +003583 +003584 +003590 +003591 +003592 +003595 +003598 +003600 +003601 +003602 +003607 +003610 +003612 +003613 +003615 +003616 +003617 +003619 +003624 +003626 +003630 +003631 +003633 +003637 +003641 +003643 +003647 +003649 +003650 +003652 +003653 +003659 +003661 +003665 +003666 +003668 +003670 +003672 +003676 +003677 +003680 +003682 +003683 +003686 +003687 +003689 +003692 +003693 +003697 +003701 +003702 +003707 +003710 +003712 +003715 +003716 +003718 +003719 +003720 +003723 +003724 +003725 +003726 +003728 +003730 +003731 +003733 +003734 +003736 +003737 +003738 +003739 +003741 +003742 +003744 +003745 +003746 +003747 +003755 +003756 +003757 +003761 +003762 +003764 +003765 +003766 +003768 +003769 +003770 +003771 +003775 +003776 +003777 +003778 +003782 +003785 +003787 +003789 +003794 +003795 +003799 +003800 +003801 +003802 +003804 +003805 +003810 +003812 +003813 +003815 +003816 +003819 +003822 +003823 +003825 +003829 +003831 +003832 +003833 +003836 +003839 +003840 +003841 +003842 +003843 +003850 +003851 +003852 +003853 +003854 +003858 +003862 +003864 +003867 +003870 +003873 +003875 +003878 +003880 +003881 +003882 +003883 +003884 +003888 +003892 +003893 +003894 +003896 +003897 +003900 +003901 +003902 +003903 +003904 +003906 +003908 +003909 +003910 +003914 +003916 +003917 +003920 +003922 +003925 +003927 +003928 +003929 +003930 +003931 +003933 +003934 +003938 +003940 +003942 +003943 +003944 +003950 +003951 +003952 +003955 +003958 +003959 +003962 +003964 +003967 +003968 +003972 +003975 +003976 +003977 +003978 +003980 +003981 +003982 +003985 +003989 +003995 +003999 +004000 +004001 +004002 +004004 +004006 +004007 +004018 +004021 +004022 +004024 +004026 +004027 +004029 +004030 +004032 +004036 +004038 +004040 +004041 +004042 +004043 +004044 +004045 +004048 +004049 +004050 +004053 +004054 +004055 +004056 +004059 +004061 +004062 +004063 +004064 +004065 +004068 +004070 +004071 +004072 +004074 +004078 +004079 +004080 +004081 +004083 +004084 +004086 +004088 +004090 +004094 +004096 +004097 +004098 +004099 +004101 +004103 +004104 +004107 +004109 +004112 +004114 +004115 +004116 +004118 +004119 +004123 +004124 +004125 +004126 +004127 +004128 +004130 +004132 +004134 +004139 +004144 +004147 +004151 +004153 +004154 +004155 +004156 +004157 +004159 +004160 +004161 +004162 +004165 +004166 +004167 +004172 +004173 +004175 +004176 +004177 +004179 +004180 +004181 +004182 +004183 +004184 +004187 +004188 +004197 +004198 +004199 +004202 +004206 +004207 +004208 +004210 +004211 +004213 +004214 +004216 +004217 +004218 +004219 +004222 +004225 +004226 +004227 +004233 +004234 +004235 +004236 +004238 +004240 +004243 +004245 +004248 +004249 +004250 +004251 +004252 +004254 +004260 +004261 +004262 +004266 +004267 +004268 +004276 +004277 +004278 +004282 +004285 +004288 +004289 +004290 +004294 +004297 +004299 +004301 +004302 +004305 +004306 +004308 +004309 +004311 +004313 +004314 +004316 +004317 +004319 +004320 +004324 +004328 +004330 +004332 +004334 +004335 +004336 +004337 +004340 +004342 +004343 +004344 +004348 +004350 +004353 +004355 +004357 +004358 +004362 +004363 +004366 +004373 +004374 +004375 +004377 +004378 +004381 +004382 +004383 +004385 +004388 +004393 +004394 +004395 +004398 +004399 +004400 +004401 +004402 +004403 +004406 +004407 +004408 +004410 +004412 +004413 +004414 +004415 +004416 +004417 +004418 +004419 +004420 +004422 +004425 +004426 +004427 +004428 +004431 +004435 +004440 +004442 +004443 +004444 +004445 +004447 +004448 +004449 +004451 +004453 +004454 +004456 +004458 +004460 +004461 +004462 +004465 +004467 +004469 +004472 +004473 +004475 +004476 +004477 +004478 +004480 +004482 +004483 +004485 +004486 +004489 +004491 +004492 +004497 +004501 +004503 +004504 +004505 +004506 +004511 +004513 +004515 +004516 +004521 +004522 +004523 +004525 +004529 +004531 +004533 +004534 +004536 +004538 +004541 +004543 +004545 +004546 +004547 +004550 +004554 +004556 +004557 +004559 +004560 +004561 +004564 +004567 +004568 +004569 +004572 +004573 +004575 +004577 +004578 +004580 +004582 +004583 +004586 +004589 +004590 +004593 +004594 +004596 +004598 +004599 +004602 +004603 +004608 +004610 +004613 +004614 +004615 +004616 +004617 +004619 +004620 +004621 +004624 +004629 +004633 +004635 +004637 +004638 +004639 +004640 +004641 +004642 +004645 +004646 +004650 +004657 +004658 +004659 +004661 +004663 +004664 +004665 +004666 +004667 +004668 +004669 +004670 +004677 +004678 +004680 +004681 +004684 +004688 +004690 +004695 +004696 +004697 +004698 +004700 +004703 +004704 +004709 +004711 +004712 +004713 +004716 +004717 +004720 +004721 +004724 +004725 +004726 +004728 +004729 +004730 +004731 +004733 +004734 +004736 +004738 +004739 +004740 +004741 +004744 +004745 +004749 +004751 +004752 +004755 +004756 +004757 +004758 +004759 +004762 +004763 +004764 +004765 +004766 +004767 +004769 +004771 +004772 +004774 +004775 +004778 +004780 +004781 +004784 +004787 +004791 +004795 +004798 +004800 +004802 +004803 +004804 +004806 +004807 +004809 +004810 +004811 +004813 +004817 +004819 +004820 +004821 +004822 +004824 +004827 +004829 +004833 +004835 +004838 +004843 +004844 +004845 +004847 +004851 +004853 +004854 +004855 +004858 +004860 +004861 +004862 +004864 +004865 +004870 +004871 +004874 +004875 +004877 +004880 +004881 +004883 +004884 +004887 +004888 +004889 +004891 +004892 +004893 +004894 +004899 +004900 +004901 +004904 +004906 +004908 +004909 +004914 +004915 +004917 +004918 +004919 +004920 +004921 +004922 +004923 +004924 +004925 +004927 +004930 +004932 +004933 +004934 +004937 +004940 +004941 +004942 +004944 +004945 +004947 +004949 +004952 +004957 +004959 +004964 +004965 +004969 +004970 +004971 +004975 +004978 +004979 +004980 +004981 +004988 +004989 +004993 +004996 +005000 +005002 +005005 +005008 +005009 +005010 +005011 +005012 +005013 +005015 +005017 +005019 +005021 +005022 +005025 +005030 +005031 +005034 +005035 +005038 +005040 +005041 +005043 +005044 +005046 +005048 +005049 +005050 +005051 +005053 +005059 +005060 +005066 +005069 +005070 +005074 +005075 +005076 +005080 +005082 +005083 +005087 +005088 +005089 +005091 +005092 +005095 +005096 +005098 +005099 +005100 +005103 +005105 +005106 +005109 +005112 +005113 +005115 +005117 +005118 +005119 +005120 +005123 +005125 +005126 +005127 +005132 +005133 +005137 +005139 +005140 +005141 +005142 +005147 +005148 +005149 +005151 +005152 +005154 +005155 +005157 +005158 +005162 +005163 +005164 +005165 +005166 +005167 +005170 +005172 +005174 +005178 +005180 +005182 +005184 +005187 +005188 +005192 +005193 +005194 +005196 +005197 +005198 +005200 +005201 +005204 +005205 +005206 +005207 +005211 +005213 +005216 +005218 +005221 +005225 +005226 +005227 +005228 +005232 +005233 +005234 +005235 +005237 +005238 +005240 +005241 +005243 +005247 +005249 +005250 +005251 +005252 +005255 +005256 +005261 +005265 +005266 +005271 +005272 +005275 +005276 +005277 +005279 +005280 +005282 +005284 +005286 +005287 +005289 +005291 +005294 +005295 +005296 +005299 +005300 +005301 +005302 +005308 +005309 +005313 +005316 +005317 +005321 +005322 +005323 +005324 +005329 +005330 +005332 +005333 +005334 +005335 +005339 +005341 +005342 +005347 +005353 +005354 +005356 +005357 +005359 +005361 +005362 +005364 +005366 +005372 +005375 +005376 +005377 +005381 +005382 +005386 +005390 +005392 +005394 +005399 +005400 +005401 +005402 +005403 +005409 +005411 +005412 +005415 +005422 +005425 +005426 +005427 +005428 +005432 +005435 +005437 +005442 +005443 +005444 +005446 +005447 +005449 +005452 +005456 +005458 +005459 +005460 +005462 +005463 +005464 +005466 +005468 +005472 +005473 +005474 +005476 +005477 +005479 +005480 +005482 +005484 +005488 +005490 +005491 +005492 +005493 +005494 +005495 +005498 +005500 +005501 +005502 +005503 +005504 +005505 +005506 +005512 +005513 +005516 +005520 +005523 +005525 +005528 +005529 +005532 +005533 +005534 +005537 +005538 +005540 +005543 +005545 +005546 +005548 +005551 +005553 +005555 +005556 +005557 +005558 +005560 +005561 +005562 +005564 +005565 +005567 +005569 +005570 +005571 +005572 +005575 +005578 +005580 +005581 +005587 +005589 +005594 +005595 +005596 +005597 +005598 +005602 +005604 +005607 +005610 +005612 +005616 +005617 +005619 +005621 +005622 +005623 +005626 +005627 +005628 +005632 +005633 +005634 +005635 +005638 +005642 +005643 +005646 +005649 +005650 +005651 +005656 +005659 +005661 +005663 +005665 +005666 +005667 +005670 +005671 +005673 +005675 +005677 +005678 +005681 +005683 +005684 +005688 +005689 +005690 +005691 +005692 +005694 +005698 +005703 +005706 +005707 +005708 +005709 +005711 +005712 +005717 +005720 +005721 +005722 +005724 +005725 +005726 +005727 +005733 +005734 +005737 +005739 +005744 +005745 +005746 +005748 +005750 +005751 +005753 +005754 +005758 +005759 +005763 +005766 +005767 +005770 +005771 +005772 +005774 +005775 +005776 +005777 +005778 +005785 +005787 +005792 +005793 +005795 +005797 +005798 +005800 +005801 +005802 +005804 +005807 +005808 +005809 +005810 +005816 +005820 +005822 +005823 +005827 +005832 +005833 +005834 +005835 +005837 +005842 +005844 +005846 +005847 +005848 +005849 +005855 +005857 +005858 +005862 +005865 +005866 +005869 +005870 +005871 +005872 +005876 +005880 +005882 +005883 +005886 +005887 +005890 +005891 +005892 +005896 +005898 +005900 +005902 +005904 +005907 +005913 +005915 +005916 +005921 +005922 +005924 +005925 +005926 +005927 +005929 +005931 +005932 +005933 +005934 +005935 +005936 +005937 +005939 +005941 +005942 +005943 +005944 +005945 +005946 +005949 +005950 +005953 +005955 +005957 +005958 +005959 +005962 +005965 +005966 +005967 +005969 +005972 +005973 +005974 +005976 +005977 +005978 +005982 +005986 +005987 +005993 +005994 +005997 +005999 +006002 +006003 +006006 +006007 +006008 +006010 +006013 +006014 +006015 +006016 +006017 +006019 +006021 +006022 +006024 +006031 +006032 +006034 +006036 +006037 +006039 +006040 +006044 +006047 +006048 +006049 +006050 +006051 +006052 +006053 +006054 +006056 +006057 +006059 +006060 +006063 +006064 +006068 +006072 +006075 +006076 +006077 +006080 +006081 +006082 +006083 +006085 +006086 +006087 +006090 +006092 +006093 +006094 +006099 +006101 +006102 +006106 +006109 +006110 +006112 +006113 +006114 +006115 +006116 +006118 +006119 +006121 +006122 +006126 +006127 +006132 +006137 +006138 +006142 +006143 +006144 +006145 +006147 +006149 +006152 +006154 +006155 +006157 +006160 +006164 +006165 +006167 +006168 +006169 +006173 +006178 +006182 +006186 +006191 +006192 +006193 +006194 +006195 +006197 +006199 +006200 +006204 +006205 +006207 +006211 +006213 +006217 +006226 +006227 +006228 +006231 +006232 +006237 +006239 +006242 +006244 +006245 +006246 +006248 +006253 +006255 +006256 +006257 +006263 +006265 +006266 +006268 +006271 +006273 +006274 +006278 +006280 +006283 +006287 +006288 +006292 +006293 +006294 +006297 +006298 +006302 +006303 +006307 +006308 +006310 +006311 +006312 +006313 +006315 +006316 +006317 +006322 +006324 +006326 +006327 +006328 +006331 +006332 +006333 +006334 +006336 +006340 +006342 +006343 +006345 +006347 +006354 +006356 +006358 +006359 +006360 +006361 +006364 +006365 +006368 +006370 +006372 +006373 +006376 +006378 +006379 +006380 +006383 +006384 +006386 +006388 +006389 +006390 +006393 +006394 +006397 +006399 +006401 +006402 +006403 +006405 +006406 +006407 +006408 +006410 +006412 +006413 +006414 +006415 +006416 +006420 +006422 +006423 +006426 +006431 +006432 +006435 +006439 +006441 +006446 +006451 +006452 +006453 +006454 +006457 +006460 +006461 +006464 +006467 +006469 +006471 +006477 +006478 +006479 +006481 +006485 +006487 +006489 +006490 +006491 +006493 +006494 +006496 +006498 +006500 +006502 +006504 +006505 +006508 +006510 +006511 +006513 +006514 +006516 +006517 +006518 +006521 +006522 +006525 +006526 +006527 +006528 +006531 +006533 +006535 +006537 +006539 +006540 +006541 +006544 +006545 +006546 +006552 +006554 +006555 +006557 +006558 +006559 +006561 +006563 +006566 +006567 +006568 +006571 +006573 +006574 +006577 +006579 +006580 +006581 +006582 +006586 +006589 +006590 +006591 +006592 +006594 +006596 +006598 +006600 +006601 +006604 +006607 +006608 +006613 +006614 +006615 +006616 +006620 +006623 +006624 +006629 +006630 +006633 +006634 +006639 +006640 +006641 +006642 +006644 +006646 +006649 +006650 +006651 +006653 +006655 +006656 +006659 +006662 +006663 +006665 +006669 +006672 +006675 +006676 +006680 +006683 +006685 +006686 +006688 +006691 +006692 +006693 +006700 +006701 +006705 +006710 +006711 +006712 +006713 +006715 +006716 +006717 +006720 +006721 +006723 +006724 +006728 +006729 +006732 +006733 +006737 +006741 +006742 +006743 +006744 +006745 +006746 +006749 +006750 +006752 +006754 +006756 +006757 +006758 +006763 +006764 +006767 +006770 +006771 +006774 +006775 +006776 +006778 +006779 +006780 +006785 +006787 +006788 +006790 +006791 +006792 +006793 +006795 +006796 +006798 +006801 +006804 +006807 +006809 +006811 +006812 +006815 +006816 +006817 +006818 +006820 +006823 +006826 +006830 +006831 +006832 +006834 +006837 +006843 +006846 +006851 +006853 +006854 +006856 +006857 +006861 +006863 +006870 +006871 +006872 +006873 +006875 +006877 +006879 +006881 +006882 +006885 +006888 +006889 +006890 +006891 +006894 +006895 +006897 +006898 +006901 +006902 +006904 +006905 +006906 +006907 +006913 +006915 +006920 +006921 +006923 +006925 +006926 +006927 +006928 +006929 +006936 +006937 +006938 +006941 +006942 +006946 +006951 +006954 +006955 +006957 +006960 +006961 +006964 +006967 +006969 +006970 +006973 +006974 +006975 +006977 +006978 +006979 +006980 +006982 +006984 +006985 +006986 +006991 +006992 +006993 +006996 +006997 +006998 +006999 +007000 +007001 +007005 +007010 +007012 +007013 +007014 +007015 +007017 +007019 +007024 +007026 +007027 +007028 +007030 +007032 +007034 +007037 +007041 +007043 +007044 +007047 +007051 +007053 +007055 +007057 +007060 +007061 +007063 +007066 +007067 +007069 +007076 +007081 +007082 +007083 +007085 +007087 +007091 +007094 +007096 +007098 +007099 +007102 +007103 +007106 +007107 +007110 +007111 +007112 +007115 +007116 +007118 +007119 +007120 +007124 +007126 +007127 +007131 +007134 +007135 +007136 +007137 +007142 +007143 +007145 +007151 +007155 +007156 +007157 +007158 +007160 +007161 +007164 +007169 +007170 +007171 +007173 +007175 +007176 +007178 +007179 +007181 +007183 +007186 +007188 +007190 +007192 +007195 +007196 +007198 +007199 +007201 +007202 +007203 +007206 +007207 +007209 +007218 +007220 +007221 +007225 +007226 +007228 +007229 +007231 +007232 +007233 +007235 +007237 +007238 +007239 +007240 +007242 +007246 +007248 +007251 +007252 +007253 +007254 +007255 +007257 +007262 +007264 +007265 +007267 +007268 +007269 +007272 +007273 +007277 +007278 +007281 +007282 +007286 +007287 +007288 +007290 +007291 +007293 +007301 +007303 +007304 +007306 +007307 +007309 +007310 +007312 +007313 +007315 +007316 +007317 +007319 +007320 +007321 +007324 +007326 +007328 +007331 +007332 +007333 +007335 +007337 +007338 +007339 +007340 +007341 +007342 +007345 +007347 +007348 +007349 +007352 +007353 +007354 +007355 +007357 +007358 +007360 +007362 +007364 +007366 +007367 +007368 +007371 +007377 +007378 +007379 +007380 +007382 +007384 +007386 +007387 +007391 +007392 +007393 +007395 +007397 +007399 +007400 +007401 +007402 +007403 +007404 +007405 +007406 +007407 +007409 +007412 +007415 +007418 +007420 +007423 +007426 +007428 +007429 +007430 +007434 +007440 +007441 +007442 +007444 +007447 +007450 +007452 +007453 +007455 +007456 +007459 +007462 +007463 +007464 +007469 +007471 +007472 +007473 +007476 +007478 +007485 +007487 +007488 +007492 +007494 +007495 +007496 +007499 +007500 +007501 +007502 +007504 +007505 +007507 +007508 +007509 +007510 +007512 +007514 +007515 +007516 +007518 +007520 +007522 +007529 +007531 +007532 +007534 +007539 +007541 +007542 +007545 +007548 +007549 +007550 +007552 +007553 +007554 +007556 +007557 +007560 +007561 +007562 +007564 +007567 +007569 +007573 +007574 +007577 +007580 +007581 +007582 +007583 +007584 +007587 +007588 +007589 +007591 +007593 +007595 +007596 +007597 +007598 +007599 +007602 +007604 +007607 +007608 +007609 +007610 +007613 +007616 +007617 +007620 +007623 +007625 +007627 +007628 +007630 +007632 +007634 +007635 +007636 +007638 +007641 +007643 +007644 +007645 +007646 +007648 +007651 +007652 +007658 +007659 +007660 +007661 +007665 +007669 +007674 +007676 +007681 +007684 +007686 +007689 +007690 +007693 +007695 +007698 +007700 +007701 +007703 +007706 +007707 +007708 +007710 +007711 +007714 +007716 +007717 +007719 +007722 +007725 +007726 +007728 +007730 +007733 +007734 +007737 +007738 +007739 +007741 +007744 +007747 +007750 +007752 +007755 +007756 +007757 +007759 +007761 +007764 +007766 +007769 +007770 +007771 +007774 +007778 +007780 +007782 +007783 +007784 +007785 +007787 +007788 +007789 +007792 +007794 +007796 +007797 +007800 +007801 +007802 +007804 +007805 +007806 +007807 +007808 +007811 +007816 +007817 +007818 +007822 +007823 +007825 +007827 +007828 +007829 +007830 +007832 +007835 +007837 +007839 +007842 +007844 +007846 +007848 +007849 +007850 +007851 +007852 +007858 +007860 +007861 +007862 +007866 +007867 +007870 +007871 +007874 +007875 +007879 +007880 +007881 +007882 +007887 +007888 +007891 +007892 +007893 +007894 +007895 +007896 +007903 +007904 +007906 +007907 +007912 +007913 +007917 +007918 +007922 +007927 +007929 +007930 +007934 +007936 +007937 +007938 +007941 +007942 +007944 +007945 +007948 +007949 +007951 +007952 +007955 +007957 +007960 +007961 +007962 +007965 +007966 +007967 +007969 +007972 +007973 +007975 +007977 +007978 +007981 +007982 +007983 +007985 +007986 +007988 +007989 +007990 +007992 +007993 +007994 +007995 +008000 +008003 +008006 +008007 +008010 +008011 +008013 +008014 +008015 +008016 +008018 +008020 +008021 +008022 +008025 +008027 +008028 +008030 +008034 +008035 +008038 +008039 +008041 +008045 +008046 +008047 +008050 +008052 +008054 +008055 +008056 +008058 +008059 +008065 +008066 +008070 +008071 +008073 +008074 +008077 +008078 +008080 +008081 +008088 +008089 +008090 +008092 +008094 +008097 +008099 +008102 +008104 +008109 +008110 +008111 +008113 +008114 +008118 +008119 +008120 +008123 +008124 +008126 +008128 +008129 +008131 +008133 +008134 +008135 +008136 +008143 +008145 +008146 +008147 +008148 +008149 +008152 +008153 +008154 +008155 +008156 +008157 +008158 +008161 +008162 +008165 +008167 +008170 +008172 +008176 +008178 +008179 +008181 +008182 +008183 +008184 +008185 +008187 +008192 +008193 +008194 +008195 +008196 +008198 +008201 +008205 +008206 +008207 +008210 +008212 +008214 +008215 +008217 +008219 +008221 +008227 +008228 +008230 +008231 +008233 +008234 +008237 +008238 +008239 +008240 +008242 +008243 +008245 +008246 +008247 +008249 +008255 +008256 +008257 +008259 +008264 +008265 +008266 +008267 +008270 +008271 +008273 +008274 +008276 +008277 +008278 +008283 +008286 +008287 +008288 +008289 +008290 +008291 +008298 +008303 +008304 +008305 +008308 +008309 +008314 +008321 +008324 +008325 +008328 +008330 +008331 +008333 +008334 +008337 +008339 +008340 +008343 +008344 +008347 +008348 +008350 +008352 +008353 +008354 +008356 +008357 +008358 +008361 +008362 +008363 +008366 +008367 +008369 +008371 +008373 +008375 +008377 +008378 +008379 +008380 +008382 +008383 +008389 +008392 +008393 +008394 +008395 +008396 +008399 +008400 +008401 +008402 +008404 +008405 +008406 +008407 +008408 +008411 +008412 +008414 +008417 +008418 +008419 +008420 +008421 +008428 +008431 +008432 +008435 +008436 +008439 +008440 +008441 +008446 +008447 +008448 +008451 +008455 +008457 +008458 +008459 +008460 +008463 +008464 +008469 +008471 +008473 +008474 +008476 +008479 +008480 +008481 +008486 +008487 +008488 +008489 +008490 +008491 +008493 +008496 +008497 +008500 +008501 +008504 +008505 +008507 +008508 +008510 +008511 +008515 +008516 +008520 +008525 +008527 +008528 +008531 +008532 +008537 +008538 +008539 +008540 +008543 +008544 +008545 +008546 +008547 +008548 +008551 +008552 +008554 +008555 +008560 +008561 +008563 +008565 +008566 +008567 +008569 +008570 +008571 +008574 +008575 +008577 +008578 +008579 +008580 +008583 +008589 +008590 +008591 +008593 +008594 +008597 +008598 +008599 +008600 +008603 +008605 +008609 +008611 +008613 +008614 +008616 +008619 +008622 +008623 +008625 +008626 +008627 +008629 +008630 +008631 +008632 +008634 +008637 +008640 +008641 +008642 +008643 +008646 +008648 +008649 +008650 +008651 +008652 +008656 +008657 +008658 +008659 +008660 +008661 +008662 +008664 +008666 +008668 +008669 +008671 +008672 +008673 +008674 +008675 +008677 +008678 +008679 +008681 +008682 +008684 +008685 +008686 +008689 +008693 +008694 +008696 +008697 +008700 +008703 +008704 +008705 +008707 +008708 +008711 +008712 +008714 +008715 +008719 +008721 +008724 +008726 +008729 +008734 +008735 +008736 +008737 +008740 +008743 +008745 +008746 +008751 +008754 +008758 +008761 +008762 +008763 +008765 +008767 +008774 +008777 +008778 +008779 +008780 +008781 +008782 +008785 +008786 +008787 +008788 +008789 +008791 +008792 +008795 +008797 +008798 +008800 +008802 +008803 +008804 +008807 +008808 +008812 +008816 +008818 +008820 +008821 +008824 +008825 +008827 +008828 +008829 +008830 +008832 +008834 +008839 +008842 +008844 +008845 +008846 +008850 +008851 +008852 +008853 +008855 +008857 +008860 +008861 +008863 +008864 +008866 +008868 +008869 +008870 +008875 +008877 +008881 +008882 +008887 +008889 +008893 +008894 +008895 +008896 +008897 +008898 +008899 +008901 +008902 +008903 +008904 +008906 +008907 +008908 +008910 +008912 +008915 +008916 +008918 +008922 +008924 +008925 +008928 +008934 +008935 +008937 +008938 +008941 +008945 +008946 +008947 +008949 +008950 +008952 +008954 +008956 +008957 +008959 +008963 +008964 +008972 +008974 +008977 +008981 +008984 +008986 +008990 +008991 +008992 +008993 +008994 +008996 +008998 +009001 +009003 +009008 +009009 +009010 +009011 +009012 +009013 +009014 +009017 +009021 +009023 +009025 +009026 +009028 +009030 +009031 +009033 +009038 +009040 +009041 +009043 +009044 +009046 +009047 +009050 +009052 +009054 +009055 +009056 +009057 +009061 +009062 +009065 +009067 +009069 +009070 +009071 +009074 +009075 +009076 +009077 +009081 +009082 +009083 +009084 +009088 +009090 +009092 +009093 +009095 +009096 +009097 +009101 +009102 +009103 +009104 +009107 +009109 +009110 +009111 +009115 +009118 +009119 +009120 +009122 +009124 +009125 +009127 +009130 +009132 +009134 +009135 +009137 +009139 +009140 +009142 +009143 +009145 +009146 +009149 +009152 +009154 +009156 +009158 +009164 +009165 +009167 +009169 +009170 +009171 +009172 +009176 +009182 +009183 +009188 +009190 +009198 +009199 +009201 +009203 +009204 +009206 +009207 +009210 +009211 +009216 +009217 +009219 +009220 +009222 +009223 +009225 +009226 +009228 +009229 +009231 +009232 +009233 +009234 +009235 +009237 +009240 +009241 +009243 +009248 +009253 +009256 +009257 +009258 +009260 +009261 +009262 +009263 +009264 +009265 +009266 +009267 +009274 +009275 +009276 +009277 +009280 +009284 +009292 +009293 +009294 +009297 +009298 +009300 +009301 +009302 +009304 +009305 +009310 +009311 +009313 +009314 +009317 +009319 +009320 +009321 +009322 +009328 +009329 +009332 +009335 +009338 +009340 +009341 +009344 +009345 +009346 +009352 +009353 +009355 +009356 +009357 +009360 +009361 +009363 +009364 +009366 +009367 +009369 +009370 +009372 +009376 +009379 +009380 +009381 +009383 +009384 +009385 +009387 +009390 +009391 +009395 +009396 +009397 +009399 +009400 +009402 +009403 +009404 +009415 +009416 +009423 +009425 +009426 +009427 +009428 +009430 +009431 +009435 +009436 +009441 +009442 +009444 +009447 +009449 +009450 +009451 +009452 +009453 +009462 +009467 +009471 +009473 +009474 +009475 +009478 +009482 +009483 +009485 +009486 +009487 +009489 +009492 +009493 +009495 +009498 +009501 +009503 +009505 +009506 +009509 +009510 +009511 +009513 +009514 +009521 +009522 +009525 +009529 +009530 +009534 +009535 +009536 +009538 +009539 +009544 +009547 +009548 +009552 +009553 +009554 +009555 +009556 +009559 +009561 +009563 +009564 +009569 +009570 +009572 +009574 +009575 +009578 +009581 +009582 +009583 +009589 +009590 +009592 +009593 +009594 +009595 +009599 +009601 +009602 +009604 +009606 +009607 +009608 +009610 +009612 +009616 +009622 +009624 +009625 +009626 +009628 +009630 +009631 +009632 +009633 +009635 +009639 +009640 +009642 +009643 +009645 +009646 +009648 +009651 +009652 +009653 +009657 +009660 +009661 +009662 +009663 +009665 +009669 +009672 +009673 +009674 +009675 +009677 +009680 +009682 +009683 +009688 +009689 +009690 +009694 +009696 +009697 +009701 +009704 +009705 +009708 +009714 +009715 +009716 +009720 +009722 +009723 +009725 +009727 +009728 +009730 +009731 +009736 +009739 +009740 +009741 +009742 +009744 +009750 +009751 +009752 +009753 +009757 +009759 +009760 +009765 +009766 +009768 +009769 +009770 +009771 +009775 +009777 +009779 +009782 +009783 +009784 +009786 +009787 +009788 +009791 +009793 +009795 +009798 +009799 +009802 +009803 +009804 +009806 +009811 +009812 +009814 +009815 +009817 +009818 +009820 +009821 +009824 +009826 +009827 +009829 +009835 +009837 +009838 +009840 +009843 +009844 +009846 +009847 +009849 +009850 +009853 +009854 +009856 +009857 +009861 +009864 +009866 +009871 +009873 +009875 +009876 +009883 +009885 +009888 +009889 +009890 +009891 +009892 +009893 +009895 +009899 +009901 +009903 +009906 +009907 +009909 +009910 +009912 +009914 +009915 +009916 +009919 +009921 +009922 +009924 +009925 +009927 +009928 +009929 +009930 +009931 +009933 +009934 +009936 +009937 +009941 +009943 +009948 +009951 +009952 +009953 +009956 +009957 +009960 +009962 +009963 diff --git a/detectron2/config/defaults.py b/detectron2/config/defaults.py index b33d8ae..c2c3a3b 100644 --- a/detectron2/config/defaults.py +++ b/detectron2/config/defaults.py @@ -616,6 +616,7 @@ _C.OWOD.CUR_INTRODUCED_CLS = 20 _C.OWOD.COMPUTE_ENERGY = False _C.OWOD.ENERGY_SAVE_PATH = '' _C.OWOD.SKIP_TRAINING_WHILE_EVAL = False +_C.OWOD.FEATURE_STORE_SAVE_PATH = '' # ---------------------------------------------------------------------------- # # Misc options diff --git a/detectron2/data/datasets/builtin.py b/detectron2/data/datasets/builtin.py index 6f5ebea..bdc9e4e 100644 --- a/detectron2/data/datasets/builtin.py +++ b/detectron2/data/datasets/builtin.py @@ -215,6 +215,7 @@ def register_all_pascal_voc(root): ("voc_2012_train", "VOC2012", "train"), ("voc_2012_val", "VOC2012", "val"), ("t1_voc_coco_2007_train", "VOC2007", "t1_train"), + ("t1_voc_coco_2007_known_test", "VOC2007", "t1_known_test"), ("voc_coco_2007_test", "VOC2007", "all_task_test"), ("voc_coco_2007_val", "VOC2007", "all_task_val"), ("t1_voc_coco_2007_ft", "VOC2007", "t1_ft"), diff --git a/detectron2/engine/defaults.py b/detectron2/engine/defaults.py index 618d45c..42edc22 100644 --- a/detectron2/engine/defaults.py +++ b/detectron2/engine/defaults.py @@ -121,6 +121,9 @@ def default_setup(cfg, args): if cfg.OWOD.COMPUTE_ENERGY: PathManager.mkdirs(os.path.join(output_dir, cfg.OWOD.ENERGY_SAVE_PATH)) + if cfg.OWOD.ENABLE_CLUSTERING: + PathManager.mkdirs(os.path.join(output_dir, cfg.OWOD.FEATURE_STORE_SAVE_PATH)) + rank = comm.get_rank() setup_logger(output_dir, distributed_rank=rank, name="fvcore") logger = setup_logger(output_dir, distributed_rank=rank) diff --git a/detectron2/evaluation/pascal_voc_evaluation.py b/detectron2/evaluation/pascal_voc_evaluation.py index 24be5bf..ba51c7c 100644 --- a/detectron2/evaluation/pascal_voc_evaluation.py +++ b/detectron2/evaluation/pascal_voc_evaluation.py @@ -4,8 +4,10 @@ import logging import numpy as np import os +import sys import tempfile import xml.etree.ElementTree as ET +import matplotlib.pyplot as plt from collections import OrderedDict, defaultdict from functools import lru_cache import torch @@ -19,6 +21,7 @@ from detectron2.utils import comm from .evaluator import DatasetEvaluator +np.set_printoptions(threshold=sys.maxsize) class PascalVOCDetectionEvaluator(DatasetEvaluator): """ @@ -42,7 +45,8 @@ class PascalVOCDetectionEvaluator(DatasetEvaluator): self._image_set_path = os.path.join(meta.dirname, "ImageSets", "Main", meta.split + ".txt") self._class_names = meta.thing_classes assert meta.year in [2007, 2012], meta.year - self._is_2007 = meta.year == 2007 + self._is_2007 = False + # self._is_2007 = meta.year == 2007 self._cpu_device = torch.device("cpu") self._logger = logging.getLogger(__name__) if cfg is not None: @@ -54,7 +58,7 @@ class PascalVOCDetectionEvaluator(DatasetEvaluator): self.known_classes = self._class_names[:self.num_seen_classes] param_save_location = os.path.join(cfg.OUTPUT_DIR,'energy_dist_' + str(self.num_seen_classes) + '.pkl') - + self.energy_distribution_loaded = False if os.path.isfile(param_save_location) and os.access(param_save_location, os.R_OK): self._logger.info('Loading energy distribution from ' + param_save_location) params = torch.load(param_save_location) @@ -103,7 +107,6 @@ class PascalVOCDetectionEvaluator(DatasetEvaluator): cls[i] = -100 else: if cls[i] != self.unknown_class_index: - # cls[i] = -100 cls[i] = self.unknown_class_index return cls @@ -127,6 +130,28 @@ class PascalVOCDetectionEvaluator(DatasetEvaluator): f"{image_id} {score:.3f} {xmin:.1f} {ymin:.1f} {xmax:.1f} {ymax:.1f}" ) + def compute_avg_precision_at_many_recall_level(self, precisions, recalls): + precs = {} + for r in range(1, 10): + r = r/10 + p = self.compute_avg_precision_at_a_recall_level(precisions, recalls, recall_level=r) + precs[r] = p + return precs + + def compute_avg_precision_at_a_recall_level(self, precisions, recalls, recall_level=0.5): + precs = {} + for iou, recall in recalls.items(): + prec = [] + for cls_id, rec in enumerate(recall): + if cls_id in range(self.num_seen_classes) and len(rec)>0: + p = precisions[iou][cls_id][min(range(len(rec)), key=lambda i: abs(rec[i] - recall_level))] + prec.append(p) + if len(prec) > 0: + precs[iou] = np.mean(prec) + else: + precs[iou] = 0 + return precs + def evaluate(self): """ Returns: @@ -152,8 +177,12 @@ class PascalVOCDetectionEvaluator(DatasetEvaluator): res_file_template = os.path.join(dirname, "{}.txt") aps = defaultdict(list) # iou -> ap per class - # recs = defaultdict(list) - # precs = defaultdict(list) + recs = defaultdict(list) + precs = defaultdict(list) + all_recs = defaultdict(list) + all_precs = defaultdict(list) + unk_det_as_knowns = defaultdict(list) + num_unks = defaultdict(list) for cls_id, cls_name in enumerate(self._class_names): lines = predictions.get(cls_id, [""]) @@ -161,47 +190,75 @@ class PascalVOCDetectionEvaluator(DatasetEvaluator): with open(res_file_template.format(cls_name), "w") as f: f.write("\n".join(lines)) - for thresh in range(50, 100, 5): - rec, prec, ap = voc_eval( - res_file_template, - self._anno_file_template, - self._image_set_path, - cls_name, - ovthresh=thresh / 100.0, - use_07_metric=self._is_2007, - known_classes=self.known_classes - ) - aps[thresh].append(ap * 100) - # recs[thresh].append(rec * 100) - # precs[thresh].append(prec * 100) + # for thresh in range(50, 100, 5): + thresh = 50 + rec, prec, ap, unk_det_as_known, num_unk = voc_eval( + res_file_template, + self._anno_file_template, + self._image_set_path, + cls_name, + ovthresh=thresh / 100.0, + use_07_metric=self._is_2007, + known_classes=self.known_classes + ) + aps[thresh].append(ap * 100) + unk_det_as_knowns[thresh].append(unk_det_as_known) + num_unks[thresh].append(num_unk) + all_precs[thresh].append(prec) + all_recs[thresh].append(rec) + try: + recs[thresh].append(rec[-1] * 100) + precs[thresh].append(prec[-1] * 100) + except: + recs[thresh].append(0) + precs[thresh].append(0) + + avg_precision = self.compute_avg_precision_at_many_recall_level(all_precs, all_recs) + self._logger.info('avg_precision: ' + str(avg_precision)) ret = OrderedDict() mAP = {iou: np.mean(x) for iou, x in aps.items()} - ret["bbox"] = {"AP": np.mean(list(mAP.values())), "AP50": mAP[50], "AP75": mAP[75]} + ret["bbox"] = {"AP": np.mean(list(mAP.values())), "AP50": mAP[50]} + + total_num_unk_det_as_known = {iou: np.sum(x) for iou, x in unk_det_as_knowns.items()} + total_num_unk = num_unks[50][0] + self._logger.info('Absolute OSE (total_num_unk_det_as_known): ' + str(total_num_unk_det_as_known)) + self._logger.info('total_num_unk ' + str(total_num_unk)) # Extra logging of class-wise APs avg_precs = list(np.mean([x for _, x in aps.items()], axis=0)) self._logger.info(self._class_names) - self._logger.info("AP__: " + str(['%.1f' % x for x in avg_precs])) + # self._logger.info("AP__: " + str(['%.1f' % x for x in avg_precs])) self._logger.info("AP50: " + str(['%.1f' % x for x in aps[50]])) - self._logger.info("AP75: " + str(['%.1f' % x for x in aps[75]])) + self._logger.info("Precisions50: " + str(['%.1f' % x for x in precs[50]])) + self._logger.info("Recall50: " + str(['%.1f' % x for x in recs[50]])) + # self._logger.info("AP75: " + str(['%.1f' % x for x in aps[75]])) if self.prev_intro_cls > 0: - self._logger.info("Prev class AP__: " + str(np.mean(avg_precs[:self.prev_intro_cls]))) + # self._logger.info("\nPrev class AP__: " + str(np.mean(avg_precs[:self.prev_intro_cls]))) self._logger.info("Prev class AP50: " + str(np.mean(aps[50][:self.prev_intro_cls]))) - self._logger.info("Prev class AP75: " + str(np.mean(aps[75][:self.prev_intro_cls]))) + self._logger.info("Prev class Precisions50: " + str(np.mean(precs[50][:self.prev_intro_cls]))) + self._logger.info("Prev class Recall50: " + str(np.mean(recs[50][:self.prev_intro_cls]))) - self._logger.info("Current class AP__: " + str(np.mean(avg_precs[self.prev_intro_cls:self.curr_intro_cls]))) - self._logger.info("Current class AP50: " + str(np.mean(aps[50][self.prev_intro_cls:self.curr_intro_cls]))) - self._logger.info("Current class AP75: " + str(np.mean(aps[75][self.prev_intro_cls:self.curr_intro_cls]))) + # self._logger.info("Prev class AP75: " + str(np.mean(aps[75][:self.prev_intro_cls]))) - self._logger.info("Known AP__: " + str(np.mean(avg_precs[:self.prev_intro_cls + self.curr_intro_cls]))) + # self._logger.info("\nCurrent class AP__: " + str(np.mean(avg_precs[self.prev_intro_cls:self.curr_intro_cls]))) + self._logger.info("Current class AP50: " + str(np.mean(aps[50][self.prev_intro_cls:self.prev_intro_cls + self.curr_intro_cls]))) + self._logger.info("Current class Precisions50: " + str(np.mean(precs[50][self.prev_intro_cls:self.prev_intro_cls + self.curr_intro_cls]))) + self._logger.info("Current class Recall50: " + str(np.mean(recs[50][self.prev_intro_cls:self.prev_intro_cls + self.curr_intro_cls]))) + # self._logger.info("Current class AP75: " + str(np.mean(aps[75][self.prev_intro_cls:self.curr_intro_cls]))) + + # self._logger.info("\nKnown AP__: " + str(np.mean(avg_precs[:self.prev_intro_cls + self.curr_intro_cls]))) self._logger.info("Known AP50: " + str(np.mean(aps[50][:self.prev_intro_cls + self.curr_intro_cls]))) - self._logger.info("Known AP75: " + str(np.mean(aps[75][:self.prev_intro_cls + self.curr_intro_cls]))) + self._logger.info("Known Precisions50: " + str(np.mean(precs[50][:self.prev_intro_cls + self.curr_intro_cls]))) + self._logger.info("Known Recall50: " + str(np.mean(recs[50][:self.prev_intro_cls + self.curr_intro_cls]))) + # self._logger.info("Known AP75: " + str(np.mean(aps[75][:self.prev_intro_cls + self.curr_intro_cls]))) - self._logger.info("Unknown AP__: " + str(avg_precs[-1])) + # self._logger.info("\nUnknown AP__: " + str(avg_precs[-1])) self._logger.info("Unknown AP50: " + str(aps[50][-1])) - self._logger.info("Unknown AP75: " + str(aps[75][-1])) + self._logger.info("Unknown Precisions50: " + str(precs[50][-1])) + self._logger.info("Unknown Recall50: " + str(recs[50][-1])) + # self._logger.info("Unknown AP75: " + str(aps[75][-1])) # self._logger.info("R__: " + str(['%.1f' % x for x in list(np.mean([x for _, x in recs.items()], axis=0))])) # self._logger.info("R50: " + str(['%.1f' % x for x in recs[50]])) @@ -362,6 +419,10 @@ def voc_eval(detpath, annopath, imagesetfile, classname, ovthresh=0.5, use_07_me nd = len(image_ids) tp = np.zeros(nd) fp = np.zeros(nd) + + # if 'unknown' not in classname: + # return tp, fp, 0 + for d in range(nd): R = class_recs[image_ids[d]] bb = BB[d, :].astype(float) @@ -407,6 +468,93 @@ def voc_eval(detpath, annopath, imagesetfile, classname, ovthresh=0.5, use_07_me # avoid divide by zero in case the first detection matches a difficult # ground truth prec = tp / np.maximum(tp + fp, np.finfo(np.float64).eps) + # plot_pr_curve(prec, rec, classname+'.png') ap = voc_ap(rec, prec, use_07_metric) - return rec, prec, ap + # print('tp: ' + str(tp[-1])) + # print('fp: ' + str(fp[-1])) + # print('tp: ') + # print(tp) + # print('fp: ') + # print(fp) + + + ''' + Computing Open-Set Error (OSE) + ============================== + + OSE = # of unknown objects classified as known objects of class 'classname' + -------------------------------------------------------------------- + # of unknown objects + ''' + logger = logging.getLogger(__name__) + + # Finding GT of unknown objects + unknown_class_recs = {} + n_unk = 0 + for imagename in imagenames: + R = [obj for obj in recs[imagename] if obj["name"] == 'unknown'] + bbox = np.array([x["bbox"] for x in R]) + difficult = np.array([x["difficult"] for x in R]).astype(np.bool) + det = [False] * len(R) + n_unk = n_unk + sum(~difficult) + unknown_class_recs[imagename] = {"bbox": bbox, "difficult": difficult, "det": det} + + if classname == 'unknown': + return rec, prec, ap, 0, n_unk + + # Go down each detection and see if it has an overlap with an unknown object. + # If so, it is an unknown object that was classified as known. + is_unk = np.zeros(nd) + for d in range(nd): + R = unknown_class_recs[image_ids[d]] + bb = BB[d, :].astype(float) + ovmax = -np.inf + BBGT = R["bbox"].astype(float) + + if BBGT.size > 0: + # compute overlaps + # intersection + ixmin = np.maximum(BBGT[:, 0], bb[0]) + iymin = np.maximum(BBGT[:, 1], bb[1]) + ixmax = np.minimum(BBGT[:, 2], bb[2]) + iymax = np.minimum(BBGT[:, 3], bb[3]) + iw = np.maximum(ixmax - ixmin + 1.0, 0.0) + ih = np.maximum(iymax - iymin + 1.0, 0.0) + inters = iw * ih + + # union + uni = ( + (bb[2] - bb[0] + 1.0) * (bb[3] - bb[1] + 1.0) + + (BBGT[:, 2] - BBGT[:, 0] + 1.0) * (BBGT[:, 3] - BBGT[:, 1] + 1.0) + - inters + ) + + overlaps = inters / uni + ovmax = np.max(overlaps) + jmax = np.argmax(overlaps) + + if ovmax > ovthresh: + is_unk[d] = 1.0 + + is_unk = np.sum(is_unk) + # OSE = is_unk / n_unk + # logger.info('Number of unknowns detected knowns (for class '+ classname + ') is ' + str(is_unk)) + # logger.info("Num of unknown instances: " + str(n_unk)) + # logger.info('OSE: ' + str(OSE)) + + return rec, prec, ap, is_unk, n_unk + + +def plot_pr_curve(precision, recall, filename, base_path='/home/fk1/workspace/OWOD/output/plots/'): + fig, ax = plt.subplots() + ax.step(recall, precision, color='r', alpha=0.99, where='post') + ax.fill_between(recall, precision, alpha=0.2, color='b', step='post') + plt.xlabel('Recall') + plt.ylabel('Precision') + plt.ylim([0.0, 1.05]) + plt.xlim([0.0, 1.0]) + plt.savefig(base_path + filename) + + # print(precision) + # print(recall) diff --git a/detectron2/modeling/roi_heads/fast_rcnn.py b/detectron2/modeling/roi_heads/fast_rcnn.py index 5084fef..eaa1b6f 100644 --- a/detectron2/modeling/roi_heads/fast_rcnn.py +++ b/detectron2/modeling/roi_heads/fast_rcnn.py @@ -2,6 +2,7 @@ import logging from typing import Dict, Union import torch +import os import math import shortuuid from fvcore.nn import giou_loss, smooth_l1_loss @@ -414,6 +415,9 @@ class FastRCNNOutputLayers(nn.Module): enable_clustering, prev_intro_cls, curr_intro_cls, + max_iterations, + output_dir, + feat_store_path, num_classes: int, test_score_thresh: float = 0.0, test_nms_thresh: float = 0.5, @@ -472,9 +476,6 @@ class FastRCNNOutputLayers(nn.Module): self.clustering_update_mu_iter = clustering_update_mu_iter self.clustering_momentum = clustering_momentum - self.feature_store = Store(num_classes + 1, clustering_items_per_class) - self.means = [None for _ in range(num_classes + 1)] - self.hingeloss = nn.HingeEmbeddingLoss(2) self.enable_clustering = enable_clustering @@ -484,6 +485,22 @@ class FastRCNNOutputLayers(nn.Module): self.invalid_class_range = list(range(self.seen_classes, self.num_classes-1)) logging.getLogger(__name__).info("Invalid class range: " + str(self.invalid_class_range)) + self.max_iterations = max_iterations + self.feature_store_is_stored = False + self.output_dir = output_dir + self.feat_store_path = feat_store_path + self.feature_store_save_loc = os.path.join(self.output_dir, self.feat_store_path, 'feat.pt') + + if os.path.isfile(self.feature_store_save_loc): + self.feature_store = torch.load(self.feature_store_save_loc) + logging.getLogger(__name__).info('Loaded feature store from ' + self.feature_store_save_loc) + else: + logging.getLogger(__name__).info('Feature store not found in ' + + self.feature_store_save_loc + '. Creating new feature store.') + self.feature_store = Store(num_classes + 1, clustering_items_per_class) + self.means = [None for _ in range(num_classes + 1)] + + # self.ae_model = AE(input_size, clustering_z_dimension) # self.ae_model.apply(Xavier) @@ -508,7 +525,10 @@ class FastRCNNOutputLayers(nn.Module): "clustering_z_dimension": cfg.OWOD.CLUSTERING.Z_DIMENSION, "enable_clustering" : cfg.OWOD.ENABLE_CLUSTERING, "prev_intro_cls" : cfg.OWOD.PREV_INTRODUCED_CLS, - "curr_intro_cls" : cfg.OWOD.CUR_INTRODUCED_CLS + "curr_intro_cls" : cfg.OWOD.CUR_INTRODUCED_CLS, + "max_iterations" : cfg.SOLVER.MAX_ITER, + "output_dir" : cfg.OUTPUT_DIR, + "feat_store_path" : cfg.OWOD.FEATURE_STORE_SAVE_PATH, # fmt: on } @@ -536,75 +556,16 @@ class FastRCNNOutputLayers(nn.Module): gt_classes = torch.cat([p.gt_classes for p in proposals]) self.feature_store.add(features, gt_classes) + storage = get_event_storage() + + if storage.iter == self.max_iterations and self.feature_store_is_stored is False: + logging.getLogger(__name__).info('Saving image store at iteration ' + str(storage.iter) + ' to ' + self.feature_store_save_loc) + torch.save(self.feature_store, self.feature_store_save_loc) + self.feature_store_is_stored = True + # self.feature_store.add(F.normalize(features, dim=0), gt_classes) # self.feature_store.add(self.ae_model.encoder(features), gt_classes) - # def clstr_loss(self, input_features, proposals): - # """ - # Get the foreground input_features, generate distributions for the class, - # get probability of each feature from each distribution; - # Compute loss: if belonging to a class -> likelihood should be higher - # else -> lower - # :param input_features: - # :param proposals: - # :return: - # """ - # loss = 0 - # gt_classes = torch.cat([p.gt_classes for p in proposals]) - # mask = gt_classes != self.num_classes - # fg_features = input_features[mask] - # classes = gt_classes[mask] - # # fg_features = self.ae_model.encoder(fg_features) - # - # # Distribution per class - # log_prob = [None for _ in range(self.num_classes + 1)] - # # https://github.com/pytorch/pytorch/issues/23780 - # for cls_index, mu in enumerate(self.means): - # if mu is not None: - # dist = Normal(loc=mu.cuda(), scale=torch.ones_like(mu.cuda())) - # log_prob[cls_index] = dist.log_prob(fg_features).mean(dim=1) - # # log_prob[cls_index] = torch.distributions.multivariate_normal. \ - # # MultivariateNormal(mu.cuda(), torch.eye(len(mu)).cuda()).log_prob(fg_features) - # # MultivariateNormal(mu, torch.eye(len(mu))).log_prob(fg_features.cpu()) - # # MultivariateNormal(mu[:2], torch.eye(len(mu[:2]))).log_prob(fg_features[:,:2].cpu()) - # else: - # log_prob[cls_index] = torch.zeros((len(fg_features))).cuda() - # - # log_prob = torch.stack(log_prob).T # num_of_fg_proposals x num_of_classes - # for i, p in enumerate(log_prob): - # weight = torch.ones_like(p) * -1 - # weight[classes[i]] = 1 - # p = p * weight - # loss += p.mean() - # return loss - - # def clstr_loss_l2(self, input_features, proposals): - # """ - # Get the foreground input_features, generate distributions for the class, - # get probability of each feature from each distribution; - # Compute loss: if belonging to a class -> likelihood should be higher - # else -> lower - # :param input_features: - # :param proposals: - # :return: - # """ - # loss = 0 - # gt_classes = torch.cat([p.gt_classes for p in proposals]) - # mask = gt_classes != self.num_classes - # fg_features = input_features[mask] - # classes = gt_classes[mask] - # fg_features = self.ae_model.encoder(fg_features) - # - # for index, feature in enumerate(fg_features): - # for cls_index, mu in enumerate(self.means): - # if mu is not None and feature is not None: - # mu = mu.cuda() - # if classes[index] == cls_index: - # loss -= F.mse_loss(feature, mu) - # else: - # loss += F.mse_loss(feature, mu) - # - # return loss def clstr_loss_l2_cdist(self, input_features, proposals): """ @@ -822,3 +783,70 @@ class FastRCNNOutputLayers(nn.Module): num_inst_per_image = [len(p) for p in proposals] probs = F.softmax(scores, dim=-1) return probs.split(num_inst_per_image, dim=0) + + # def clstr_loss(self, input_features, proposals): + # """ + # Get the foreground input_features, generate distributions for the class, + # get probability of each feature from each distribution; + # Compute loss: if belonging to a class -> likelihood should be higher + # else -> lower + # :param input_features: + # :param proposals: + # :return: + # """ + # loss = 0 + # gt_classes = torch.cat([p.gt_classes for p in proposals]) + # mask = gt_classes != self.num_classes + # fg_features = input_features[mask] + # classes = gt_classes[mask] + # # fg_features = self.ae_model.encoder(fg_features) + # + # # Distribution per class + # log_prob = [None for _ in range(self.num_classes + 1)] + # # https://github.com/pytorch/pytorch/issues/23780 + # for cls_index, mu in enumerate(self.means): + # if mu is not None: + # dist = Normal(loc=mu.cuda(), scale=torch.ones_like(mu.cuda())) + # log_prob[cls_index] = dist.log_prob(fg_features).mean(dim=1) + # # log_prob[cls_index] = torch.distributions.multivariate_normal. \ + # # MultivariateNormal(mu.cuda(), torch.eye(len(mu)).cuda()).log_prob(fg_features) + # # MultivariateNormal(mu, torch.eye(len(mu))).log_prob(fg_features.cpu()) + # # MultivariateNormal(mu[:2], torch.eye(len(mu[:2]))).log_prob(fg_features[:,:2].cpu()) + # else: + # log_prob[cls_index] = torch.zeros((len(fg_features))).cuda() + # + # log_prob = torch.stack(log_prob).T # num_of_fg_proposals x num_of_classes + # for i, p in enumerate(log_prob): + # weight = torch.ones_like(p) * -1 + # weight[classes[i]] = 1 + # p = p * weight + # loss += p.mean() + # return loss + + # def clstr_loss_l2(self, input_features, proposals): + # """ + # Get the foreground input_features, generate distributions for the class, + # get probability of each feature from each distribution; + # Compute loss: if belonging to a class -> likelihood should be higher + # else -> lower + # :param input_features: + # :param proposals: + # :return: + # """ + # loss = 0 + # gt_classes = torch.cat([p.gt_classes for p in proposals]) + # mask = gt_classes != self.num_classes + # fg_features = input_features[mask] + # classes = gt_classes[mask] + # fg_features = self.ae_model.encoder(fg_features) + # + # for index, feature in enumerate(fg_features): + # for cls_index, mu in enumerate(self.means): + # if mu is not None and feature is not None: + # mu = mu.cuda() + # if classes[index] == cls_index: + # loss -= F.mse_loss(feature, mu) + # else: + # loss += F.mse_loss(feature, mu) + # + # return loss \ No newline at end of file