2021-08-17 19:52:42 +08:00
|
|
|
# Copyright (c) OpenMMLab. All rights reserved.
|
Add evaluation metrics for multilabel task (#123)
* add mean_ap
* add difficult_examples in mAP to support dataset without difficult_examples
* fix docstring
* add CP,CR,CF1,OP,OR,OF1 as multilabel metrics
* fix docstring
* temporary solution to ci until new version of mmcv is avaliable (#127)
* temporary solution to ci until new version of mmcv is avaliable
* temporary solution to ci until new version of mmcv is avaliable
* add mean_ap
* add difficult_examples in mAP to support dataset without difficult_examples
* fix docstring
* add CP,CR,CF1,OP,OR,OF1 as multilabel metrics
* fix docstring
* Swap -1 and 0 for labels
* Revised according to comments
* Revised according to comments
* Revised according to comments
* Revert "Revised according to comments"
It is suggested that we should not include paper from arxiv.
This reverts commit 48a781cd6aca7be40821b55679bebb30295d4336.
* Revert "Revert "Revised according to comments""
This reverts commit 6d3b0f1a7b52afc8ed7a01a28e2911afa4f67c2b.
* Revert "Revised according to comments"
It is suggested we should not cite paper from arxiv.
This reverts commit 120ecda884a43d20a53f375fe1e97938bbd4cffd.
* Revised according to comments
* revised according to comments
* Revised according to comments
2021-01-04 12:25:33 +08:00
|
|
|
import pytest
|
|
|
|
|
import torch
|
|
|
|
|
|
|
|
|
|
from mmcls.core import average_performance, mAP
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_mAP():
|
|
|
|
|
target = torch.Tensor([[1, 1, 0, -1], [1, 1, 0, -1], [0, -1, 1, -1],
|
|
|
|
|
[0, 1, 0, -1]])
|
|
|
|
|
pred = torch.Tensor([[0.9, 0.8, 0.3, 0.2], [0.1, 0.2, 0.2, 0.1],
|
|
|
|
|
[0.7, 0.5, 0.9, 0.3], [0.8, 0.1, 0.1, 0.2]])
|
|
|
|
|
|
|
|
|
|
# target and pred should both be np.ndarray or torch.Tensor
|
|
|
|
|
with pytest.raises(TypeError):
|
|
|
|
|
target_list = target.tolist()
|
|
|
|
|
_ = mAP(pred, target_list)
|
|
|
|
|
|
|
|
|
|
# target and pred should be in the same shape
|
|
|
|
|
with pytest.raises(AssertionError):
|
|
|
|
|
target_shorter = target[:-1]
|
|
|
|
|
_ = mAP(pred, target_shorter)
|
|
|
|
|
|
|
|
|
|
assert mAP(pred, target) == pytest.approx(68.75, rel=1e-2)
|
|
|
|
|
|
|
|
|
|
target_no_difficult = torch.Tensor([[1, 1, 0, 0], [0, 1, 0, 0],
|
|
|
|
|
[0, 0, 1, 0], [1, 0, 0, 0]])
|
|
|
|
|
assert mAP(pred, target_no_difficult) == pytest.approx(70.83, rel=1e-2)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_average_performance():
|
|
|
|
|
target = torch.Tensor([[1, 1, 0, -1], [1, 1, 0, -1], [0, -1, 1, -1],
|
|
|
|
|
[0, 1, 0, -1], [0, 1, 0, -1]])
|
|
|
|
|
pred = torch.Tensor([[0.9, 0.8, 0.3, 0.2], [0.1, 0.2, 0.2, 0.1],
|
|
|
|
|
[0.7, 0.5, 0.9, 0.3], [0.8, 0.1, 0.1, 0.2],
|
|
|
|
|
[0.8, 0.1, 0.1, 0.2]])
|
|
|
|
|
|
|
|
|
|
# target and pred should both be np.ndarray or torch.Tensor
|
|
|
|
|
with pytest.raises(TypeError):
|
|
|
|
|
target_list = target.tolist()
|
|
|
|
|
_ = average_performance(pred, target_list)
|
|
|
|
|
|
|
|
|
|
# target and pred should be in the same shape
|
|
|
|
|
with pytest.raises(AssertionError):
|
|
|
|
|
target_shorter = target[:-1]
|
|
|
|
|
_ = average_performance(pred, target_shorter)
|
|
|
|
|
|
|
|
|
|
assert average_performance(pred, target) == average_performance(
|
|
|
|
|
pred, target, thr=0.5)
|
|
|
|
|
assert average_performance(pred, target, thr=0.5, k=2) \
|
|
|
|
|
== average_performance(pred, target, thr=0.5)
|
|
|
|
|
assert average_performance(
|
|
|
|
|
pred, target, thr=0.3) == pytest.approx(
|
|
|
|
|
(31.25, 43.75, 36.46, 33.33, 42.86, 37.50), rel=1e-2)
|
|
|
|
|
assert average_performance(
|
|
|
|
|
pred, target, k=2) == pytest.approx(
|
|
|
|
|
(43.75, 50.00, 46.67, 40.00, 57.14, 47.06), rel=1e-2)
|
