mirror of
https://github.com/PaddlePaddle/PaddleOCR.git
synced 2025-06-03 21:53:39 +08:00
68099c2d5b
* Add custom detection and recognition model usage instructions in re * update * Add custom detection and recognition model usage instructions in re * add db net for benchmark * rename benckmark to PaddleOCR_benchmark * add addict to req * rename
351 lines
13 KiB
Python
351 lines
13 KiB
Python
#!/usr/bin/env python
|
|
# -*- coding: utf-8 -*-
|
|
from collections import namedtuple
|
|
from . import rrc_evaluation_funcs
|
|
import Polygon as plg
|
|
import numpy as np
|
|
|
|
|
|
def default_evaluation_params():
|
|
"""
|
|
default_evaluation_params: Default parameters to use for the validation and evaluation.
|
|
"""
|
|
return {
|
|
'IOU_CONSTRAINT': 0.5,
|
|
'AREA_PRECISION_CONSTRAINT': 0.5,
|
|
'GT_SAMPLE_NAME_2_ID': 'gt_img_([0-9]+).txt',
|
|
'DET_SAMPLE_NAME_2_ID': 'res_img_([0-9]+).txt',
|
|
'LTRB':
|
|
False, # LTRB:2points(left,top,right,bottom) or 4 points(x1,y1,x2,y2,x3,y3,x4,y4)
|
|
'CRLF': False, # Lines are delimited by Windows CRLF format
|
|
'CONFIDENCES':
|
|
False, # Detections must include confidence value. AP will be calculated
|
|
'PER_SAMPLE_RESULTS':
|
|
True # Generate per sample results and produce data for visualization
|
|
}
|
|
|
|
|
|
def validate_data(gtFilePath, submFilePath, evaluationParams):
|
|
"""
|
|
Method validate_data: validates that all files in the results folder are correct (have the correct name contents).
|
|
Validates also that there are no missing files in the folder.
|
|
If some error detected, the method raises the error
|
|
"""
|
|
gt = rrc_evaluation_funcs.load_folder_file(
|
|
gtFilePath, evaluationParams['GT_SAMPLE_NAME_2_ID'])
|
|
|
|
subm = rrc_evaluation_funcs.load_folder_file(
|
|
submFilePath, evaluationParams['DET_SAMPLE_NAME_2_ID'], True)
|
|
|
|
# Validate format of GroundTruth
|
|
for k in gt:
|
|
rrc_evaluation_funcs.validate_lines_in_file(
|
|
k, gt[k], evaluationParams['CRLF'], evaluationParams['LTRB'], True)
|
|
|
|
# Validate format of results
|
|
for k in subm:
|
|
if (k in gt) == False:
|
|
raise Exception("The sample %s not present in GT" % k)
|
|
|
|
rrc_evaluation_funcs.validate_lines_in_file(
|
|
k, subm[k], evaluationParams['CRLF'], evaluationParams['LTRB'],
|
|
False, evaluationParams['CONFIDENCES'])
|
|
|
|
|
|
def evaluate_method(gtFilePath, submFilePath, evaluationParams):
|
|
"""
|
|
Method evaluate_method: evaluate method and returns the results
|
|
Results. Dictionary with the following values:
|
|
- method (required) Global method metrics. Ex: { 'Precision':0.8,'Recall':0.9 }
|
|
- samples (optional) Per sample metrics. Ex: {'sample1' : { 'Precision':0.8,'Recall':0.9 } , 'sample2' : { 'Precision':0.8,'Recall':0.9 }
|
|
"""
|
|
|
|
def polygon_from_points(points):
|
|
"""
|
|
Returns a Polygon object to use with the Polygon2 class from a list of 8 points: x1,y1,x2,y2,x3,y3,x4,y4
|
|
"""
|
|
resBoxes = np.empty([1, 8], dtype='int32')
|
|
resBoxes[0, 0] = int(points[0])
|
|
resBoxes[0, 4] = int(points[1])
|
|
resBoxes[0, 1] = int(points[2])
|
|
resBoxes[0, 5] = int(points[3])
|
|
resBoxes[0, 2] = int(points[4])
|
|
resBoxes[0, 6] = int(points[5])
|
|
resBoxes[0, 3] = int(points[6])
|
|
resBoxes[0, 7] = int(points[7])
|
|
pointMat = resBoxes[0].reshape([2, 4]).T
|
|
return plg.Polygon(pointMat)
|
|
|
|
def rectangle_to_polygon(rect):
|
|
resBoxes = np.empty([1, 8], dtype='int32')
|
|
resBoxes[0, 0] = int(rect.xmin)
|
|
resBoxes[0, 4] = int(rect.ymax)
|
|
resBoxes[0, 1] = int(rect.xmin)
|
|
resBoxes[0, 5] = int(rect.ymin)
|
|
resBoxes[0, 2] = int(rect.xmax)
|
|
resBoxes[0, 6] = int(rect.ymin)
|
|
resBoxes[0, 3] = int(rect.xmax)
|
|
resBoxes[0, 7] = int(rect.ymax)
|
|
|
|
pointMat = resBoxes[0].reshape([2, 4]).T
|
|
|
|
return plg.Polygon(pointMat)
|
|
|
|
def rectangle_to_points(rect):
|
|
points = [
|
|
int(rect.xmin), int(rect.ymax), int(rect.xmax), int(rect.ymax),
|
|
int(rect.xmax), int(rect.ymin), int(rect.xmin), int(rect.ymin)
|
|
]
|
|
return points
|
|
|
|
def get_union(pD, pG):
|
|
areaA = pD.area()
|
|
areaB = pG.area()
|
|
return areaA + areaB - get_intersection(pD, pG)
|
|
|
|
def get_intersection_over_union(pD, pG):
|
|
try:
|
|
return get_intersection(pD, pG) / get_union(pD, pG)
|
|
except:
|
|
return 0
|
|
|
|
def get_intersection(pD, pG):
|
|
pInt = pD & pG
|
|
if len(pInt) == 0:
|
|
return 0
|
|
return pInt.area()
|
|
|
|
def compute_ap(confList, matchList, numGtCare):
|
|
correct = 0
|
|
AP = 0
|
|
if len(confList) > 0:
|
|
confList = np.array(confList)
|
|
matchList = np.array(matchList)
|
|
sorted_ind = np.argsort(-confList)
|
|
confList = confList[sorted_ind]
|
|
matchList = matchList[sorted_ind]
|
|
for n in range(len(confList)):
|
|
match = matchList[n]
|
|
if match:
|
|
correct += 1
|
|
AP += float(correct) / (n + 1)
|
|
|
|
if numGtCare > 0:
|
|
AP /= numGtCare
|
|
|
|
return AP
|
|
|
|
perSampleMetrics = {}
|
|
|
|
matchedSum = 0
|
|
|
|
Rectangle = namedtuple('Rectangle', 'xmin ymin xmax ymax')
|
|
|
|
gt = rrc_evaluation_funcs.load_folder_file(
|
|
gtFilePath, evaluationParams['GT_SAMPLE_NAME_2_ID'])
|
|
subm = rrc_evaluation_funcs.load_folder_file(
|
|
submFilePath, evaluationParams['DET_SAMPLE_NAME_2_ID'], True)
|
|
|
|
numGlobalCareGt = 0
|
|
numGlobalCareDet = 0
|
|
|
|
arrGlobalConfidences = []
|
|
arrGlobalMatches = []
|
|
|
|
for resFile in gt:
|
|
|
|
gtFile = gt[resFile] # rrc_evaluation_funcs.decode_utf8(gt[resFile])
|
|
recall = 0
|
|
precision = 0
|
|
hmean = 0
|
|
|
|
detMatched = 0
|
|
|
|
iouMat = np.empty([1, 1])
|
|
|
|
gtPols = []
|
|
detPols = []
|
|
|
|
gtPolPoints = []
|
|
detPolPoints = []
|
|
|
|
# Array of Ground Truth Polygons' keys marked as don't Care
|
|
gtDontCarePolsNum = []
|
|
# Array of Detected Polygons' matched with a don't Care GT
|
|
detDontCarePolsNum = []
|
|
|
|
pairs = []
|
|
detMatchedNums = []
|
|
|
|
arrSampleConfidences = []
|
|
arrSampleMatch = []
|
|
sampleAP = 0
|
|
|
|
evaluationLog = ""
|
|
|
|
pointsList, _, transcriptionsList = rrc_evaluation_funcs.get_tl_line_values_from_file_contents(
|
|
gtFile, evaluationParams['CRLF'], evaluationParams['LTRB'], True,
|
|
False)
|
|
for n in range(len(pointsList)):
|
|
points = pointsList[n]
|
|
transcription = transcriptionsList[n]
|
|
dontCare = transcription == "###"
|
|
if evaluationParams['LTRB']:
|
|
gtRect = Rectangle(*points)
|
|
gtPol = rectangle_to_polygon(gtRect)
|
|
else:
|
|
gtPol = polygon_from_points(points)
|
|
gtPols.append(gtPol)
|
|
gtPolPoints.append(points)
|
|
if dontCare:
|
|
gtDontCarePolsNum.append(len(gtPols) - 1)
|
|
|
|
evaluationLog += "GT polygons: " + str(len(gtPols)) + (
|
|
" (" + str(len(gtDontCarePolsNum)) + " don't care)\n"
|
|
if len(gtDontCarePolsNum) > 0 else "\n")
|
|
|
|
if resFile in subm:
|
|
|
|
detFile = subm[
|
|
resFile] # rrc_evaluation_funcs.decode_utf8(subm[resFile])
|
|
|
|
pointsList, confidencesList, _ = rrc_evaluation_funcs.get_tl_line_values_from_file_contents(
|
|
detFile, evaluationParams['CRLF'], evaluationParams['LTRB'],
|
|
False, evaluationParams['CONFIDENCES'])
|
|
for n in range(len(pointsList)):
|
|
points = pointsList[n]
|
|
|
|
if evaluationParams['LTRB']:
|
|
detRect = Rectangle(*points)
|
|
detPol = rectangle_to_polygon(detRect)
|
|
else:
|
|
detPol = polygon_from_points(points)
|
|
detPols.append(detPol)
|
|
detPolPoints.append(points)
|
|
if len(gtDontCarePolsNum) > 0:
|
|
for dontCarePol in gtDontCarePolsNum:
|
|
dontCarePol = gtPols[dontCarePol]
|
|
intersected_area = get_intersection(dontCarePol, detPol)
|
|
pdDimensions = detPol.area()
|
|
precision = 0 if pdDimensions == 0 else intersected_area / pdDimensions
|
|
if (precision >
|
|
evaluationParams['AREA_PRECISION_CONSTRAINT']):
|
|
detDontCarePolsNum.append(len(detPols) - 1)
|
|
break
|
|
|
|
evaluationLog += "DET polygons: " + str(len(detPols)) + (
|
|
" (" + str(len(detDontCarePolsNum)) + " don't care)\n"
|
|
if len(detDontCarePolsNum) > 0 else "\n")
|
|
|
|
if len(gtPols) > 0 and len(detPols) > 0:
|
|
# Calculate IoU and precision matrixs
|
|
outputShape = [len(gtPols), len(detPols)]
|
|
iouMat = np.empty(outputShape)
|
|
gtRectMat = np.zeros(len(gtPols), np.int8)
|
|
detRectMat = np.zeros(len(detPols), np.int8)
|
|
for gtNum in range(len(gtPols)):
|
|
for detNum in range(len(detPols)):
|
|
pG = gtPols[gtNum]
|
|
pD = detPols[detNum]
|
|
iouMat[gtNum, detNum] = get_intersection_over_union(pD,
|
|
pG)
|
|
|
|
for gtNum in range(len(gtPols)):
|
|
for detNum in range(len(detPols)):
|
|
if gtRectMat[gtNum] == 0 and detRectMat[
|
|
detNum] == 0 and gtNum not in gtDontCarePolsNum and detNum not in detDontCarePolsNum:
|
|
if iouMat[gtNum, detNum] > evaluationParams[
|
|
'IOU_CONSTRAINT']:
|
|
gtRectMat[gtNum] = 1
|
|
detRectMat[detNum] = 1
|
|
detMatched += 1
|
|
pairs.append({'gt': gtNum, 'det': detNum})
|
|
detMatchedNums.append(detNum)
|
|
evaluationLog += "Match GT #" + str(
|
|
gtNum) + " with Det #" + str(detNum) + "\n"
|
|
|
|
if evaluationParams['CONFIDENCES']:
|
|
for detNum in range(len(detPols)):
|
|
if detNum not in detDontCarePolsNum:
|
|
# we exclude the don't care detections
|
|
match = detNum in detMatchedNums
|
|
|
|
arrSampleConfidences.append(confidencesList[detNum])
|
|
arrSampleMatch.append(match)
|
|
|
|
arrGlobalConfidences.append(confidencesList[detNum])
|
|
arrGlobalMatches.append(match)
|
|
|
|
numGtCare = (len(gtPols) - len(gtDontCarePolsNum))
|
|
numDetCare = (len(detPols) - len(detDontCarePolsNum))
|
|
if numGtCare == 0:
|
|
recall = float(1)
|
|
precision = float(0) if numDetCare > 0 else float(1)
|
|
sampleAP = precision
|
|
else:
|
|
recall = float(detMatched) / numGtCare
|
|
precision = 0 if numDetCare == 0 else float(detMatched) / numDetCare
|
|
if evaluationParams['CONFIDENCES'] and evaluationParams[
|
|
'PER_SAMPLE_RESULTS']:
|
|
sampleAP = compute_ap(arrSampleConfidences, arrSampleMatch,
|
|
numGtCare)
|
|
|
|
hmean = 0 if (precision + recall) == 0 else 2.0 * precision * recall / (
|
|
precision + recall)
|
|
|
|
matchedSum += detMatched
|
|
numGlobalCareGt += numGtCare
|
|
numGlobalCareDet += numDetCare
|
|
|
|
if evaluationParams['PER_SAMPLE_RESULTS']:
|
|
perSampleMetrics[resFile] = {
|
|
'precision': precision,
|
|
'recall': recall,
|
|
'hmean': hmean,
|
|
'pairs': pairs,
|
|
'AP': sampleAP,
|
|
'iouMat': [] if len(detPols) > 100 else iouMat.tolist(),
|
|
'gtPolPoints': gtPolPoints,
|
|
'detPolPoints': detPolPoints,
|
|
'gtDontCare': gtDontCarePolsNum,
|
|
'detDontCare': detDontCarePolsNum,
|
|
'evaluationParams': evaluationParams,
|
|
'evaluationLog': evaluationLog
|
|
}
|
|
|
|
# Compute MAP and MAR
|
|
AP = 0
|
|
if evaluationParams['CONFIDENCES']:
|
|
AP = compute_ap(arrGlobalConfidences, arrGlobalMatches, numGlobalCareGt)
|
|
|
|
methodRecall = 0 if numGlobalCareGt == 0 else float(
|
|
matchedSum) / numGlobalCareGt
|
|
methodPrecision = 0 if numGlobalCareDet == 0 else float(
|
|
matchedSum) / numGlobalCareDet
|
|
methodHmean = 0 if methodRecall + methodPrecision == 0 else 2 * methodRecall * methodPrecision / (
|
|
methodRecall + methodPrecision)
|
|
|
|
methodMetrics = {
|
|
'precision': methodPrecision,
|
|
'recall': methodRecall,
|
|
'hmean': methodHmean,
|
|
'AP': AP
|
|
}
|
|
|
|
resDict = {
|
|
'calculated': True,
|
|
'Message': '',
|
|
'method': methodMetrics,
|
|
'per_sample': perSampleMetrics
|
|
}
|
|
|
|
return resDict
|
|
|
|
|
|
def cal_recall_precison_f1(gt_path, result_path, show_result=False):
|
|
p = {'g': gt_path, 's': result_path}
|
|
result = rrc_evaluation_funcs.main_evaluation(p, default_evaluation_params,
|
|
validate_data,
|
|
evaluate_method, show_result)
|
|
return result['method']
|